From ea9e06762ebecb4ef17ac8b1fb92b3a5b966ff2b Mon Sep 17 00:00:00 2001
From: YangYanbin <warlock.yyb@alibaba-inc.com>
Date: Fri, 1 Nov 2024 15:15:53 +0800
Subject: [PATCH] Deployed 513594c with MkDocs version: 1.5.3

---
 index.html               |   2 +-
 releases/index.html      |   2 +-
 search/search_index.json |   2 +-
 sitemap.xml.gz           | Bin 127 -> 127 bytes
 4 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/index.html b/index.html
index 131c4afa..f7b02c4c 100644
--- a/index.html
+++ b/index.html
@@ -538,5 +538,5 @@ <h4 class="modal-title" id="keyboardModalLabel">Keyboard Shortcuts</h4>
 
 <!--
 MkDocs version : 1.5.3
-Build Date UTC : 2024-11-01 06:44:25.740157+00:00
+Build Date UTC : 2024-11-01 07:15:52.608631+00:00
 -->
diff --git a/releases/index.html b/releases/index.html
index 1a44e77d..91467fa2 100644
--- a/releases/index.html
+++ b/releases/index.html
@@ -540,7 +540,7 @@ <h3 id="_1">版本概要<a class="headerlink" href="#_1" title="Permanent link">
 <h3 id="_2">介绍<a class="headerlink" href="#_2" title="Permanent link">§</a></h3>
 <ul>
 <li>JindoSDK 更新 <a href="../jindosdk/oss-maven/">6.7.3 的 Maven 仓库</a> 和 <a href="../jindosdk/jindosdk_download/">下载地址</a>。</li>
-<li>打包时带入 jindoauth plugin，升级 jindosdk 后需将 <code>fs.jdo.plugin.dir</code> 指向 jindosdk plugins 目录。</li>
+<li>打包时带入 jindoauth plugin，支持升级 JindoSDK 后仍兼容老版本 EMR-Ranger，具体参见<a href="https://aliyun.github.io/alibabacloud-jindodata/upgrade/emr2_upgrade_jindosdk/#41-emr-ranger">升级文档</a>。</li>
 <li>nextarch classifier 修复 hudi 读写 log 时，对 AlreadyBeingCreatedException 异常处理。</li>
 <li>nextarch classifier 修复在使用 http 请求更新 STS 时，偶现卡住问题。</li>
 <li>nextarch classifier 修复开启 Ranger 后持续写入一个 OSS-HDFS 文件超过 1 小时后，文件无法继续写入的问题。</li>
diff --git a/search/search_index.json b/search/search_index.json
index 02dd7aa2..1a85686b 100644
--- a/search/search_index.json
+++ b/search/search_index.json
@@ -1 +1 @@
-{"config":{"indexing":"full","lang":["en"],"min_search_length":3,"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"JindoData \u4f7f\u7528\u6587\u6863 \u00a7 JindoData \u662f\u963f\u91cc\u4e91\u5f00\u6e90\u5927\u6570\u636e\u56e2\u961f\u81ea\u7814\u7684\u6570\u636e\u6e56\u5b58\u50a8\u52a0\u901f\u5957\u4ef6\uff0c\u9762\u5411\u5927\u6570\u636e\u548c AI \u751f\u6001\uff0c\u4e3a\u963f\u91cc\u4e91\u548c\u4e1a\u754c\u4e3b\u8981\u6570\u636e\u6e56\u5b58\u50a8\u7cfb\u7edf\u63d0\u4f9b\u5168\u65b9\u4f4d\u8bbf\u95ee\u52a0\u901f\u89e3\u51b3\u65b9\u6848\u3002 JindoData \u5957\u4ef6\u57fa\u4e8e\u7edf\u4e00\u67b6\u6784\u548c\u5185\u6838\u5b9e\u73b0\uff0c\u652f\u6301\u901a\u8fc7\u6570\u636e\u6e56\u4e07\u80fd\u7684 SDK\uff08JindoSDK\uff09 \u8bbf\u95ee OSS/OSS-HDFS\u3002JindoSDK \u517c\u5bb9 HCFS \u63a5\u53e3\uff0c\u5bf9\u8c61\u5b58\u50a8\u63a5\u53e3 \u548c POSIX \u63a5\u53e3\u3001\u4e5f\u652f\u6301 Python\u3001Tensorflow \u7b49\uff0c\u62e5\u6709\u5168\u9762\u517c\u5bb9\u7684\u751f\u6001\u5de5\u5177\uff08Jindo Cli\u3001JindoFuse\u3001JindoDistCp\uff09\u548c\u63d2\u4ef6\u652f\u6301\u3002 \u4e0b\u8f7d\u5b89\u88c5 JindoSDK \u00a7 JindoSDK \u662f\u8bbf\u95ee JindoData \u7ec4\u4ef6\u7684\u6807\u51c6\u5ba2\u6237\u7aef\uff0c\u8bf7\u53c2\u8003 JindoSDK \u4e0b\u8f7d \u548c JindoSDK \u5feb\u901f\u5165\u95e8 \u8fdb\u884c\u5b89\u88c5\u548c\u9a8c\u8bc1\u3002 \u5173\u4e8e\u591a\u5e73\u53f0\u652f\u6301\uff0c\u8bf7\u53c2\u8003 JindoSDK \u591a\u5e73\u53f0\u652f\u6301 \u3002 \u5347\u7ea7 JindoSDK \u00a7 JindoSDK \u662f\u6d3b\u8dc3\u66f4\u65b0\u7684\u5ba2\u6237\u7aef\uff0c\u6301\u7eed\u4e3a\u7528\u6237\u5e26\u6765\u963f\u91cc\u4e91EMR\u6570\u636e\u6e56\u7684\u6700\u65b0\u529f\u80fd\u548c\u6700\u4f73\u6027\u80fd\u3002 \u56e0\u6b64\u6211\u4eec\u63a8\u8350\u5ba2\u6237\u4f7f\u7528\u65b0\u7248 JindoSDK\uff0c\u4ee5\u83b7\u53d6\u6301\u7eed\u652f\u6301\u548c\u66f4\u597d\u7684\u4f7f\u7528\u4f53\u9a8c\u3002 \u6211\u4eec\u63d0\u4f9b\u4e86\u4fbf\u6377\u7684\u811a\u672c\u5e2e\u52a9\u7528\u6237\u5347\u7ea7\u96c6\u7fa4\u91cc\u7684 JindoSDK\uff0c\u8bf7\u53c2\u8003 JindoSDK\u5347\u7ea7\u6587\u6863 \u3002 \u5206\u573a\u666f\u4f7f\u7528\u8bf4\u660e \u00a7 \u5728 Hadoop \u751f\u6001\u4f7f\u7528JindoSDK \u5728 AI \u751f\u6001\u4f7f\u7528JindoSDK Jindo Python SDK \u4f7f\u7528 Jindo TensorFlow Connector \u4f7f\u7528 JindoFuse \u4f7f\u7528 Jindo Flink Sink \u4f7f\u7528 \u5728 Fluid \u4e2d\u4f7f\u7528 JindoRuntime JindoData \u9274\u6743\u65b9\u6848 \u6570\u636e\u62f7\u8d1d\u5de5\u5177 JindoDistCp \u6570\u4ed3\u8fc1\u79fb\u5de5\u5177 JindoTable MoveTo \u6570\u4ed3\u5206\u5c42\u5b58\u50a8\u7ba1\u7406\u5de5\u5177 JindoTable SetStorage OSS-HDFS\u670d\u52a1\uff08JindoFS\uff09\u5ba2\u6237\u7aef\u5de5\u5177 \u5e38\u89c1\u95ee\u9898 \u00a7 \u8bf7\u53c2\u89c1 JindoData \u5e38\u89c1\u95ee\u9898 \u5386\u53f2\u7248\u672c \u00a7 \u8bf7\u53c2\u89c1 JindoSDK \u7248\u672c\u8bb0\u5f55 \u3002","title":"\u9996\u9875"},{"location":"#jindodata","text":"JindoData \u662f\u963f\u91cc\u4e91\u5f00\u6e90\u5927\u6570\u636e\u56e2\u961f\u81ea\u7814\u7684\u6570\u636e\u6e56\u5b58\u50a8\u52a0\u901f\u5957\u4ef6\uff0c\u9762\u5411\u5927\u6570\u636e\u548c AI \u751f\u6001\uff0c\u4e3a\u963f\u91cc\u4e91\u548c\u4e1a\u754c\u4e3b\u8981\u6570\u636e\u6e56\u5b58\u50a8\u7cfb\u7edf\u63d0\u4f9b\u5168\u65b9\u4f4d\u8bbf\u95ee\u52a0\u901f\u89e3\u51b3\u65b9\u6848\u3002 JindoData \u5957\u4ef6\u57fa\u4e8e\u7edf\u4e00\u67b6\u6784\u548c\u5185\u6838\u5b9e\u73b0\uff0c\u652f\u6301\u901a\u8fc7\u6570\u636e\u6e56\u4e07\u80fd\u7684 SDK\uff08JindoSDK\uff09 \u8bbf\u95ee OSS/OSS-HDFS\u3002JindoSDK \u517c\u5bb9 HCFS \u63a5\u53e3\uff0c\u5bf9\u8c61\u5b58\u50a8\u63a5\u53e3 \u548c POSIX \u63a5\u53e3\u3001\u4e5f\u652f\u6301 Python\u3001Tensorflow \u7b49\uff0c\u62e5\u6709\u5168\u9762\u517c\u5bb9\u7684\u751f\u6001\u5de5\u5177\uff08Jindo Cli\u3001JindoFuse\u3001JindoDistCp\uff09\u548c\u63d2\u4ef6\u652f\u6301\u3002","title":"JindoData \u4f7f\u7528\u6587\u6863"},{"location":"#jindosdk","text":"JindoSDK \u662f\u8bbf\u95ee JindoData \u7ec4\u4ef6\u7684\u6807\u51c6\u5ba2\u6237\u7aef\uff0c\u8bf7\u53c2\u8003 JindoSDK \u4e0b\u8f7d \u548c JindoSDK \u5feb\u901f\u5165\u95e8 \u8fdb\u884c\u5b89\u88c5\u548c\u9a8c\u8bc1\u3002 \u5173\u4e8e\u591a\u5e73\u53f0\u652f\u6301\uff0c\u8bf7\u53c2\u8003 JindoSDK \u591a\u5e73\u53f0\u652f\u6301 \u3002","title":"\u4e0b\u8f7d\u5b89\u88c5 JindoSDK"},{"location":"#jindosdk_1","text":"JindoSDK \u662f\u6d3b\u8dc3\u66f4\u65b0\u7684\u5ba2\u6237\u7aef\uff0c\u6301\u7eed\u4e3a\u7528\u6237\u5e26\u6765\u963f\u91cc\u4e91EMR\u6570\u636e\u6e56\u7684\u6700\u65b0\u529f\u80fd\u548c\u6700\u4f73\u6027\u80fd\u3002 \u56e0\u6b64\u6211\u4eec\u63a8\u8350\u5ba2\u6237\u4f7f\u7528\u65b0\u7248 JindoSDK\uff0c\u4ee5\u83b7\u53d6\u6301\u7eed\u652f\u6301\u548c\u66f4\u597d\u7684\u4f7f\u7528\u4f53\u9a8c\u3002 \u6211\u4eec\u63d0\u4f9b\u4e86\u4fbf\u6377\u7684\u811a\u672c\u5e2e\u52a9\u7528\u6237\u5347\u7ea7\u96c6\u7fa4\u91cc\u7684 JindoSDK\uff0c\u8bf7\u53c2\u8003 JindoSDK\u5347\u7ea7\u6587\u6863 \u3002","title":"\u5347\u7ea7 JindoSDK"},{"location":"#_1","text":"\u5728 Hadoop \u751f\u6001\u4f7f\u7528JindoSDK \u5728 AI \u751f\u6001\u4f7f\u7528JindoSDK Jindo Python SDK \u4f7f\u7528 Jindo TensorFlow Connector \u4f7f\u7528 JindoFuse \u4f7f\u7528 Jindo Flink Sink \u4f7f\u7528 \u5728 Fluid \u4e2d\u4f7f\u7528 JindoRuntime JindoData \u9274\u6743\u65b9\u6848 \u6570\u636e\u62f7\u8d1d\u5de5\u5177 JindoDistCp \u6570\u4ed3\u8fc1\u79fb\u5de5\u5177 JindoTable MoveTo \u6570\u4ed3\u5206\u5c42\u5b58\u50a8\u7ba1\u7406\u5de5\u5177 JindoTable SetStorage OSS-HDFS\u670d\u52a1\uff08JindoFS\uff09\u5ba2\u6237\u7aef\u5de5\u5177","title":"\u5206\u573a\u666f\u4f7f\u7528\u8bf4\u660e"},{"location":"#_2","text":"\u8bf7\u53c2\u89c1 JindoData \u5e38\u89c1\u95ee\u9898","title":"\u5e38\u89c1\u95ee\u9898"},{"location":"#_3","text":"\u8bf7\u53c2\u89c1 JindoSDK \u7248\u672c\u8bb0\u5f55 \u3002","title":"\u5386\u53f2\u7248\u672c"},{"location":"faq/","text":"JindoData \u5e38\u89c1\u95ee\u9898 \u00a7 \u5e38\u89c1\u95ee\u9898 \u00a7 JindoData \u5957\u4ef6\u548c\u963f\u91cc\u4e91 EMR \u662f\u4ec0\u4e48\u5173\u7cfb\uff1f JindoData \u5957\u4ef6\u662f\u963f\u91cc\u4e91\u5f00\u6e90\u5927\u6570\u636e JindoData \u56e2\u961f\u5f00\u53d1\uff0c\u9762\u5411\u5f00\u6e90\u751f\u6001\u548c\u793e\u533a\u5168\u9762\u5f00\u653e\u548c\u63d0\u4f9b\uff0c\u9664\u4e86 EMR \u4ea7\u54c1\u9ed8\u8ba4\u96c6\u6210\uff0c\u8fd8\u652f\u6301\u963f\u91cc\u4e91\u4e0a\u591a\u79cd\u573a\u666f\u90e8\u7f72\u4f7f\u7528\uff0c\u5305\u62ec ECS \u81ea\u5efa\u96c6\u7fa4\u3001ACK \u4e91\u539f\u751f Fluid \u96c6\u6210\uff0c\u751a\u81f3 IDC \u96c6\u7fa4\u73af\u5883\u3002\u963f\u91cc\u4e91 EMR \u4ea7\u54c1\u9488\u5bf9\u591a\u79cd\u573a\u666f\u63d0\u4f9b\u7684\u591a\u79cd\u96c6\u7fa4\u7c7b\u578b\uff08DataLake\u3001DataFlow\u3001DataScience \u548c OLAP\uff09\u4f1a\u9ed8\u8ba4\u96c6\u6210\u6240\u9700\u8981\u7684 JindoData \u76f8\u5e94\u7ec4\u4ef6\uff0c\u63d0\u4f9b\u66f4\u52a0\u4f18\u5316\u3001\u6613\u7528\u7684\u4f7f\u7528\u4f53\u9a8c\u3002 JindoData \u5957\u4ef6\u548c EMR SmartData \u7ec4\u4ef6\u662f\u4ec0\u4e48\u5173\u7cfb\uff1f JindoData \u5957\u4ef6\u524d\u8eab\u5c31\u662f\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1 SmartData \u7ec4\u4ef6\uff0c\u662f\u5bf9 SmartData \u7684\u67b6\u6784\u5347\u7ea7\uff0c\u66f4\u52a0\u6a21\u5757\u5316\uff0c\u9762\u5411\u793e\u533a\u751f\u6001\u7cfb\u7edf\u66f4\u52a0\u5f00\u653e\u548c\u517c\u5bb9\u3002 JindoData \u5957\u4ef6\u548c JindoFS \u662f\u4ec0\u4e48\u5173\u7cfb\uff1f JindoData \u5957\u4ef6\u57fa\u4e8e\u7edf\u4e00\u7684\u67b6\u6784\u548c\u5185\u6838\u5b9e\u73b0\uff0c\u4e0d\u4ec5\u5305\u62ec JindoFS\uff08\u5b58\u50a8\u7cfb\u7edf\uff09\uff0c\u8fd8\u5305\u542b JindoFSx\uff08\u7f13\u5b58\u7cfb\u7edf\uff09\uff0cJindoSDK \u548c\u751f\u6001\u5de5\u5177\u3001\u63d2\u4ef6\u3002 JindoData \u5957\u4ef6\u548c Jindo \u662f\u4ec0\u4e48\u5173\u7cfb\uff1f Jindo \u7ea6\u7b49\u4e8e JindoData\uff0c\u662f\u540e\u8005\u7684\u7b80\u4fbf\u53eb\u6cd5\u3002 JindoFS \u548c OSS-HDFS \u670d\u52a1\u4e4b\u95f4\u7684\u5173\u7cfb OSS-HDFS \u670d\u52a1\uff08\u6709\u65f6\u4e5f\u63d0 JindoFS \u670d\u52a1\uff09\u662f JindoFS \u5b58\u50a8\u7cfb\u7edf\u5728\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u670d\u52a1\u5316\u90e8\u7f72\u5f62\u6001\uff0c\u548c OSS \u5bf9\u8c61\u5b58\u50a8\u4ea7\u54c1\u6df1\u5ea6\u878d\u5408\uff0c\u4f5c\u4e3a\u4e00\u79cd\u65b0\u7684 bucket \u7c7b\u578b\uff0c\u5f00\u7bb1\u5373\u7528\u3002 JindoFS \u670d\u52a1\u5316\u6210\u4e3a OSS-HDFS \u670d\u52a1\uff0c\u4e3a\u4ec0\u4e48\u53eb HDFS\uff1f JindoFS \u5b58\u50a8\u7cfb\u7edf\u4e8c\u8fdb\u5236\u517c\u5bb9 Apache HDFS\uff0c\u5e76\u4e14\u57fa\u672c\u529f\u80fd\u5b8c\u5168\u5bf9\u9f50\uff0c\u63d0\u4f9b\u4f18\u5316\u7684 HDFS \u4f7f\u7528\u4f53\u9a8c\uff0c\u53ef\u5b9e\u73b0 HDFS \u81ea\u5efa\u96c6\u7fa4\u5e73\u8fc1\u4e0a\u4e91\u4e0a\u963f\u91cc\u4e91 OSS\u3002 \u5386\u53f2\u7248\u672c\u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.12 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.11 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.10 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.9 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.8 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.7 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.6 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.5 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.4 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.3 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.2 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.1 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.0 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.5.1 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.5.0 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.4.0 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.3.0 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.2.0 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.1.0 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.0.0 \u5df2\u77e5\u95ee\u9898 \u00a7","title":"\u5e38\u89c1\u95ee\u9898"},{"location":"faq/#jindodata","text":"","title":"JindoData \u5e38\u89c1\u95ee\u9898"},{"location":"faq/#_1","text":"JindoData \u5957\u4ef6\u548c\u963f\u91cc\u4e91 EMR \u662f\u4ec0\u4e48\u5173\u7cfb\uff1f JindoData \u5957\u4ef6\u662f\u963f\u91cc\u4e91\u5f00\u6e90\u5927\u6570\u636e JindoData \u56e2\u961f\u5f00\u53d1\uff0c\u9762\u5411\u5f00\u6e90\u751f\u6001\u548c\u793e\u533a\u5168\u9762\u5f00\u653e\u548c\u63d0\u4f9b\uff0c\u9664\u4e86 EMR \u4ea7\u54c1\u9ed8\u8ba4\u96c6\u6210\uff0c\u8fd8\u652f\u6301\u963f\u91cc\u4e91\u4e0a\u591a\u79cd\u573a\u666f\u90e8\u7f72\u4f7f\u7528\uff0c\u5305\u62ec ECS \u81ea\u5efa\u96c6\u7fa4\u3001ACK \u4e91\u539f\u751f Fluid \u96c6\u6210\uff0c\u751a\u81f3 IDC \u96c6\u7fa4\u73af\u5883\u3002\u963f\u91cc\u4e91 EMR \u4ea7\u54c1\u9488\u5bf9\u591a\u79cd\u573a\u666f\u63d0\u4f9b\u7684\u591a\u79cd\u96c6\u7fa4\u7c7b\u578b\uff08DataLake\u3001DataFlow\u3001DataScience \u548c OLAP\uff09\u4f1a\u9ed8\u8ba4\u96c6\u6210\u6240\u9700\u8981\u7684 JindoData \u76f8\u5e94\u7ec4\u4ef6\uff0c\u63d0\u4f9b\u66f4\u52a0\u4f18\u5316\u3001\u6613\u7528\u7684\u4f7f\u7528\u4f53\u9a8c\u3002 JindoData \u5957\u4ef6\u548c EMR SmartData \u7ec4\u4ef6\u662f\u4ec0\u4e48\u5173\u7cfb\uff1f JindoData \u5957\u4ef6\u524d\u8eab\u5c31\u662f\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1 SmartData \u7ec4\u4ef6\uff0c\u662f\u5bf9 SmartData \u7684\u67b6\u6784\u5347\u7ea7\uff0c\u66f4\u52a0\u6a21\u5757\u5316\uff0c\u9762\u5411\u793e\u533a\u751f\u6001\u7cfb\u7edf\u66f4\u52a0\u5f00\u653e\u548c\u517c\u5bb9\u3002 JindoData \u5957\u4ef6\u548c JindoFS \u662f\u4ec0\u4e48\u5173\u7cfb\uff1f JindoData \u5957\u4ef6\u57fa\u4e8e\u7edf\u4e00\u7684\u67b6\u6784\u548c\u5185\u6838\u5b9e\u73b0\uff0c\u4e0d\u4ec5\u5305\u62ec JindoFS\uff08\u5b58\u50a8\u7cfb\u7edf\uff09\uff0c\u8fd8\u5305\u542b JindoFSx\uff08\u7f13\u5b58\u7cfb\u7edf\uff09\uff0cJindoSDK \u548c\u751f\u6001\u5de5\u5177\u3001\u63d2\u4ef6\u3002 JindoData \u5957\u4ef6\u548c Jindo \u662f\u4ec0\u4e48\u5173\u7cfb\uff1f Jindo \u7ea6\u7b49\u4e8e JindoData\uff0c\u662f\u540e\u8005\u7684\u7b80\u4fbf\u53eb\u6cd5\u3002 JindoFS \u548c OSS-HDFS \u670d\u52a1\u4e4b\u95f4\u7684\u5173\u7cfb OSS-HDFS \u670d\u52a1\uff08\u6709\u65f6\u4e5f\u63d0 JindoFS \u670d\u52a1\uff09\u662f JindoFS \u5b58\u50a8\u7cfb\u7edf\u5728\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u670d\u52a1\u5316\u90e8\u7f72\u5f62\u6001\uff0c\u548c OSS \u5bf9\u8c61\u5b58\u50a8\u4ea7\u54c1\u6df1\u5ea6\u878d\u5408\uff0c\u4f5c\u4e3a\u4e00\u79cd\u65b0\u7684 bucket \u7c7b\u578b\uff0c\u5f00\u7bb1\u5373\u7528\u3002 JindoFS \u670d\u52a1\u5316\u6210\u4e3a OSS-HDFS \u670d\u52a1\uff0c\u4e3a\u4ec0\u4e48\u53eb HDFS\uff1f JindoFS \u5b58\u50a8\u7cfb\u7edf\u4e8c\u8fdb\u5236\u517c\u5bb9 Apache HDFS\uff0c\u5e76\u4e14\u57fa\u672c\u529f\u80fd\u5b8c\u5168\u5bf9\u9f50\uff0c\u63d0\u4f9b\u4f18\u5316\u7684 HDFS \u4f7f\u7528\u4f53\u9a8c\uff0c\u53ef\u5b9e\u73b0 HDFS \u81ea\u5efa\u96c6\u7fa4\u5e73\u8fc1\u4e0a\u4e91\u4e0a\u963f\u91cc\u4e91 OSS\u3002","title":"\u5e38\u89c1\u95ee\u9898"},{"location":"faq/#_2","text":"","title":"\u5386\u53f2\u7248\u672c\u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-4612","text":"","title":"JindoSDK 4.6.12 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-4611","text":"","title":"JindoSDK 4.6.11 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-4610","text":"","title":"JindoSDK 4.6.10 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-469","text":"","title":"JindoSDK 4.6.9 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-468","text":"","title":"JindoSDK 4.6.8 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-467","text":"","title":"JindoSDK 4.6.7 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-466","text":"","title":"JindoSDK 4.6.6 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-465","text":"","title":"JindoSDK 4.6.5 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-464","text":"","title":"JindoSDK 4.6.4 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-463","text":"","title":"JindoSDK 4.6.3 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-462","text":"","title":"JindoSDK 4.6.2 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-461","text":"","title":"JindoSDK 4.6.1 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-460","text":"","title":"JindoSDK 4.6.0 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-451","text":"","title":"JindoSDK 4.5.1 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-450","text":"","title":"JindoSDK 4.5.0 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-440","text":"","title":"JindoSDK 4.4.0 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-430","text":"","title":"JindoSDK 4.3.0 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-420","text":"","title":"JindoSDK 4.2.0 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-410","text":"","title":"JindoSDK 4.1.0 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-400","text":"","title":"JindoSDK 4.0.0 \u5df2\u77e5\u95ee\u9898"},{"location":"releases/","text":"JindoSDK \u7248\u672c\u8bb0\u5f55 \u00a7 6.7.3\uff0c2024-11-01 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.7.3 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.7.3 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u6253\u5305\u65f6\u5e26\u5165 jindoauth plugin\uff0c\u5347\u7ea7 jindosdk \u540e\u9700\u5c06 fs.jdo.plugin.dir \u6307\u5411 jindosdk plugins \u76ee\u5f55\u3002 nextarch classifier \u4fee\u590d hudi \u8bfb\u5199 log \u65f6\uff0c\u5bf9 AlreadyBeingCreatedException \u5f02\u5e38\u5904\u7406\u3002 nextarch classifier \u4fee\u590d\u5728\u4f7f\u7528 http \u8bf7\u6c42\u66f4\u65b0 STS \u65f6\uff0c\u5076\u73b0\u5361\u4f4f\u95ee\u9898\u3002 nextarch classifier \u4fee\u590d\u5f00\u542f Ranger \u540e\u6301\u7eed\u5199\u5165\u4e00\u4e2a OSS-HDFS \u6587\u4ef6\u8d85\u8fc7 1 \u5c0f\u65f6\u540e\uff0c\u6587\u4ef6\u65e0\u6cd5\u7ee7\u7eed\u5199\u5165\u7684\u95ee\u9898\u3002 6.4.1\uff0c2024-10-23 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.4.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.4.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d\u5076\u73b0 Signer V4 \u7b7e\u540d\u95ee\u9898\uff0c\u5c06\u7b7e\u540d\u65f6\u95f4\u6233\u8f6c\u6362\u51fd\u6570 localtime\u3001gmtime \u66ff\u6362\u4e3a localtime_r\u3001gmtime_r \u907f\u514d\u5e76\u53d1\u95ee\u9898\u3002 \u4fee\u590d libjindosdk_c.so \u8bbf\u95ee OSS-HDFS \u5076\u73b0 crash \u95ee\u9898\uff0c\u53ca\u91cd\u8bd5\u5931\u8d25\u95ee\u9898\u3002 \u4fee\u590d hudi \u8bfb\u5199 log \u65f6\uff0c\u5bf9 AlreadyBeingCreatedException \u5f02\u5e38\u5904\u7406\u3002 \u4fee\u590d JindoOssFileSystem \u7684 Delegation Token Renew \u673a\u5236\u3002 \u4fee\u590d JindoCommitter \u8bbf\u95ee OSS-HDFS \u672a\u6e05\u7406 task \u6b8b\u7559\u7684\u4e34\u65f6\u76ee\u5f55\u3002 \u63d0\u5347 JindoSDK \u5bf9\u4e8e\u4f4e\u4e8e Hadoop 2.8.x \u7248\u672c\u63a5\u53e3\u7684\u517c\u5bb9\u6027\uff0c\u5982 CallerContext \u53ca FsServerDefaults \u3002 \u4f18\u5316 classloader \u52a0\u8f7d\uff0c\u5728\u591a\u4e2a classloader \u52a0\u8f7d jindosdk \u65f6\u7684 so \u6b8b\u7559\u95ee\u9898\uff0c\u5e76\u4e14\u5728\u914d\u7f6e\u4e86 java.io.tmpdir \u65f6\uff0c\u9ed8\u8ba4\u89e3\u538b\u81f3\u914d\u7f6e\u76ee\u5f55\u4e0b\u3002 6.7.2\uff0c2024-10-21 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.7.2 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.7.2 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u4f18\u5316 metrics \u7edf\u8ba1\u4fe1\u606f\u3002 nextarch classifier \u4fee\u590d\u5728\u5199 OSS-HDFS \u573a\u666f flush \u540e\u51fa\u73b0\u5c0f\u5757\u7684\u95ee\u9898\u3002 nextarch classifier \u4fee\u590d\u540e\u53f0 metrics \u7ebf\u7a0b\u5f15\u53d1\u7684\u5076\u73b0 crash \u95ee\u9898\u3002 Pyjindo \u4fee\u590d 6.7.x \u7248\u672c\u62a5 invalid pointer \u95ee\u9898\uff0c6.7.0 \u53ca 6.7.1 \u7248\u672c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u73af\u5883\u53d8\u91cf JINDO_STAT_MEMORY=0 \u7ed5\u8fc7\u3002 6.7.1\uff0c2024-10-15 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.7.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.7.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u4f18\u5316\u9ed8\u8ba4\u5e76\u53d1\u4e0b\u7684\u5199\u6027\u80fd\u3002 \u4fee\u590d JindoCommitter \u8bbf\u95ee OSS-HDFS \u672a\u6e05\u7406 task \u6b8b\u7559\u7684\u4e34\u65f6\u76ee\u5f55\u3002 jindo-fuse \u4fee\u590d\u5728 s3 scheme \u4e0a\u8ffd\u52a0\u5199\u7684\u652f\u6301\u3002 6.7.0\uff0c2024-09-29 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.7.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.7.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u652f\u6301\u7edf\u4e00\u7ba1\u7406\u5185\u5b58\uff0c\u4f7f\u7528 fs.jdo.memory.pool.size.max.mb \u914d\u7f6e\u6700\u5927\u4f7f\u7528\u5185\u5b58\uff0c\u5e76\u652f\u6301\u5185\u5b58\u5b9e\u65f6\u4f7f\u7528 metrics\u3002 nextarch classifier \u652f\u6301 io buffer \u5185\u5b58\u4f7f\u7528\u63a7\u5236\uff0c\u4f7f\u7528 fs.jdo.memory.io.buffer.size.max.ratio \u914d\u7f6e\u6700\u5927\u4f7f\u7528\u5185\u5b58\u6c60\u7684\u6bd4\u4f8b\u3002 nextarch classifier \u652f\u6301 Golang SDK\u3002 nextarch classifier \u652f\u6301 OSS \u5bf9\u8c61\u5b58\u50a8\u6df1\u51b7\u5f52\u6863\u3002 nextarch classifier \u4f18\u5316 pread \u63a5\u53e3\uff0c\u5185\u5b58\u96f6\u62f7\u8d1d\u3002 nextarch classifier \u4f18\u5316 sendfile \u5728\u5199\u5c0f\u6587\u4ef6\u573a\u666f\u7684\u4f7f\u7528\u3002 nextarch classifier \u4f18\u5316 metrics \u6846\u67b6\uff0c\u652f\u6301\u6bcf\u6b21\u8f93\u51fa\u5230\u5355\u72ec\u6587\u4ef6\uff0c\u65b9\u4fbf\u91c7\u96c6\u3002 nextarch classifier \u4f18\u5316 distjob \u6846\u4e2d\uff0c\u6392\u9664 log4j \u76f8\u5173\u4f9d\u8d56\u3002 nextarch classifier \u4f18\u5316 classloader \u52a0\u8f7d\uff0c\u5728\u591a\u4e2a classloader \u52a0\u8f7d jindosdk \u65f6\u7684 so \u6b8b\u7559\u95ee\u9898\uff0c\u5e76\u4e14\u5728\u914d\u7f6e\u4e86 java.io.tmpdir \u65f6\uff0c\u9ed8\u8ba4\u89e3\u538b\u81f3\u914d\u7f6e\u76ee\u5f55\u4e0b\u3002 nextarch classifier \u4fee\u590d OSS-HDFS StoragePolicy \u76f8\u5173\u63a5\u53e3\u3002 nextarch classifier \u4fee\u590d OSS \u5bf9\u8c61\u5b58\u50a8\uff0cListDirectory \u8fd4\u56de\u7ed3\u679c\u7684 Mtime \u5355\u4f4d\u3002 nextarch classifier \u4fee\u590d JindoCommitter \u5728 Cleanup \u9636\u6bb5\uff0c\u4e0d\u5e94\u629b\u51fa\u5f02\u5e38\u3002 nextarch classifier \u4fee\u590d DLF \u76f8\u5173 CredentialProvider \u4f7f\u7528 MagicCommitter \u65f6\u7684\u6743\u9650\u95ee\u9898\u3002 nextarch classifier \u4fee\u590d JindoCache \u7684\u82e5\u5e72\u95ee\u9898\u3002 6.6.3\uff0c2024-09-14 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.6.3 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.6.3 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u4fee\u590d\u5199\u573a\u666f\u5c0f\u6982\u7387\u56e0 sendfile \u51fa\u73b0\u9519\u8bef\u672a\u91cd\u8bd5\uff0c\u5bfc\u81f4\u6587\u4ef6 close \u65f6\u62a5\u5931\u8d25\u7684\u60c5\u51b5\u3002 6.6.2\uff0c2024-09-12 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.6.2 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.6.2 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u8bbf\u95ee OSS \u652f\u6301\u6df1\u51b7\u5f52\u6863 nextarch classifier \u4fee\u590d DLF \u76f8\u5173 CredentialProvider \u4f7f\u7528 MagicCommitter \u65f6\u7684\u6743\u9650\u95ee\u9898\u3002 6.6.1\uff0c2024-09-03 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.6.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.6.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u4f18\u5316\u5728\u591a\u4e2a classloader \u52a0\u8f7d jindosdk \u65f6\u7684 so \u6b8b\u7559\u95ee\u9898\uff0c\u5e76\u4e14\u5728\u914d\u7f6e\u4e86 java.io.tmpdir \u65f6\uff0c\u9ed8\u8ba4\u89e3\u538b\u81f3\u914d\u7f6e\u76ee\u5f55\u4e0b\u3002 nextarch classifier \u4fee\u590d\u8bbf\u95ee JindoCache \u7684\u82e5\u5e72\u95ee\u9898\u3002 nextarch classifier \u4f18\u5316 metrics \u6846\u67b6\u3002 6.6.0\uff0c2024-08-25 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.6.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.6.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u4f18\u5316 io \u6027\u80fd\uff0c\u4f7f\u7528\u534f\u7a0b\u5f02\u6b65\u5316\uff0c\u76f8\u540c\u914d\u7f6e\u4e0b\u652f\u6301\u66f4\u9ad8\u5e76\u53d1\u3002\u5199\u573a\u666f\u4f7f\u7528 sendfile \u5b9e\u73b0\u96f6\u62f7\u8d1d\u4f18\u5316\uff0c\u8282\u7701\u5185\u5b58\uff0c\u5e76\u63d0\u5347\u6027\u80fd\u3002 nextarch classifier \u652f\u6301\u5355\u72ec\u6307\u5b9a io \u8d85\u65f6\u65f6\u95f4\u3002 nextarch classifier \u652f\u6301 metrics \u6846\u67b6\u3002 \u4fee\u590d nextarch classifier \u5bf9 JindoCache \u652f\u6301\u95ee\u9898\u3002 jindo-fuse \u9ed8\u8ba4\u6539\u4e3a\u4f7f\u7528 nextarch \u5b9e\u73b0\u3002 6.5.4\uff0c2024-08-20 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.5.4 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.5.4 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4f18\u5316 flink connector \u4f9d\u8d56\uff0c\u5bf9\u90e8\u5206\u4f9d\u8d56\u8fdb\u884c shade\u3002 6.5.3\uff0c2024-08-15 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.5.3 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.5.3 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d JindoOssFileSystem \u7684 Delegation Token Renew \u673a\u5236\u3002 6.5.2\uff0c2024-08-12 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.5.2 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.5.2 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d getfacl \u7f3a\u5c11 entries \u95ee\u9898\u3002 6.5.1\uff0c2024-08-05 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.5.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.5.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u63d0\u5347 JindoSDK \u5bf9\u4e8e\u4f4e\u4e8e Hadoop 2.8.x \u7248\u672c\u63a5\u53e3\u7684\u517c\u5bb9\u6027\uff0c\u5982 CallerContext \u53ca FsServerDefaults \u3002 \u4f18\u5316 JindoCommitter \u6027\u80fd\u3002 \u4f18\u5316\u5185\u5b58\u4e0d\u8db3\u65f6\u7684\u9884\u8bfb\u7b97\u6cd5\uff0c\u56de\u6536\u4e0d\u4f7f\u7528\u7684\u5185\u5b58\u3002 \u4f18\u5316 plugin \u52a0\u8f7d\u65b9\u5f0f\uff0c fs.jdo.plugin.dir \u652f\u6301\u4ee5\u591a\u8def\u5f84\u52a0\u8f7d\uff0c\u5982 /dir1,/dir2 \u4fee\u590d list \u5bf9\u8c61\u5b58\u50a8\u65f6\uff0c\u8def\u5f84\u4e2d\u5e26\u6709 // \uff0c\u53ef\u80fd\u4f1alist\u51fa\u81ea\u8eab\u7684\u95ee\u9898 \u4fee\u590d jindo-dependence-shaded.jar \u5b89\u5168\u6027\u95ee\u9898\uff0c\u53bb\u9664 log4j \u53ca apache commons text \u4f9d\u8d56 \u4fee\u590d nextarch classifier \u8dd1 hadoop-compat-bench \u82e5\u5e72\u95ee\u9898\u3002 \u4fee\u590d nextarch classifier \u5bf9 RootPolicy \u652f\u6301\u95ee\u9898\u3002 6.5.0\uff0c2024-07-12 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.5.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.5.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d\u5076\u73b0 Signer V4 \u7b7e\u540d\u95ee\u9898\uff0c\u5c06\u7b7e\u540d\u65f6\u95f4\u6233\u8f6c\u6362\u51fd\u6570 localtime\u3001gmtime \u66ff\u6362\u4e3a localtime_r\u3001gmtime_r \u907f\u514d\u5e76\u53d1\u95ee\u9898\u3002 \u4fee\u590d libjindosdk_c.so \u8bbf\u95ee OSS-HDFS \u5076\u73b0 crash \u95ee\u9898\uff0c\u53ca\u91cd\u8bd5\u5931\u8d25\u95ee\u9898\u3002 jindo-core.jar nextarch classifer \u652f\u6301\u65b0\u7248\u5185\u6838\u3002 \u4f18\u5316\u5185\u5b58\u4e0d\u8db3\u65f6\u7684\u9884\u8bfb\u7b97\u6cd5\uff0c\u9632\u6b62\u5185\u5b58\u8fc7\u5ea6\u62a2\u5360\u3002 commiter \u652f\u6301 setXAttr\u3002 6.3.5\uff0c2024-06-28 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.3.5 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.3.5 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d\u5076\u73b0 Signer V4 \u7b7e\u540d\u95ee\u9898\uff0c\u5c06\u7b7e\u540d\u65f6\u95f4\u6233\u8f6c\u6362\u51fd\u6570 localtime\u3001gmtime \u66ff\u6362\u4e3a localtime_r\u3001gmtime_r \u907f\u514d\u5e76\u53d1\u95ee\u9898\u3002 \u4fee\u590d libjindosdk_c.so \u8bbf\u95ee OSS-HDFS \u5076\u73b0 crash \u95ee\u9898\uff0c\u53ca\u91cd\u8bd5\u5931\u8d25\u95ee\u9898\u3002 6.4.0\uff0c2024-05-16 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.4.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.4.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 JindoSDK \u652f\u6301 CallerContext\u3002 JindoSDK \u652f\u6301\u5199\u65f6 flush\u3002 \u4fee\u590d concat \u95ee\u9898\u3002 \u652f\u6301 RDMA \u8bbf\u95ee JindoCache\u3002 \u4f18\u5316\u4e86\u8bfb AppendObject \u5bf9\u8c61\u7684\u6027\u80fd\u3002 6.1.7\uff0c2024-04-12 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.1.7 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.1.7 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d\u82e5\u5e72\u95ee\u9898 JindoSDK \u652f\u6301 JindoCache + JindoAuth \u8bbf\u95ee DLF\u3002 6.3.4\uff0c2024-04-11 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.3.4 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.3.4 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 JindoFuse \u652f\u6301\u6307\u5b9a metrics_ip\u3001metrics_port \u6307\u5b9a promethues \u76d1\u542c ip \u548c \u7aef\u53e3\u3002 \u4fee\u590d JindoOssFileSystem \u7684 Delegation Token Renew \u673a\u5236\u3002 \u4fee\u590d fs.accessPolicies.discovery \u672b\u5c3e\u4e0d\u5e26 / \u65f6\uff0cgetTrashRoot \u4e3a\u7a7a\u62a5\u9519\uff08\u4ec5\u5f71\u54cd 6.3.3 \u7248\u672c\uff09\u3002 \u4fee\u590d listStatusIterator \u4e0d\u652f\u6301 ListObjectV2\u3002\u5373\u914d\u7f6e fs.oss.list.type \u4e3a 2 \uff0c\u53ef\u80fd\u5bfc\u81f4 listStatusIterator \u6b7b\u5faa\u73af\uff08\u9ed8\u8ba4\u914d\u7f6e\u4e0d\u5f71\u54cd\uff09\u3002 6.3.3\uff0c2024-03-20 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.3.3 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.3.3 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u4fee\u590d JindoFuse \u521b\u5efa symlink \u903b\u8f91\uff0c\u652f\u6301 csi plugin \u573a\u666f\u4f7f\u7528 \u652f\u6301\u901a\u8fc7\u8bbe\u7f6e\u7684\u53cd\u5411\u4ee3\u7406\u8bbf\u95ee\u5bf9\u8c61\u5b58\u50a8 6.3.2\uff0c2024-02-26 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.3.2 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.3.2 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u4fee\u590d JindoFuse \u8bfb\u53d6 symlink \u903b\u8f91\uff0csymlink target \u6539\u4e3a\u76f8\u5bf9\u8def\u5f84\u4ee5\u652f\u6301 csi plugin \u4fee\u590d JindoCache \u5bf9 fs.oss.signer.version \u7684\u652f\u6301 \u4fee\u590d jindo CLI \u53c2\u6570 -version 6.3.1\uff0c2024-02-21 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.3.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.3.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u4fee\u590d\u5728 OSS-HDFS \u6587\u4ef6\u4e0a\u8fdb\u884c\u53cd\u5411 seek \u5bfc\u81f4\u9884\u8bfb\u5931\u6548\u7684\u95ee\u9898 \u4fee\u590d\u5728\u7531\u968f\u673a\u5199\u4ea7\u751f\u7684 OSS-HDFS \u6587\u4ef6\u4e0a pread \u9884\u8bfb\u9519\u8bef\u7684\u95ee\u9898 \u4fee\u590d getRealPath \u63a5\u53e3 Root Policy \u8def\u5f84\u6539\u5199\u652f\u6301\u6839\u636e\u5b50\u8def\u5f84\u533a\u5206\u7684\u4e00\u5bf9\u591a\u66ff\u6362 6.3.0\uff0c2024-01-31 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.3.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.3.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u652f\u6301\u4f7f\u7528 V4 \u7b7e\u540d\u8bbf\u95ee OSS/OSS-HDFS \u63d0\u4f9b Python SDK\uff0c\u652f\u6301\u8bbf\u95ee OSS \u548c OSS-HDFS \u4fee\u590d Jindo \u547d\u4ee4\u884c\u5220\u9664 OSS-HDFS \u76ee\u5f55\u65f6\u53d1\u8d77\u8fc7\u591a\u8bf7\u6c42\u7684\u95ee\u9898 JindoTable \u5206\u5c42\u5b58\u50a8\u5de5\u5177\u652f\u6301\u5f52\u6863\u6570\u636e\u76f4\u63a5\u8f6c\u51b7\u5f52\u6863 JindoTable MoveTo \u5de5\u5177\u652f\u6301 Iceberg \u8868\u6574\u4f53\u8fc1\u79fb \u4fee\u590d JindoFuse \u8bfb\u53d6 symlink \u5931\u8d25 6.1.6\uff0c2023-12-28 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.1.6 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.1.6 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u652f\u6301\u4f7f\u7528 V4 \u7b7e\u540d\u8bbf\u95ee OSS/OSS-HDFS 6.2.0\uff0c2023-12-22 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.2.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.2.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u65b0\u589e\u652f\u6301\u5f02\u6b65\u7684 CSDK libjindosdk_c.so \uff0c\u8001\u7248\u672c libjindo-csdk.so \u9010\u6e10\u5f03\u7528\u3002 \u4fee\u590d\u8def\u5f84\u5408\u6cd5\u6027\u68c0\u67e5\uff0c\u652f\u6301\u5e26 * \u7684\u8def\u5f84 \u4f18\u5316 JindoFuse \u8bbf\u95ee oss \u65f6\u6709\u975e\u6cd5\u540d\u79f0\u7684\u8def\u5f84\u65f6\u7684\u62a5\u9519\u4fe1\u606f Jindo \u547d\u4ee4\u884c\u652f\u6301 OSS-HDFS setStoragePolicy \u89e3\u51bb\u76f8\u5173\u7b56\u7565 \u91c7\u7528\u65b0\u7684\u9884\u8bfb\u7b97\u6cd5\uff0c\u987a\u5e8f\u8bfb\u5927\u6587\u4ef6\u7684\u6027\u80fd\u63d0\u9ad8 \u65b0\u9884\u8bfb\u7b97\u6cd5\u53ef\u914d\u7f6e\u53c2\u6570\u5982\u4e0b - fs.oss.read.readahead.prefetcher.version = default \u9884\u8bfb\u5b9e\u73b0\u5207\u6362\u5f00\u5173 > - \u53ef\u9009\u503c\uff1a legacy \u539f\u9884\u8bfb\u7b97\u6cd5, default \u65b0\u9884\u8bfb\u7b97\u6cd5 - fs.oss.read.readahead.prefetch.size.max = 268435456 \u9884\u8bfb\u6700\u5927\u957f\u5ea6\uff08\u5355\u4f4d\uff1abyte\uff09 \u65b0\u9884\u8bfb\u7b97\u6cd5\u53ef\u80fd\u4f7f\u7528\u66f4\u591a\u5185\u5b58\u3002\u82e5\u914d\u7f6e\u4e86\u65b0\u9884\u8bfb\u7b97\u6cd5\u540e\u53d1\u751f\u6027\u80fd\u4e0b\u964d\uff0c\u53ef\u80fd\u662f\u7531\u4e8e\u5185\u5b58\u6c60\u5bb9\u91cf\u4e0d\u8db3\u5bfc\u81f4\u9884\u8bfb\u7684\u5757\u5728\u88ab\u8bbf\u95ee\u5230\u4e4b\u524d\u5c31\u88ab\u63d0\u524d\u9010\u51fa\u3002\u4e3a\u4e86\u907f\u514d\u8be5\u60c5\u51b5\u53d1\u751f\uff0c\u53ef\u4ee5\u8003\u8651\u7f29\u51cf\u6700\u5927\u9884\u8bfb\u957f\u5ea6\uff0c\u6216\u5141\u8bb8\u9884\u8bfb\u4f7f\u7528\u66f4\u591a\u5185\u5b58\u3002\u5185\u5b58\u6c60\u914d\u7f6e\u76f8\u5173\u53c2\u6570\u5982\u4e0b - fs.oss.memory.buffer.size.max.mb = 6144 \u5185\u5b58\u6c60\u603b\u5bb9\u91cf\uff08\u5355\u4f4d\uff1aMB\uff09 - fs.oss.memory.buffer.size.watermark = 0.3 \u5185\u5b58\u6c60\u7528\u4e8e\u9884\u8bfb\u7684\u5bb9\u91cf\u6bd4\u4f8b \u4fee\u590d Hive \u4f7f\u7528 LDAP + Kerberos + OSS-HDFS \u65f6\u7684\u8ba4\u8bc1\u95ee\u9898 6.1.5\uff0c2023-12-19 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.1.5 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.1.5 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 OSS-HDFS \u8bfb\u5199 IO \u4f18\u5316 6.1.4\uff0c2023-12-14 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.1.4 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.1.4 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u652f\u6301 HiveServer2 \u5728 Kerberos & LDAP \u53cc\u8ba4\u8bc1\u573a\u666f\u4e0b\u4f7f\u7528 JindoSDK Java Jni \u5ffd\u7565 AddressSanitizer \u8bef\u62a5 6.1.3\uff0c2023-12-01 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.1.3 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.1.3 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u4fee\u590d dls:// \u524d\u7f00\u8bbf\u95ee jindofs \u7684\u82e5\u5e72\u95ee\u9898\u3002 6.1.2\uff0c2023-11-21 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.1.2 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.1.2 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u901a\u8fc7\u914d\u7f6e fs.oss.list.type=2 \u652f\u6301ListObjectV2\uff0c\u4ee5\u907f\u514d\u8bbf\u95ee\u5f00\u542f\u4e86\u591a\u7248\u672cbucket\u65f6\u7684\u64cd\u4f5c\u8d85\u65f6\u95ee\u9898\u3002 \u652f\u6301\u4f7f\u7528 dls:// \u524d\u7f00\u8bbf\u95ee jindofs\u3002 \u4fee\u590d\u8bfb\u6d41 close \u4f1a\u5361\u4f4f\u7684\u95ee\u9898\u3002 \u4fee\u590d\u5199\u6d41\u51fa\u73b0\u4e00\u6b21 write \u5931\u8d25\u540e\uff0c\u540e\u7eed write \u51fa\u73b0\u7684 NPE \u95ee\u9898\u3002 6.1.1\uff0c2023-10-20 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.1.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.1.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u652f\u6301\u901a\u8fc7\u914d\u7f6e fs.oss.legacy.version \u4e3a 3.8\uff0c\u517c\u5bb9 JindoSDK 3.8.x \u914d\u7f6e 6.1.0\uff0c2023-09-28 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.1.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.1.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 JindoSDK \u652f\u6301 FileSystem close \u540e\u4ecd\u652f\u6301\u4f7f\u7528\u8bfb\u5199\u6d41\u8fdb\u884c\u8bfb\u5199 6.0.1\uff0c2023-08-18 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.0.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.0.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 JindoSDK \u652f\u6301 isFileClosed \u65b9\u6cd5\uff0c\u7528\u4e8e\u5224\u65ad\u8bbf\u95ee OSS-HDFS \u7684\u8bfb\u5199\u6d41\u662f\u5426\u5df2\u5173\u95ed\u3002 JindoSDK \u5bf9 flush \u964d\u9891\u505a\u4e86\u4f18\u5316\uff0c\u5728\u964d\u9891\u914d\u7f6e\u751f\u6548\u65f6\uff0c\u5bf9\u7b2c\u4e00\u6b21\u6709\u6570\u636e\u7684flush\u64cd\u4f5c\u4fdd\u8bc1\u6267\u884c\uff0c\u9632\u6b62flush\u51fa\u4e00\u4e2a\u7a7a\u6587\u4ef6\u3002\u964d\u9891\u914d\u7f6e\uff0c\u53ef\u53c2\u8003 Flume \u4f7f\u7528 JindoSDK \u5199\u5165 OSS 6.0.0\uff0c2023-08-15 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.0.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.0.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \uff0cJindoSDK \u5408\u5e76 jindo-mapreduce (JindoCommitterFactory\u7b49) \u5230 jindo-sdk JindoSDK \u65b0\u589e\u82e5\u5e72\u4fee\u590d\uff0c\u5305\u542b flush \u964d\u9891\uff0c\u4f18\u5316 GC \u56de\u6536\uff0c\u9002\u914d OSS-HDFS \u56de\u6536\u7ad9\u7b49 \u591a\u5e73\u53f0\u652f\u6301 MACOS \u548c \u4e3b\u6d41 linux \u573a\u666f\uff0c\u5176\u4e2d CentOS 6, Ubuntu22, \u963f\u91cc\u4e91\u501a\u5929\u673a\u578b(ARM)\uff0cMacOS (Intel/M1) \u9700\u8981\u4f7f\u7528\u7279\u5b9a\u7684\u6269\u5c55\u5305 \u5347\u7ea7 g++ \u7f16\u8bd1\u5668\u5230 10.4\uff0c\u4f7f\u7528 c++20 \u6807\u51c6 \u652f\u6301 yalangtinglibs \u8bbf\u95ee OSS/OSS-HDFS","title":"\u5386\u53f2\u7248\u672c"},{"location":"releases/#jindosdk","text":"","title":"JindoSDK \u7248\u672c\u8bb0\u5f55"},{"location":"releases/#6732024-11-01","text":"","title":"6.7.3\uff0c2024-11-01"},{"location":"releases/#_1","text":"\u53d1\u5e03 JindoSDK 6.7.3 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_2","text":"JindoSDK \u66f4\u65b0 6.7.3 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u6253\u5305\u65f6\u5e26\u5165 jindoauth plugin\uff0c\u5347\u7ea7 jindosdk \u540e\u9700\u5c06 fs.jdo.plugin.dir \u6307\u5411 jindosdk plugins \u76ee\u5f55\u3002 nextarch classifier \u4fee\u590d hudi \u8bfb\u5199 log \u65f6\uff0c\u5bf9 AlreadyBeingCreatedException \u5f02\u5e38\u5904\u7406\u3002 nextarch classifier \u4fee\u590d\u5728\u4f7f\u7528 http \u8bf7\u6c42\u66f4\u65b0 STS \u65f6\uff0c\u5076\u73b0\u5361\u4f4f\u95ee\u9898\u3002 nextarch classifier \u4fee\u590d\u5f00\u542f Ranger \u540e\u6301\u7eed\u5199\u5165\u4e00\u4e2a OSS-HDFS \u6587\u4ef6\u8d85\u8fc7 1 \u5c0f\u65f6\u540e\uff0c\u6587\u4ef6\u65e0\u6cd5\u7ee7\u7eed\u5199\u5165\u7684\u95ee\u9898\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6412024-10-23","text":"","title":"6.4.1\uff0c2024-10-23"},{"location":"releases/#_3","text":"\u53d1\u5e03 JindoSDK 6.4.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_4","text":"JindoSDK \u66f4\u65b0 6.4.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d\u5076\u73b0 Signer V4 \u7b7e\u540d\u95ee\u9898\uff0c\u5c06\u7b7e\u540d\u65f6\u95f4\u6233\u8f6c\u6362\u51fd\u6570 localtime\u3001gmtime \u66ff\u6362\u4e3a localtime_r\u3001gmtime_r \u907f\u514d\u5e76\u53d1\u95ee\u9898\u3002 \u4fee\u590d libjindosdk_c.so \u8bbf\u95ee OSS-HDFS \u5076\u73b0 crash \u95ee\u9898\uff0c\u53ca\u91cd\u8bd5\u5931\u8d25\u95ee\u9898\u3002 \u4fee\u590d hudi \u8bfb\u5199 log \u65f6\uff0c\u5bf9 AlreadyBeingCreatedException \u5f02\u5e38\u5904\u7406\u3002 \u4fee\u590d JindoOssFileSystem \u7684 Delegation Token Renew \u673a\u5236\u3002 \u4fee\u590d JindoCommitter \u8bbf\u95ee OSS-HDFS \u672a\u6e05\u7406 task \u6b8b\u7559\u7684\u4e34\u65f6\u76ee\u5f55\u3002 \u63d0\u5347 JindoSDK \u5bf9\u4e8e\u4f4e\u4e8e Hadoop 2.8.x \u7248\u672c\u63a5\u53e3\u7684\u517c\u5bb9\u6027\uff0c\u5982 CallerContext \u53ca FsServerDefaults \u3002 \u4f18\u5316 classloader \u52a0\u8f7d\uff0c\u5728\u591a\u4e2a classloader \u52a0\u8f7d jindosdk \u65f6\u7684 so \u6b8b\u7559\u95ee\u9898\uff0c\u5e76\u4e14\u5728\u914d\u7f6e\u4e86 java.io.tmpdir \u65f6\uff0c\u9ed8\u8ba4\u89e3\u538b\u81f3\u914d\u7f6e\u76ee\u5f55\u4e0b\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6722024-10-21","text":"","title":"6.7.2\uff0c2024-10-21"},{"location":"releases/#_5","text":"\u53d1\u5e03 JindoSDK 6.7.2 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_6","text":"JindoSDK \u66f4\u65b0 6.7.2 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u4f18\u5316 metrics \u7edf\u8ba1\u4fe1\u606f\u3002 nextarch classifier \u4fee\u590d\u5728\u5199 OSS-HDFS \u573a\u666f flush \u540e\u51fa\u73b0\u5c0f\u5757\u7684\u95ee\u9898\u3002 nextarch classifier \u4fee\u590d\u540e\u53f0 metrics \u7ebf\u7a0b\u5f15\u53d1\u7684\u5076\u73b0 crash \u95ee\u9898\u3002 Pyjindo \u4fee\u590d 6.7.x \u7248\u672c\u62a5 invalid pointer \u95ee\u9898\uff0c6.7.0 \u53ca 6.7.1 \u7248\u672c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u73af\u5883\u53d8\u91cf JINDO_STAT_MEMORY=0 \u7ed5\u8fc7\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6712024-10-15","text":"","title":"6.7.1\uff0c2024-10-15"},{"location":"releases/#_7","text":"\u53d1\u5e03 JindoSDK 6.7.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_8","text":"JindoSDK \u66f4\u65b0 6.7.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u4f18\u5316\u9ed8\u8ba4\u5e76\u53d1\u4e0b\u7684\u5199\u6027\u80fd\u3002 \u4fee\u590d JindoCommitter \u8bbf\u95ee OSS-HDFS \u672a\u6e05\u7406 task \u6b8b\u7559\u7684\u4e34\u65f6\u76ee\u5f55\u3002 jindo-fuse \u4fee\u590d\u5728 s3 scheme \u4e0a\u8ffd\u52a0\u5199\u7684\u652f\u6301\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6702024-09-29","text":"","title":"6.7.0\uff0c2024-09-29"},{"location":"releases/#_9","text":"\u53d1\u5e03 JindoSDK 6.7.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_10","text":"JindoSDK \u66f4\u65b0 6.7.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u652f\u6301\u7edf\u4e00\u7ba1\u7406\u5185\u5b58\uff0c\u4f7f\u7528 fs.jdo.memory.pool.size.max.mb \u914d\u7f6e\u6700\u5927\u4f7f\u7528\u5185\u5b58\uff0c\u5e76\u652f\u6301\u5185\u5b58\u5b9e\u65f6\u4f7f\u7528 metrics\u3002 nextarch classifier \u652f\u6301 io buffer \u5185\u5b58\u4f7f\u7528\u63a7\u5236\uff0c\u4f7f\u7528 fs.jdo.memory.io.buffer.size.max.ratio \u914d\u7f6e\u6700\u5927\u4f7f\u7528\u5185\u5b58\u6c60\u7684\u6bd4\u4f8b\u3002 nextarch classifier \u652f\u6301 Golang SDK\u3002 nextarch classifier \u652f\u6301 OSS \u5bf9\u8c61\u5b58\u50a8\u6df1\u51b7\u5f52\u6863\u3002 nextarch classifier \u4f18\u5316 pread \u63a5\u53e3\uff0c\u5185\u5b58\u96f6\u62f7\u8d1d\u3002 nextarch classifier \u4f18\u5316 sendfile \u5728\u5199\u5c0f\u6587\u4ef6\u573a\u666f\u7684\u4f7f\u7528\u3002 nextarch classifier \u4f18\u5316 metrics \u6846\u67b6\uff0c\u652f\u6301\u6bcf\u6b21\u8f93\u51fa\u5230\u5355\u72ec\u6587\u4ef6\uff0c\u65b9\u4fbf\u91c7\u96c6\u3002 nextarch classifier \u4f18\u5316 distjob \u6846\u4e2d\uff0c\u6392\u9664 log4j \u76f8\u5173\u4f9d\u8d56\u3002 nextarch classifier \u4f18\u5316 classloader \u52a0\u8f7d\uff0c\u5728\u591a\u4e2a classloader \u52a0\u8f7d jindosdk \u65f6\u7684 so \u6b8b\u7559\u95ee\u9898\uff0c\u5e76\u4e14\u5728\u914d\u7f6e\u4e86 java.io.tmpdir \u65f6\uff0c\u9ed8\u8ba4\u89e3\u538b\u81f3\u914d\u7f6e\u76ee\u5f55\u4e0b\u3002 nextarch classifier \u4fee\u590d OSS-HDFS StoragePolicy \u76f8\u5173\u63a5\u53e3\u3002 nextarch classifier \u4fee\u590d OSS \u5bf9\u8c61\u5b58\u50a8\uff0cListDirectory \u8fd4\u56de\u7ed3\u679c\u7684 Mtime \u5355\u4f4d\u3002 nextarch classifier \u4fee\u590d JindoCommitter \u5728 Cleanup \u9636\u6bb5\uff0c\u4e0d\u5e94\u629b\u51fa\u5f02\u5e38\u3002 nextarch classifier \u4fee\u590d DLF \u76f8\u5173 CredentialProvider \u4f7f\u7528 MagicCommitter \u65f6\u7684\u6743\u9650\u95ee\u9898\u3002 nextarch classifier \u4fee\u590d JindoCache \u7684\u82e5\u5e72\u95ee\u9898\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6632024-09-14","text":"","title":"6.6.3\uff0c2024-09-14"},{"location":"releases/#_11","text":"\u53d1\u5e03 JindoSDK 6.6.3 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_12","text":"JindoSDK \u66f4\u65b0 6.6.3 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u4fee\u590d\u5199\u573a\u666f\u5c0f\u6982\u7387\u56e0 sendfile \u51fa\u73b0\u9519\u8bef\u672a\u91cd\u8bd5\uff0c\u5bfc\u81f4\u6587\u4ef6 close \u65f6\u62a5\u5931\u8d25\u7684\u60c5\u51b5\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6622024-09-12","text":"","title":"6.6.2\uff0c2024-09-12"},{"location":"releases/#_13","text":"\u53d1\u5e03 JindoSDK 6.6.2 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_14","text":"JindoSDK \u66f4\u65b0 6.6.2 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u8bbf\u95ee OSS \u652f\u6301\u6df1\u51b7\u5f52\u6863 nextarch classifier \u4fee\u590d DLF \u76f8\u5173 CredentialProvider \u4f7f\u7528 MagicCommitter \u65f6\u7684\u6743\u9650\u95ee\u9898\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6612024-09-03","text":"","title":"6.6.1\uff0c2024-09-03"},{"location":"releases/#_15","text":"\u53d1\u5e03 JindoSDK 6.6.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_16","text":"JindoSDK \u66f4\u65b0 6.6.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u4f18\u5316\u5728\u591a\u4e2a classloader \u52a0\u8f7d jindosdk \u65f6\u7684 so \u6b8b\u7559\u95ee\u9898\uff0c\u5e76\u4e14\u5728\u914d\u7f6e\u4e86 java.io.tmpdir \u65f6\uff0c\u9ed8\u8ba4\u89e3\u538b\u81f3\u914d\u7f6e\u76ee\u5f55\u4e0b\u3002 nextarch classifier \u4fee\u590d\u8bbf\u95ee JindoCache \u7684\u82e5\u5e72\u95ee\u9898\u3002 nextarch classifier \u4f18\u5316 metrics \u6846\u67b6\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6602024-08-25","text":"","title":"6.6.0\uff0c2024-08-25"},{"location":"releases/#_17","text":"\u53d1\u5e03 JindoSDK 6.6.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_18","text":"JindoSDK \u66f4\u65b0 6.6.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u4f18\u5316 io \u6027\u80fd\uff0c\u4f7f\u7528\u534f\u7a0b\u5f02\u6b65\u5316\uff0c\u76f8\u540c\u914d\u7f6e\u4e0b\u652f\u6301\u66f4\u9ad8\u5e76\u53d1\u3002\u5199\u573a\u666f\u4f7f\u7528 sendfile \u5b9e\u73b0\u96f6\u62f7\u8d1d\u4f18\u5316\uff0c\u8282\u7701\u5185\u5b58\uff0c\u5e76\u63d0\u5347\u6027\u80fd\u3002 nextarch classifier \u652f\u6301\u5355\u72ec\u6307\u5b9a io \u8d85\u65f6\u65f6\u95f4\u3002 nextarch classifier \u652f\u6301 metrics \u6846\u67b6\u3002 \u4fee\u590d nextarch classifier \u5bf9 JindoCache \u652f\u6301\u95ee\u9898\u3002 jindo-fuse \u9ed8\u8ba4\u6539\u4e3a\u4f7f\u7528 nextarch \u5b9e\u73b0\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6542024-08-20","text":"","title":"6.5.4\uff0c2024-08-20"},{"location":"releases/#_19","text":"\u53d1\u5e03 JindoSDK 6.5.4 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_20","text":"JindoSDK \u66f4\u65b0 6.5.4 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4f18\u5316 flink connector \u4f9d\u8d56\uff0c\u5bf9\u90e8\u5206\u4f9d\u8d56\u8fdb\u884c shade\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6532024-08-15","text":"","title":"6.5.3\uff0c2024-08-15"},{"location":"releases/#_21","text":"\u53d1\u5e03 JindoSDK 6.5.3 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_22","text":"JindoSDK \u66f4\u65b0 6.5.3 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d JindoOssFileSystem \u7684 Delegation Token Renew \u673a\u5236\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6522024-08-12","text":"","title":"6.5.2\uff0c2024-08-12"},{"location":"releases/#_23","text":"\u53d1\u5e03 JindoSDK 6.5.2 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_24","text":"JindoSDK \u66f4\u65b0 6.5.2 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d getfacl \u7f3a\u5c11 entries \u95ee\u9898\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6512024-08-05","text":"","title":"6.5.1\uff0c2024-08-05"},{"location":"releases/#_25","text":"\u53d1\u5e03 JindoSDK 6.5.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_26","text":"JindoSDK \u66f4\u65b0 6.5.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u63d0\u5347 JindoSDK \u5bf9\u4e8e\u4f4e\u4e8e Hadoop 2.8.x \u7248\u672c\u63a5\u53e3\u7684\u517c\u5bb9\u6027\uff0c\u5982 CallerContext \u53ca FsServerDefaults \u3002 \u4f18\u5316 JindoCommitter \u6027\u80fd\u3002 \u4f18\u5316\u5185\u5b58\u4e0d\u8db3\u65f6\u7684\u9884\u8bfb\u7b97\u6cd5\uff0c\u56de\u6536\u4e0d\u4f7f\u7528\u7684\u5185\u5b58\u3002 \u4f18\u5316 plugin \u52a0\u8f7d\u65b9\u5f0f\uff0c fs.jdo.plugin.dir \u652f\u6301\u4ee5\u591a\u8def\u5f84\u52a0\u8f7d\uff0c\u5982 /dir1,/dir2 \u4fee\u590d list \u5bf9\u8c61\u5b58\u50a8\u65f6\uff0c\u8def\u5f84\u4e2d\u5e26\u6709 // \uff0c\u53ef\u80fd\u4f1alist\u51fa\u81ea\u8eab\u7684\u95ee\u9898 \u4fee\u590d jindo-dependence-shaded.jar \u5b89\u5168\u6027\u95ee\u9898\uff0c\u53bb\u9664 log4j \u53ca apache commons text \u4f9d\u8d56 \u4fee\u590d nextarch classifier \u8dd1 hadoop-compat-bench \u82e5\u5e72\u95ee\u9898\u3002 \u4fee\u590d nextarch classifier \u5bf9 RootPolicy \u652f\u6301\u95ee\u9898\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6502024-07-12","text":"","title":"6.5.0\uff0c2024-07-12"},{"location":"releases/#_27","text":"\u53d1\u5e03 JindoSDK 6.5.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_28","text":"JindoSDK \u66f4\u65b0 6.5.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d\u5076\u73b0 Signer V4 \u7b7e\u540d\u95ee\u9898\uff0c\u5c06\u7b7e\u540d\u65f6\u95f4\u6233\u8f6c\u6362\u51fd\u6570 localtime\u3001gmtime \u66ff\u6362\u4e3a localtime_r\u3001gmtime_r \u907f\u514d\u5e76\u53d1\u95ee\u9898\u3002 \u4fee\u590d libjindosdk_c.so \u8bbf\u95ee OSS-HDFS \u5076\u73b0 crash \u95ee\u9898\uff0c\u53ca\u91cd\u8bd5\u5931\u8d25\u95ee\u9898\u3002 jindo-core.jar nextarch classifer \u652f\u6301\u65b0\u7248\u5185\u6838\u3002 \u4f18\u5316\u5185\u5b58\u4e0d\u8db3\u65f6\u7684\u9884\u8bfb\u7b97\u6cd5\uff0c\u9632\u6b62\u5185\u5b58\u8fc7\u5ea6\u62a2\u5360\u3002 commiter \u652f\u6301 setXAttr\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6352024-06-28","text":"","title":"6.3.5\uff0c2024-06-28"},{"location":"releases/#_29","text":"\u53d1\u5e03 JindoSDK 6.3.5 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_30","text":"JindoSDK \u66f4\u65b0 6.3.5 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d\u5076\u73b0 Signer V4 \u7b7e\u540d\u95ee\u9898\uff0c\u5c06\u7b7e\u540d\u65f6\u95f4\u6233\u8f6c\u6362\u51fd\u6570 localtime\u3001gmtime \u66ff\u6362\u4e3a localtime_r\u3001gmtime_r \u907f\u514d\u5e76\u53d1\u95ee\u9898\u3002 \u4fee\u590d libjindosdk_c.so \u8bbf\u95ee OSS-HDFS \u5076\u73b0 crash \u95ee\u9898\uff0c\u53ca\u91cd\u8bd5\u5931\u8d25\u95ee\u9898\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6402024-05-16","text":"","title":"6.4.0\uff0c2024-05-16"},{"location":"releases/#_31","text":"\u53d1\u5e03 JindoSDK 6.4.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_32","text":"JindoSDK \u66f4\u65b0 6.4.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 JindoSDK \u652f\u6301 CallerContext\u3002 JindoSDK \u652f\u6301\u5199\u65f6 flush\u3002 \u4fee\u590d concat \u95ee\u9898\u3002 \u652f\u6301 RDMA \u8bbf\u95ee JindoCache\u3002 \u4f18\u5316\u4e86\u8bfb AppendObject \u5bf9\u8c61\u7684\u6027\u80fd\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6172024-04-12","text":"","title":"6.1.7\uff0c2024-04-12"},{"location":"releases/#_33","text":"\u53d1\u5e03 JindoSDK 6.1.7 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_34","text":"JindoSDK \u66f4\u65b0 6.1.7 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d\u82e5\u5e72\u95ee\u9898 JindoSDK \u652f\u6301 JindoCache + JindoAuth \u8bbf\u95ee DLF\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6342024-04-11","text":"","title":"6.3.4\uff0c2024-04-11"},{"location":"releases/#_35","text":"\u53d1\u5e03 JindoSDK 6.3.4 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_36","text":"JindoSDK \u66f4\u65b0 6.3.4 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 JindoFuse \u652f\u6301\u6307\u5b9a metrics_ip\u3001metrics_port \u6307\u5b9a promethues \u76d1\u542c ip \u548c \u7aef\u53e3\u3002 \u4fee\u590d JindoOssFileSystem \u7684 Delegation Token Renew \u673a\u5236\u3002 \u4fee\u590d fs.accessPolicies.discovery \u672b\u5c3e\u4e0d\u5e26 / \u65f6\uff0cgetTrashRoot \u4e3a\u7a7a\u62a5\u9519\uff08\u4ec5\u5f71\u54cd 6.3.3 \u7248\u672c\uff09\u3002 \u4fee\u590d listStatusIterator \u4e0d\u652f\u6301 ListObjectV2\u3002\u5373\u914d\u7f6e fs.oss.list.type \u4e3a 2 \uff0c\u53ef\u80fd\u5bfc\u81f4 listStatusIterator \u6b7b\u5faa\u73af\uff08\u9ed8\u8ba4\u914d\u7f6e\u4e0d\u5f71\u54cd\uff09\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6332024-03-20","text":"","title":"6.3.3\uff0c2024-03-20"},{"location":"releases/#_37","text":"\u53d1\u5e03 JindoSDK 6.3.3 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_38","text":"JindoSDK \u66f4\u65b0 6.3.3 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u4fee\u590d JindoFuse \u521b\u5efa symlink \u903b\u8f91\uff0c\u652f\u6301 csi plugin \u573a\u666f\u4f7f\u7528 \u652f\u6301\u901a\u8fc7\u8bbe\u7f6e\u7684\u53cd\u5411\u4ee3\u7406\u8bbf\u95ee\u5bf9\u8c61\u5b58\u50a8","title":"\u4ecb\u7ecd"},{"location":"releases/#6322024-02-26","text":"","title":"6.3.2\uff0c2024-02-26"},{"location":"releases/#_39","text":"\u53d1\u5e03 JindoSDK 6.3.2 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_40","text":"JindoSDK \u66f4\u65b0 6.3.2 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u4fee\u590d JindoFuse \u8bfb\u53d6 symlink \u903b\u8f91\uff0csymlink target \u6539\u4e3a\u76f8\u5bf9\u8def\u5f84\u4ee5\u652f\u6301 csi plugin \u4fee\u590d JindoCache \u5bf9 fs.oss.signer.version \u7684\u652f\u6301 \u4fee\u590d jindo CLI \u53c2\u6570 -version","title":"\u4ecb\u7ecd"},{"location":"releases/#6312024-02-21","text":"","title":"6.3.1\uff0c2024-02-21"},{"location":"releases/#_41","text":"\u53d1\u5e03 JindoSDK 6.3.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_42","text":"JindoSDK \u66f4\u65b0 6.3.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u4fee\u590d\u5728 OSS-HDFS \u6587\u4ef6\u4e0a\u8fdb\u884c\u53cd\u5411 seek \u5bfc\u81f4\u9884\u8bfb\u5931\u6548\u7684\u95ee\u9898 \u4fee\u590d\u5728\u7531\u968f\u673a\u5199\u4ea7\u751f\u7684 OSS-HDFS \u6587\u4ef6\u4e0a pread \u9884\u8bfb\u9519\u8bef\u7684\u95ee\u9898 \u4fee\u590d getRealPath \u63a5\u53e3 Root Policy \u8def\u5f84\u6539\u5199\u652f\u6301\u6839\u636e\u5b50\u8def\u5f84\u533a\u5206\u7684\u4e00\u5bf9\u591a\u66ff\u6362","title":"\u4ecb\u7ecd"},{"location":"releases/#6302024-01-31","text":"","title":"6.3.0\uff0c2024-01-31"},{"location":"releases/#_43","text":"\u53d1\u5e03 JindoSDK 6.3.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_44","text":"JindoSDK \u66f4\u65b0 6.3.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u652f\u6301\u4f7f\u7528 V4 \u7b7e\u540d\u8bbf\u95ee OSS/OSS-HDFS \u63d0\u4f9b Python SDK\uff0c\u652f\u6301\u8bbf\u95ee OSS \u548c OSS-HDFS \u4fee\u590d Jindo \u547d\u4ee4\u884c\u5220\u9664 OSS-HDFS \u76ee\u5f55\u65f6\u53d1\u8d77\u8fc7\u591a\u8bf7\u6c42\u7684\u95ee\u9898 JindoTable \u5206\u5c42\u5b58\u50a8\u5de5\u5177\u652f\u6301\u5f52\u6863\u6570\u636e\u76f4\u63a5\u8f6c\u51b7\u5f52\u6863 JindoTable MoveTo \u5de5\u5177\u652f\u6301 Iceberg \u8868\u6574\u4f53\u8fc1\u79fb \u4fee\u590d JindoFuse \u8bfb\u53d6 symlink \u5931\u8d25","title":"\u4ecb\u7ecd"},{"location":"releases/#6162023-12-28","text":"","title":"6.1.6\uff0c2023-12-28"},{"location":"releases/#_45","text":"\u53d1\u5e03 JindoSDK 6.1.6 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_46","text":"JindoSDK \u66f4\u65b0 6.1.6 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u652f\u6301\u4f7f\u7528 V4 \u7b7e\u540d\u8bbf\u95ee OSS/OSS-HDFS","title":"\u4ecb\u7ecd"},{"location":"releases/#6202023-12-22","text":"","title":"6.2.0\uff0c2023-12-22"},{"location":"releases/#_47","text":"\u53d1\u5e03 JindoSDK 6.2.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_48","text":"JindoSDK \u66f4\u65b0 6.2.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u65b0\u589e\u652f\u6301\u5f02\u6b65\u7684 CSDK libjindosdk_c.so \uff0c\u8001\u7248\u672c libjindo-csdk.so \u9010\u6e10\u5f03\u7528\u3002 \u4fee\u590d\u8def\u5f84\u5408\u6cd5\u6027\u68c0\u67e5\uff0c\u652f\u6301\u5e26 * \u7684\u8def\u5f84 \u4f18\u5316 JindoFuse \u8bbf\u95ee oss \u65f6\u6709\u975e\u6cd5\u540d\u79f0\u7684\u8def\u5f84\u65f6\u7684\u62a5\u9519\u4fe1\u606f Jindo \u547d\u4ee4\u884c\u652f\u6301 OSS-HDFS setStoragePolicy \u89e3\u51bb\u76f8\u5173\u7b56\u7565 \u91c7\u7528\u65b0\u7684\u9884\u8bfb\u7b97\u6cd5\uff0c\u987a\u5e8f\u8bfb\u5927\u6587\u4ef6\u7684\u6027\u80fd\u63d0\u9ad8 \u65b0\u9884\u8bfb\u7b97\u6cd5\u53ef\u914d\u7f6e\u53c2\u6570\u5982\u4e0b - fs.oss.read.readahead.prefetcher.version = default \u9884\u8bfb\u5b9e\u73b0\u5207\u6362\u5f00\u5173 > - \u53ef\u9009\u503c\uff1a legacy \u539f\u9884\u8bfb\u7b97\u6cd5, default \u65b0\u9884\u8bfb\u7b97\u6cd5 - fs.oss.read.readahead.prefetch.size.max = 268435456 \u9884\u8bfb\u6700\u5927\u957f\u5ea6\uff08\u5355\u4f4d\uff1abyte\uff09 \u65b0\u9884\u8bfb\u7b97\u6cd5\u53ef\u80fd\u4f7f\u7528\u66f4\u591a\u5185\u5b58\u3002\u82e5\u914d\u7f6e\u4e86\u65b0\u9884\u8bfb\u7b97\u6cd5\u540e\u53d1\u751f\u6027\u80fd\u4e0b\u964d\uff0c\u53ef\u80fd\u662f\u7531\u4e8e\u5185\u5b58\u6c60\u5bb9\u91cf\u4e0d\u8db3\u5bfc\u81f4\u9884\u8bfb\u7684\u5757\u5728\u88ab\u8bbf\u95ee\u5230\u4e4b\u524d\u5c31\u88ab\u63d0\u524d\u9010\u51fa\u3002\u4e3a\u4e86\u907f\u514d\u8be5\u60c5\u51b5\u53d1\u751f\uff0c\u53ef\u4ee5\u8003\u8651\u7f29\u51cf\u6700\u5927\u9884\u8bfb\u957f\u5ea6\uff0c\u6216\u5141\u8bb8\u9884\u8bfb\u4f7f\u7528\u66f4\u591a\u5185\u5b58\u3002\u5185\u5b58\u6c60\u914d\u7f6e\u76f8\u5173\u53c2\u6570\u5982\u4e0b - fs.oss.memory.buffer.size.max.mb = 6144 \u5185\u5b58\u6c60\u603b\u5bb9\u91cf\uff08\u5355\u4f4d\uff1aMB\uff09 - fs.oss.memory.buffer.size.watermark = 0.3 \u5185\u5b58\u6c60\u7528\u4e8e\u9884\u8bfb\u7684\u5bb9\u91cf\u6bd4\u4f8b \u4fee\u590d Hive \u4f7f\u7528 LDAP + Kerberos + OSS-HDFS \u65f6\u7684\u8ba4\u8bc1\u95ee\u9898","title":"\u4ecb\u7ecd"},{"location":"releases/#6152023-12-19","text":"","title":"6.1.5\uff0c2023-12-19"},{"location":"releases/#_49","text":"\u53d1\u5e03 JindoSDK 6.1.5 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_50","text":"JindoSDK \u66f4\u65b0 6.1.5 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 OSS-HDFS \u8bfb\u5199 IO \u4f18\u5316","title":"\u4ecb\u7ecd"},{"location":"releases/#6142023-12-14","text":"","title":"6.1.4\uff0c2023-12-14"},{"location":"releases/#_51","text":"\u53d1\u5e03 JindoSDK 6.1.4 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_52","text":"JindoSDK \u66f4\u65b0 6.1.4 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u652f\u6301 HiveServer2 \u5728 Kerberos & LDAP \u53cc\u8ba4\u8bc1\u573a\u666f\u4e0b\u4f7f\u7528 JindoSDK Java Jni \u5ffd\u7565 AddressSanitizer \u8bef\u62a5","title":"\u4ecb\u7ecd"},{"location":"releases/#6132023-12-01","text":"","title":"6.1.3\uff0c2023-12-01"},{"location":"releases/#_53","text":"\u53d1\u5e03 JindoSDK 6.1.3 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_54","text":"JindoSDK \u66f4\u65b0 6.1.3 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u4fee\u590d dls:// \u524d\u7f00\u8bbf\u95ee jindofs \u7684\u82e5\u5e72\u95ee\u9898\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6122023-11-21","text":"","title":"6.1.2\uff0c2023-11-21"},{"location":"releases/#_55","text":"\u53d1\u5e03 JindoSDK 6.1.2 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_56","text":"JindoSDK \u66f4\u65b0 6.1.2 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u901a\u8fc7\u914d\u7f6e fs.oss.list.type=2 \u652f\u6301ListObjectV2\uff0c\u4ee5\u907f\u514d\u8bbf\u95ee\u5f00\u542f\u4e86\u591a\u7248\u672cbucket\u65f6\u7684\u64cd\u4f5c\u8d85\u65f6\u95ee\u9898\u3002 \u652f\u6301\u4f7f\u7528 dls:// \u524d\u7f00\u8bbf\u95ee jindofs\u3002 \u4fee\u590d\u8bfb\u6d41 close \u4f1a\u5361\u4f4f\u7684\u95ee\u9898\u3002 \u4fee\u590d\u5199\u6d41\u51fa\u73b0\u4e00\u6b21 write \u5931\u8d25\u540e\uff0c\u540e\u7eed write \u51fa\u73b0\u7684 NPE \u95ee\u9898\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6112023-10-20","text":"","title":"6.1.1\uff0c2023-10-20"},{"location":"releases/#_57","text":"\u53d1\u5e03 JindoSDK 6.1.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_58","text":"JindoSDK \u66f4\u65b0 6.1.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u652f\u6301\u901a\u8fc7\u914d\u7f6e fs.oss.legacy.version \u4e3a 3.8\uff0c\u517c\u5bb9 JindoSDK 3.8.x \u914d\u7f6e","title":"\u4ecb\u7ecd"},{"location":"releases/#6102023-09-28","text":"","title":"6.1.0\uff0c2023-09-28"},{"location":"releases/#_59","text":"\u53d1\u5e03 JindoSDK 6.1.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_60","text":"JindoSDK \u66f4\u65b0 6.1.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 JindoSDK \u652f\u6301 FileSystem close \u540e\u4ecd\u652f\u6301\u4f7f\u7528\u8bfb\u5199\u6d41\u8fdb\u884c\u8bfb\u5199","title":"\u4ecb\u7ecd"},{"location":"releases/#6012023-08-18","text":"","title":"6.0.1\uff0c2023-08-18"},{"location":"releases/#_61","text":"\u53d1\u5e03 JindoSDK 6.0.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_62","text":"JindoSDK \u66f4\u65b0 6.0.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 JindoSDK \u652f\u6301 isFileClosed \u65b9\u6cd5\uff0c\u7528\u4e8e\u5224\u65ad\u8bbf\u95ee OSS-HDFS \u7684\u8bfb\u5199\u6d41\u662f\u5426\u5df2\u5173\u95ed\u3002 JindoSDK \u5bf9 flush \u964d\u9891\u505a\u4e86\u4f18\u5316\uff0c\u5728\u964d\u9891\u914d\u7f6e\u751f\u6548\u65f6\uff0c\u5bf9\u7b2c\u4e00\u6b21\u6709\u6570\u636e\u7684flush\u64cd\u4f5c\u4fdd\u8bc1\u6267\u884c\uff0c\u9632\u6b62flush\u51fa\u4e00\u4e2a\u7a7a\u6587\u4ef6\u3002\u964d\u9891\u914d\u7f6e\uff0c\u53ef\u53c2\u8003 Flume \u4f7f\u7528 JindoSDK \u5199\u5165 OSS","title":"\u4ecb\u7ecd"},{"location":"releases/#6002023-08-15","text":"","title":"6.0.0\uff0c2023-08-15"},{"location":"releases/#_63","text":"\u53d1\u5e03 JindoSDK 6.0.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_64","text":"JindoSDK \u66f4\u65b0 6.0.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \uff0cJindoSDK \u5408\u5e76 jindo-mapreduce (JindoCommitterFactory\u7b49) \u5230 jindo-sdk JindoSDK \u65b0\u589e\u82e5\u5e72\u4fee\u590d\uff0c\u5305\u542b flush \u964d\u9891\uff0c\u4f18\u5316 GC \u56de\u6536\uff0c\u9002\u914d OSS-HDFS \u56de\u6536\u7ad9\u7b49 \u591a\u5e73\u53f0\u652f\u6301 MACOS \u548c \u4e3b\u6d41 linux \u573a\u666f\uff0c\u5176\u4e2d CentOS 6, Ubuntu22, \u963f\u91cc\u4e91\u501a\u5929\u673a\u578b(ARM)\uff0cMacOS (Intel/M1) \u9700\u8981\u4f7f\u7528\u7279\u5b9a\u7684\u6269\u5c55\u5305 \u5347\u7ea7 g++ \u7f16\u8bd1\u5668\u5230 10.4\uff0c\u4f7f\u7528 c++20 \u6807\u51c6 \u652f\u6301 yalangtinglibs \u8bbf\u95ee OSS/OSS-HDFS","title":"\u4ecb\u7ecd"},{"location":"backup/jindodata_QA/","text":"JindoData \u5e38\u89c1\u95ee\u9898 \u00a7 OSS-HDFS \u670d\u52a1 \u00a7 \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u663e\u793a\u5185\u5bb9\u4e0e\u5b9e\u9645\u4e0d\u7b26 \u00a7 \u73b0\u8c61 \u00a7 \u8bbf\u95ee oss://<bucket>.oss-dls.aliyuncs.com/ \u663e\u793a\u5185\u5bb9\u4e0e\u5b9e\u9645\u4e0d\u7b26\u3002 \u89e3\u51b3\u529e\u6cd5 \u00a7 \u5347\u7ea7 JindoSDK \u5230 4.x \u7248\u672c\u3002 OSS SDK \u65b9\u5f0f \u00a7 JindoSDK \u8bbf\u95ee OSS \u51fa\u73b0 CopyNotExecuted \u95ee\u9898\u6392\u67e5 \u00a7 \u73b0\u8c61 \u00a7 \u51fa\u73b0\u62a5\u9519 java.io.IOException: ErrorCode : 25201 , ErrorMsg: OSS Op Error. [ErrorMessage]: Error in xxxxxxxxxx/.hive-staging_hive_xxxxxxxxxxx/_tmp.-ext-10002/xxxxx to xxxxxxxxxx/.hive-staging_hive_xxxxxxxxxxx/_tmp.-ext-10002.moved/xxxx, err: CopyNotExcecuted; \u89e3\u51b3\u529e\u6cd5 \u00a7 CopyNotExecuted \u662f OSS \u670d\u52a1\u7aef\u8fd4\u56de\u7684\u5f02\u5e38\uff0c\u4e00\u822c\u662f OSS BatchCopy \u670d\u52a1\u7aef\u8fc7\u8f7d\u8fd4\u56de\u5f02\u5e38\u3002\u65e9\u671f\u7248\u672c OSS \u672a\u5b9a\u4e49 BatchCopy \u5f02\u5e38\u65f6\u5982\u4f55\u5904\u7406\uff0c\u9700\u8981\u5173\u95ed BatchCopy \u624d\u80fd\u5728\u8001\u7248\u672c\u4e0a\u89e3\u51b3\u3002\u5728EMR\u63a7\u5236\u53f0\uff0c\u4fee\u6539 smartdata-site \u7684 fs.jfs.cache.batch-copy.size \u4e3a 0\uff0c\u5c31\u53ef\u4ee5\u5173\u95ed batch copy\uff0c\u5e76\u4f9d\u6b21\u4fdd\u5b58->\u90e8\u7f72-> \u52fe\u9009\u540c\u6b65\u66f4\u65b0 Gateway","title":"JindoData \u5e38\u89c1\u95ee\u9898"},{"location":"backup/jindodata_QA/#jindodata","text":"","title":"JindoData \u5e38\u89c1\u95ee\u9898"},{"location":"backup/jindodata_QA/#oss-hdfs","text":"","title":"OSS-HDFS \u670d\u52a1"},{"location":"backup/jindodata_QA/#oss-hdfs_1","text":"","title":"\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u663e\u793a\u5185\u5bb9\u4e0e\u5b9e\u9645\u4e0d\u7b26"},{"location":"backup/jindodata_QA/#_1","text":"\u8bbf\u95ee oss://<bucket>.oss-dls.aliyuncs.com/ \u663e\u793a\u5185\u5bb9\u4e0e\u5b9e\u9645\u4e0d\u7b26\u3002","title":"\u73b0\u8c61"},{"location":"backup/jindodata_QA/#_2","text":"\u5347\u7ea7 JindoSDK \u5230 4.x \u7248\u672c\u3002","title":"\u89e3\u51b3\u529e\u6cd5"},{"location":"backup/jindodata_QA/#oss-sdk","text":"","title":"OSS SDK \u65b9\u5f0f"},{"location":"backup/jindodata_QA/#jindosdk-oss-copynotexecuted","text":"","title":"JindoSDK \u8bbf\u95ee OSS \u51fa\u73b0 CopyNotExecuted \u95ee\u9898\u6392\u67e5"},{"location":"backup/jindodata_QA/#_3","text":"\u51fa\u73b0\u62a5\u9519 java.io.IOException: ErrorCode : 25201 , ErrorMsg: OSS Op Error. [ErrorMessage]: Error in xxxxxxxxxx/.hive-staging_hive_xxxxxxxxxxx/_tmp.-ext-10002/xxxxx to xxxxxxxxxx/.hive-staging_hive_xxxxxxxxxxx/_tmp.-ext-10002.moved/xxxx, err: CopyNotExcecuted;","title":"\u73b0\u8c61"},{"location":"backup/jindodata_QA/#_4","text":"CopyNotExecuted \u662f OSS \u670d\u52a1\u7aef\u8fd4\u56de\u7684\u5f02\u5e38\uff0c\u4e00\u822c\u662f OSS BatchCopy \u670d\u52a1\u7aef\u8fc7\u8f7d\u8fd4\u56de\u5f02\u5e38\u3002\u65e9\u671f\u7248\u672c OSS \u672a\u5b9a\u4e49 BatchCopy \u5f02\u5e38\u65f6\u5982\u4f55\u5904\u7406\uff0c\u9700\u8981\u5173\u95ed BatchCopy \u624d\u80fd\u5728\u8001\u7248\u672c\u4e0a\u89e3\u51b3\u3002\u5728EMR\u63a7\u5236\u53f0\uff0c\u4fee\u6539 smartdata-site \u7684 fs.jfs.cache.batch-copy.size \u4e3a 0\uff0c\u5c31\u53ef\u4ee5\u5173\u95ed batch copy\uff0c\u5e76\u4f9d\u6b21\u4fdd\u5b58->\u90e8\u7f72-> \u52fe\u9009\u540c\u6b65\u66f4\u65b0 Gateway","title":"\u89e3\u51b3\u529e\u6cd5"},{"location":"backup/known-issues/","text":"\u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.1 \u5728 EMR \u96c6\u7fa4\u4f7f\u7528\u514d\u5bc6\u8bbf\u95ee OSS-HDFS \u51fa\u73b0\u7b49\u5f85 Token \u66f4\u65b0\uff0c\u5bfc\u81f4\u90e8\u5206\u4f5c\u4e1a\u5361\u4f4f\u7684\u95ee\u9898\uff0c\u5982\u9700\u89e3\u51b3\u6b64\u95ee\u9898\uff0c\u4f7f\u7528\u56fa\u5b9a AK \u6216\u8005\u66f4\u65b0\u5230 4.6.4 \u53ca\u4ee5\u4e0a\u7248\u672c\uff0cEMR\u96c6\u7fa4\u5347\u7ea7\u53ef\u53c2\u8003 EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b \uff1b","title":"\u5df2\u77e5\u95ee\u9898"},{"location":"backup/known-issues/#_1","text":"JindoSDK 4.6.1 \u5728 EMR \u96c6\u7fa4\u4f7f\u7528\u514d\u5bc6\u8bbf\u95ee OSS-HDFS \u51fa\u73b0\u7b49\u5f85 Token \u66f4\u65b0\uff0c\u5bfc\u81f4\u90e8\u5206\u4f5c\u4e1a\u5361\u4f4f\u7684\u95ee\u9898\uff0c\u5982\u9700\u89e3\u51b3\u6b64\u95ee\u9898\uff0c\u4f7f\u7528\u56fa\u5b9a AK \u6216\u8005\u66f4\u65b0\u5230 4.6.4 \u53ca\u4ee5\u4e0a\u7248\u672c\uff0cEMR\u96c6\u7fa4\u5347\u7ea7\u53ef\u53c2\u8003 EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b \uff1b","title":"\u5df2\u77e5\u95ee\u9898"},{"location":"backup/release-notes/","text":"JindoData 4.6.12 \u7248\u672c\u8bf4\u660e \u00a7 \u7248\u672c\u6982\u8981 \u00a7 JindoData 4.6.12 \u7248\u672c\u4f18\u5316\u4e86\u5bf9 Hadoop \u793e\u533a\u4f7f\u7528\u6587\u4ef6\u8bfb\u5199\u6d41\u7684\u517c\u5bb9\u6027\u3002 \u4fee\u590d\u4ecb\u7ecd \u00a7 JindoSDK \u5728 FileSystem close \u540e\u4ecd\u652f\u6301\u4f7f\u7528\u8bfb\u5199\u6d41\u8fdb\u884c\u8bfb\u5199\u3002","title":"JindoData 4.6.12 \u7248\u672c\u8bf4\u660e"},{"location":"backup/release-notes/#jindodata-4612","text":"","title":"JindoData 4.6.12 \u7248\u672c\u8bf4\u660e"},{"location":"backup/release-notes/#_1","text":"JindoData 4.6.12 \u7248\u672c\u4f18\u5316\u4e86\u5bf9 Hadoop \u793e\u533a\u4f7f\u7528\u6587\u4ef6\u8bfb\u5199\u6d41\u7684\u517c\u5bb9\u6027\u3002","title":"\u7248\u672c\u6982\u8981"},{"location":"backup/release-notes/#_2","text":"JindoSDK \u5728 FileSystem close \u540e\u4ecd\u652f\u6301\u4f7f\u7528\u8bfb\u5199\u6d41\u8fdb\u884c\u8bfb\u5199\u3002","title":"\u4fee\u590d\u4ecb\u7ecd"},{"location":"backup/jindofs/jindo_dls_quickstart/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8 \u00a7 OSS-HDFS \u670d\u52a1\u662f\u963f\u91cc\u4e91\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9HDFS\u63a5\u53e3, \u652f\u6301\u76ee\u5f55\u4ee5\u53ca\u76ee\u5f55\u5c42\u7ea7\uff0cJindoSDK \u4e3a Apache Hadoop\u7684\u8ba1\u7b97\u5206\u6790\u5e94\u7528\uff08\u4f8b\u5982MapReduce\u3001Hive\u3001Spark\u3001Flink\u7b49\uff09\u63d0\u4f9b\u4e86\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u529f\u80fd\u3002\u5728\u7528\u6237\u73b0\u6709\u7684 Hadoop \u73af\u5883\u3001Hadoop \u96c6\u7fa4\u6216\u8005 Hadoop \u5ba2\u6237\u7aef\uff0c\u901a\u8fc7\u4fee\u6539 core-site.xml\uff0c\u5982\u4f55\u5bf9\u63a5\u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff0c\u53ef\u4ee5\u5feb\u901f\u67e5\u770b\u672c\u6587\u6863\u3002 1. \u670d\u52a1\u5f00\u901a \u00a7 \u8be6\u60c5\u53c2\u8003 \u5f00\u901a\u5e76\u6388\u6743\u8bbf\u95ee OSS-HDFS \u670d\u52a1 2. \u83b7\u53d6HDFS\u670d\u52a1\u57df\u540d \u00a7 \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u5728OSS\u7ba1\u7406\u63a7\u5236\u53f0\u7684\u6982\u89c8\u9875\u9762\uff0c\u590d\u5236HDFS\u670d\u52a1\u7684\u57df\u540d\u3002 3. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 4. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 5. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002 \u57fa\u672c\u64cd\u4f5c\u793a\u4f8b \u00a7 OSS-HDFS \u670d\u52a1\u521b\u5efa\u4ee5\u53ca\u914d\u7f6e\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7 hdfs dfs \u547d\u4ee4\u8fdb\u884c\u76f8\u5173\u6587\u4ef6/\u76ee\u5f55\u64cd\u4f5c \u65b0\u5efa\u76ee\u5f55 \u00a7 \u5728 OSS-HDFS \u670d\u52a1\u4e0a\u521b\u5efa\u76ee\u5f55\u3002 \u7528\u4f8b: hdfs dfs -mkdir oss://<Bucket>.<Endpoint>/Test/subdir [root@emr-header-1 ~]# hdfs dfs -mkdir oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test Found 1 items drwxr-x--x - root supergroup 0 2021-12-01 20:19 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir \u65b0\u5efa\u6587\u4ef6 \u00a7 \u5229\u7528 hdfs dfs -put \u547d\u4ee4\u4e0a\u4f20\u672c\u5730\u6587\u4ef6\u5230 OSS-HDFS \u670d\u52a1\u3002 \u7528\u4f8b\uff1a hdfs dfs -put <localfile> oss://<Bucket>.<Endpoint>/Test [root@emr-header-1 ~]# hdfs dfs -put /etc/hosts oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/ [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test Found 2 items -rw-r----- 1 root supergroup 5824 2021-12-01 20:24 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/hosts drwxr-x--x - root supergroup 0 2021-12-01 20:19 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir \u67e5\u770b\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u4fe1\u606f \u00a7 \u5728\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u521b\u5efa\u5b8c\u4e4b\u540e\uff0c\u53ef\u4ee5\u67e5\u770b\u6307\u5b9a\u8def\u5f84\u4e0b\u7684\u6587\u4ef6/\u76ee\u5f55\u4fe1\u606f\u3002hdfs dfs \u6ca1\u6709\u8fdb\u5165\u67d0\u4e2a\u76ee\u5f55\u4e0b\u7684\u6982\u5ff5\u3002\u5728\u67e5\u770b\u76ee\u5f55\u548c\u6587\u4ef6\u7684\u4fe1\u606f\u7684\u65f6\u5019\u9700\u8981\u7ed9\u51fa\u6587\u4ef6/\u76ee\u5f55\u7684\u7edd\u5bf9\u8def\u5f84\u3002 \u6307\u4ee4\uff1als \u7528\u4f8b\uff1a hdfs dfs -ls oss://<Bucket>.<Endpoint>/Test [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test Found 2 items -rw-r----- 1 root supergroup 5824 2021-12-01 20:24 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/hosts drwxr-x--x - root supergroup 0 2021-12-01 20:19 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir \u67e5\u770b\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u7684\u5927\u5c0f \u00a7 \u67e5\u770b\u5df2\u6709\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u7684\u5927\u5c0f \u7528\u4f8b\uff1a hdfs dfs -du oss://<Bucket>.<Endpoint>/Test [root@emr-header-1 ~]# hdfs dfs -du oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test 5824 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/hosts 0 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir \u67e5\u770b\u6587\u4ef6\u7684\u5185\u5bb9 \u00a7 \u6709\u65f6\u5019\u6211\u4eec\u9700\u8981\u67e5\u770b\u4e00\u4e0b\u5728 OSS-HDFS \u670d\u52a1\u6587\u4ef6\u7684\u5185\u5bb9\u3002hdfs dfs \u547d\u4ee4\u652f\u6301\u6211\u4eec\u5c06\u6587\u4ef6\u5185\u5bb9\u6253\u5370\u5728\u5c4f\u5e55\u4e0a\u3002\uff08\u8bf7\u6ce8\u610f\uff0c\u6587\u4ef6\u5185\u5bb9\u5c06\u4f1a\u4ee5\u7eaf\u6587\u672c\u5f62\u5f0f\u6253\u5370\u51fa\u6765\uff0c\u5982\u679c\u6587\u4ef6\u8fdb\u884c\u4e86\u7279\u5b9a\u683c\u5f0f\u7684\u7f16\u7801\uff0c\u8bf7\u4f7f\u7528 HDFS \u7684 JavaAPI \u5c06\u6587\u4ef6\u5185\u5bb9\u8bfb\u53d6\u5e76\u8fdb\u884c\u76f8\u5e94\u7684\u89e3\u7801\u83b7\u53d6\u6587\u4ef6\u5185\u5bb9\uff09 \u7528\u4f8b\uff1a hdfs dfs -cat oss://<Bucket>.<Endpoint>/Test/helloworld.txt [root@emr-header-1 ~]# hdfs dfs -cat oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/helloworld.txt hello world! \u590d\u5236\u76ee\u5f55/\u6587\u4ef6 \u00a7 \u6709\u65f6\u5019\u6211\u4eec\u9700\u8981\u5c06 OSS-HDFS \u670d\u52a1\u7684\u4e00\u4e2a\u6587\u4ef6/\u76ee\u5f55\u62f7\u8d1d\u5230\u53e6\u4e00\u4e2a\u4f4d\u7f6e\uff0c\u5e76\u4e14\u4fdd\u6301\u6e90\u6587\u4ef6\u548c\u76ee\u5f55\u7ed3\u6784\u548c\u5185\u5bb9\u4e0d\u53d8\u3002 \u7528\u4f8b\uff1a hdfs dfs -cp oss://<Bucket>.<Endpoint>/Test/subdir oss://<Bucket>.<Endpoint>/TestTarget/sudir2 [root@emr-header-1 ~]# hdfs dfs -cp oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/TestTarget/subdir1 [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/TestTarget/ Found 1 items drwxr-x--x - root supergroup 0 2021-12-01 20:37 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/TestTarget/subdir1 \u79fb\u52a8\u76ee\u5f55/\u6587\u4ef6 \u00a7 \u5728\u5f88\u591a\u5927\u6570\u636e\u5904\u7406\u7684\u4f8b\u5b50\u4e2d\uff0c\u6211\u4eec\u4f1a\u5c06\u6587\u4ef6\u5199\u5165\u4e00\u4e2a\u4e34\u65f6\u76ee\u5f55\uff0c\u7136\u540e\u5c06\u8be5\u76ee\u5f55\u79fb\u52a8\u5230\u53e6\u4e00\u4e2a\u4f4d\u7f6e\u4f5c\u4e3a\u6700\u7ec8\u7ed3\u679c\u3002\u6e90\u6587\u4ef6\u548c\u76ee\u5f55\u7ed3\u6784\u548c\u5185\u5bb9\u4e0d\u505a\u4fdd\u7559\u3002\u4e0b\u9762\u7684\u547d\u4ee4\u53ef\u4ee5\u5b8c\u6210\u8fd9\u4e9b\u64cd\u4f5c\u3002 \u7528\u4f8b\uff1a hdfs dfs -mv oss://<Bucket>.<Endpoint>/Test/subdir oss://<Bucket>.<Endpoint>/Test/subdir1 [root@emr-header-1 ~]# hdfs dfs -mv oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/newdir [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test Found 3 items -rw-r----- 1 root supergroup 13 2021-12-01 20:33 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/helloworld.txt -rw-r----- 1 root supergroup 5824 2021-12-01 20:24 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/hosts drwxr-x--x - root supergroup 0 2021-12-01 20:19 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/newdir \u4e0b\u8f7d\u6587\u4ef6\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf \u00a7 \u67d0\u4e9b\u60c5\u51b5\u4e0b\uff0c\u6211\u4eec\u9700\u8981\u5c06 OSS-HDFS \u670d\u52a1\u4e0a\u4e2d\u7684\u67d0\u4e9b\u6587\u4ef6\u4e0b\u8f7d\u5230\u672c\u5730\uff0c\u518d\u8fdb\u884c\u5904\u7406\u6216\u8005\u67e5\u770b\u5185\u5bb9\u3002\u8fd9\u4e2a\u53ef\u4ee5\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u5b8c\u6210\u3002 \u7528\u4f8b\uff1a hdfs dfs -get oss://<Bucket>.<Endpoint>/Test/helloworld.txt <localpath> [root@emr-header-1 ~]# hdfs dfs -get oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/helloworld.txt /tmp/ [root@emr-header-1 ~]# ll /tmp/helloworld.txt -rw-r----- 1 root root 13 12\u6708 1 20:44 /tmp/helloworld.txt \u5220\u9664\u76ee\u5f55/\u6587\u4ef6 \u00a7 \u5728\u5f88\u591a\u60c5\u51b5\u4e0b\uff0c\u6211\u4eec\u5728\u5b8c\u6210\u5de5\u4f5c\u540e\uff0c\u9700\u8981\u5220\u9664\u5728 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u67d0\u4e9b\u4e34\u65f6\u6587\u4ef6\u6216\u8005\u5e9f\u5f03\u6587\u4ef6\u3002\u8fd9\u4e9b\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u547d\u4ee4\u5b8c\u6210\u3002 \u7528\u4f8b\uff1a hdfs dfs -rm oss://<Bucket>.<Endpoint>/Test/helloworld.txt [root@emr-header-1 ~]# hdfs dfs -rm oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/helloworld.txt 21/12/01 20:46:44 INFO fs.TrashPolicyDefault: Moved: 'oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/helloworld.txt' to trash at: oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/user/root/.Trash/Current/Test/helloworld.txt [root@emr-header-1 ~]# hdfs dfs -rm -r oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/newdir 21/12/01 20:47:16 INFO fs.TrashPolicyDefault: Moved: 'oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/newdir' to trash at: oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/user/root/.Trash/Current/Test/newdir [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/ Found 1 items -rw-r----- 1 root supergroup 5824 2021-12-01 20:24 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/hosts","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8"},{"location":"backup/jindofs/jindo_dls_quickstart/#oss-hdfs-jindofs","text":"OSS-HDFS \u670d\u52a1\u662f\u963f\u91cc\u4e91\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9HDFS\u63a5\u53e3, \u652f\u6301\u76ee\u5f55\u4ee5\u53ca\u76ee\u5f55\u5c42\u7ea7\uff0cJindoSDK \u4e3a Apache Hadoop\u7684\u8ba1\u7b97\u5206\u6790\u5e94\u7528\uff08\u4f8b\u5982MapReduce\u3001Hive\u3001Spark\u3001Flink\u7b49\uff09\u63d0\u4f9b\u4e86\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u529f\u80fd\u3002\u5728\u7528\u6237\u73b0\u6709\u7684 Hadoop \u73af\u5883\u3001Hadoop \u96c6\u7fa4\u6216\u8005 Hadoop \u5ba2\u6237\u7aef\uff0c\u901a\u8fc7\u4fee\u6539 core-site.xml\uff0c\u5982\u4f55\u5bf9\u63a5\u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff0c\u53ef\u4ee5\u5feb\u901f\u67e5\u770b\u672c\u6587\u6863\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8"},{"location":"backup/jindofs/jindo_dls_quickstart/#1","text":"\u8be6\u60c5\u53c2\u8003 \u5f00\u901a\u5e76\u6388\u6743\u8bbf\u95ee OSS-HDFS \u670d\u52a1","title":"1. \u670d\u52a1\u5f00\u901a"},{"location":"backup/jindofs/jindo_dls_quickstart/#2-hdfs","text":"\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u5728OSS\u7ba1\u7406\u63a7\u5236\u53f0\u7684\u6982\u89c8\u9875\u9762\uff0c\u590d\u5236HDFS\u670d\u52a1\u7684\u57df\u540d\u3002","title":"2. \u83b7\u53d6HDFS\u670d\u52a1\u57df\u540d"},{"location":"backup/jindofs/jindo_dls_quickstart/#3-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"3. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindofs/jindo_dls_quickstart/#4-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"4. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofs/jindo_dls_quickstart/#5-oss-hdfs-access-key","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002","title":"5. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"backup/jindofs/jindo_dls_quickstart/#_1","text":"OSS-HDFS \u670d\u52a1\u521b\u5efa\u4ee5\u53ca\u914d\u7f6e\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7 hdfs dfs \u547d\u4ee4\u8fdb\u884c\u76f8\u5173\u6587\u4ef6/\u76ee\u5f55\u64cd\u4f5c","title":"\u57fa\u672c\u64cd\u4f5c\u793a\u4f8b"},{"location":"backup/jindofs/jindo_dls_quickstart/#_2","text":"\u5728 OSS-HDFS \u670d\u52a1\u4e0a\u521b\u5efa\u76ee\u5f55\u3002 \u7528\u4f8b: hdfs dfs -mkdir oss://<Bucket>.<Endpoint>/Test/subdir [root@emr-header-1 ~]# hdfs dfs -mkdir oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test Found 1 items drwxr-x--x - root supergroup 0 2021-12-01 20:19 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir","title":"\u65b0\u5efa\u76ee\u5f55"},{"location":"backup/jindofs/jindo_dls_quickstart/#_3","text":"\u5229\u7528 hdfs dfs -put \u547d\u4ee4\u4e0a\u4f20\u672c\u5730\u6587\u4ef6\u5230 OSS-HDFS \u670d\u52a1\u3002 \u7528\u4f8b\uff1a hdfs dfs -put <localfile> oss://<Bucket>.<Endpoint>/Test [root@emr-header-1 ~]# hdfs dfs -put /etc/hosts oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/ [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test Found 2 items -rw-r----- 1 root supergroup 5824 2021-12-01 20:24 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/hosts drwxr-x--x - root supergroup 0 2021-12-01 20:19 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir","title":"\u65b0\u5efa\u6587\u4ef6"},{"location":"backup/jindofs/jindo_dls_quickstart/#_4","text":"\u5728\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u521b\u5efa\u5b8c\u4e4b\u540e\uff0c\u53ef\u4ee5\u67e5\u770b\u6307\u5b9a\u8def\u5f84\u4e0b\u7684\u6587\u4ef6/\u76ee\u5f55\u4fe1\u606f\u3002hdfs dfs \u6ca1\u6709\u8fdb\u5165\u67d0\u4e2a\u76ee\u5f55\u4e0b\u7684\u6982\u5ff5\u3002\u5728\u67e5\u770b\u76ee\u5f55\u548c\u6587\u4ef6\u7684\u4fe1\u606f\u7684\u65f6\u5019\u9700\u8981\u7ed9\u51fa\u6587\u4ef6/\u76ee\u5f55\u7684\u7edd\u5bf9\u8def\u5f84\u3002 \u6307\u4ee4\uff1als \u7528\u4f8b\uff1a hdfs dfs -ls oss://<Bucket>.<Endpoint>/Test [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test Found 2 items -rw-r----- 1 root supergroup 5824 2021-12-01 20:24 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/hosts drwxr-x--x - root supergroup 0 2021-12-01 20:19 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir","title":"\u67e5\u770b\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u4fe1\u606f"},{"location":"backup/jindofs/jindo_dls_quickstart/#_5","text":"\u67e5\u770b\u5df2\u6709\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u7684\u5927\u5c0f \u7528\u4f8b\uff1a hdfs dfs -du oss://<Bucket>.<Endpoint>/Test [root@emr-header-1 ~]# hdfs dfs -du oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test 5824 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/hosts 0 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir","title":"\u67e5\u770b\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u7684\u5927\u5c0f"},{"location":"backup/jindofs/jindo_dls_quickstart/#_6","text":"\u6709\u65f6\u5019\u6211\u4eec\u9700\u8981\u67e5\u770b\u4e00\u4e0b\u5728 OSS-HDFS \u670d\u52a1\u6587\u4ef6\u7684\u5185\u5bb9\u3002hdfs dfs \u547d\u4ee4\u652f\u6301\u6211\u4eec\u5c06\u6587\u4ef6\u5185\u5bb9\u6253\u5370\u5728\u5c4f\u5e55\u4e0a\u3002\uff08\u8bf7\u6ce8\u610f\uff0c\u6587\u4ef6\u5185\u5bb9\u5c06\u4f1a\u4ee5\u7eaf\u6587\u672c\u5f62\u5f0f\u6253\u5370\u51fa\u6765\uff0c\u5982\u679c\u6587\u4ef6\u8fdb\u884c\u4e86\u7279\u5b9a\u683c\u5f0f\u7684\u7f16\u7801\uff0c\u8bf7\u4f7f\u7528 HDFS \u7684 JavaAPI \u5c06\u6587\u4ef6\u5185\u5bb9\u8bfb\u53d6\u5e76\u8fdb\u884c\u76f8\u5e94\u7684\u89e3\u7801\u83b7\u53d6\u6587\u4ef6\u5185\u5bb9\uff09 \u7528\u4f8b\uff1a hdfs dfs -cat oss://<Bucket>.<Endpoint>/Test/helloworld.txt [root@emr-header-1 ~]# hdfs dfs -cat oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/helloworld.txt hello world!","title":"\u67e5\u770b\u6587\u4ef6\u7684\u5185\u5bb9"},{"location":"backup/jindofs/jindo_dls_quickstart/#_7","text":"\u6709\u65f6\u5019\u6211\u4eec\u9700\u8981\u5c06 OSS-HDFS \u670d\u52a1\u7684\u4e00\u4e2a\u6587\u4ef6/\u76ee\u5f55\u62f7\u8d1d\u5230\u53e6\u4e00\u4e2a\u4f4d\u7f6e\uff0c\u5e76\u4e14\u4fdd\u6301\u6e90\u6587\u4ef6\u548c\u76ee\u5f55\u7ed3\u6784\u548c\u5185\u5bb9\u4e0d\u53d8\u3002 \u7528\u4f8b\uff1a hdfs dfs -cp oss://<Bucket>.<Endpoint>/Test/subdir oss://<Bucket>.<Endpoint>/TestTarget/sudir2 [root@emr-header-1 ~]# hdfs dfs -cp oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/TestTarget/subdir1 [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/TestTarget/ Found 1 items drwxr-x--x - root supergroup 0 2021-12-01 20:37 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/TestTarget/subdir1","title":"\u590d\u5236\u76ee\u5f55/\u6587\u4ef6"},{"location":"backup/jindofs/jindo_dls_quickstart/#_8","text":"\u5728\u5f88\u591a\u5927\u6570\u636e\u5904\u7406\u7684\u4f8b\u5b50\u4e2d\uff0c\u6211\u4eec\u4f1a\u5c06\u6587\u4ef6\u5199\u5165\u4e00\u4e2a\u4e34\u65f6\u76ee\u5f55\uff0c\u7136\u540e\u5c06\u8be5\u76ee\u5f55\u79fb\u52a8\u5230\u53e6\u4e00\u4e2a\u4f4d\u7f6e\u4f5c\u4e3a\u6700\u7ec8\u7ed3\u679c\u3002\u6e90\u6587\u4ef6\u548c\u76ee\u5f55\u7ed3\u6784\u548c\u5185\u5bb9\u4e0d\u505a\u4fdd\u7559\u3002\u4e0b\u9762\u7684\u547d\u4ee4\u53ef\u4ee5\u5b8c\u6210\u8fd9\u4e9b\u64cd\u4f5c\u3002 \u7528\u4f8b\uff1a hdfs dfs -mv oss://<Bucket>.<Endpoint>/Test/subdir oss://<Bucket>.<Endpoint>/Test/subdir1 [root@emr-header-1 ~]# hdfs dfs -mv oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/newdir [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test Found 3 items -rw-r----- 1 root supergroup 13 2021-12-01 20:33 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/helloworld.txt -rw-r----- 1 root supergroup 5824 2021-12-01 20:24 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/hosts drwxr-x--x - root supergroup 0 2021-12-01 20:19 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/newdir","title":"\u79fb\u52a8\u76ee\u5f55/\u6587\u4ef6"},{"location":"backup/jindofs/jindo_dls_quickstart/#_9","text":"\u67d0\u4e9b\u60c5\u51b5\u4e0b\uff0c\u6211\u4eec\u9700\u8981\u5c06 OSS-HDFS \u670d\u52a1\u4e0a\u4e2d\u7684\u67d0\u4e9b\u6587\u4ef6\u4e0b\u8f7d\u5230\u672c\u5730\uff0c\u518d\u8fdb\u884c\u5904\u7406\u6216\u8005\u67e5\u770b\u5185\u5bb9\u3002\u8fd9\u4e2a\u53ef\u4ee5\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u5b8c\u6210\u3002 \u7528\u4f8b\uff1a hdfs dfs -get oss://<Bucket>.<Endpoint>/Test/helloworld.txt <localpath> [root@emr-header-1 ~]# hdfs dfs -get oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/helloworld.txt /tmp/ [root@emr-header-1 ~]# ll /tmp/helloworld.txt -rw-r----- 1 root root 13 12\u6708 1 20:44 /tmp/helloworld.txt","title":"\u4e0b\u8f7d\u6587\u4ef6\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf"},{"location":"backup/jindofs/jindo_dls_quickstart/#_10","text":"\u5728\u5f88\u591a\u60c5\u51b5\u4e0b\uff0c\u6211\u4eec\u5728\u5b8c\u6210\u5de5\u4f5c\u540e\uff0c\u9700\u8981\u5220\u9664\u5728 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u67d0\u4e9b\u4e34\u65f6\u6587\u4ef6\u6216\u8005\u5e9f\u5f03\u6587\u4ef6\u3002\u8fd9\u4e9b\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u547d\u4ee4\u5b8c\u6210\u3002 \u7528\u4f8b\uff1a hdfs dfs -rm oss://<Bucket>.<Endpoint>/Test/helloworld.txt [root@emr-header-1 ~]# hdfs dfs -rm oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/helloworld.txt 21/12/01 20:46:44 INFO fs.TrashPolicyDefault: Moved: 'oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/helloworld.txt' to trash at: oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/user/root/.Trash/Current/Test/helloworld.txt [root@emr-header-1 ~]# hdfs dfs -rm -r oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/newdir 21/12/01 20:47:16 INFO fs.TrashPolicyDefault: Moved: 'oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/newdir' to trash at: oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/user/root/.Trash/Current/Test/newdir [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/ Found 1 items -rw-r----- 1 root supergroup 5824 2021-12-01 20:24 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/hosts","title":"\u5220\u9664\u76ee\u5f55/\u6587\u4ef6"},{"location":"backup/jindofs/jindo_util_intro/","text":"jindo \u547d\u4ee4\u4f7f\u7528\u65b9\u6cd5 \u00a7 (\u4ece 4.5.0 \u5f00\u59cb\u652f\u6301) \u4e0b\u8f7d jindo \u547d\u4ee4\u884c\u5de5\u5177\u5305\uff08\u5f53\u524d\u53ea\u652f\u6301 Linux \u73af\u5883\u4f7f\u7528\uff09 \u00a7 1\u3001\u4e0b\u8f7d jindosdk-4.6.12.tar.gz 2\u3001\u89e3\u538b tar \u5305 tar -zxvf jindosdk-4.6.12.tar.gz 3\u3001\u5728 /jindosdk-4.6.12/bin/ \u76ee\u5f55\u4e0b\u627e\u5230 jindo-util \u4e8c\u8fdb\u5236\u6587\u4ef6 chmod 700 jindo-util mv jindo-util jindo 4\u3001\u67e5\u770b jindo \u6240\u6709\u652f\u6301\u7684\u547d\u4ee4\u548c\u4f7f\u7528\u65b9\u6cd5 # ./jindo fs -help Usage: jindo fs [-appendToFile <localsrc> ... <dst>] [-archive [-i] [-c] <path>] [-cat <src> ...] [-count [-p] [-r] <path> ...] [-cp [-f] <src> ... <dst>] [-checksum <src> ...] [-chgrp [-R] GROUP PATH...] [-chmod [-R] <MODE[,MODE]... | OCTALMODE> PATH...] [-chown [-R] [OWNER][:[GROUP]] PATH...] [-copyFromLocal [-f] <localsrc> ... <dst>] [-copyToLocal [-f] <src> ... <localdst>] [-createSnapshot <snapshotDir> [<snapshotName>]] [-checkStoragePolicy -path <path>] [-deleteSnapshot <snapshotDir> <snapshotName>] [-du [-s] [-p] [-r] <path> ...] [-decommission -hosts <hosts>] [-formatCache] [-get [-f] <src> ... <localdst>] [-getfacl [-R] <path>] [-getfattr [-R] {-n name | -d} <path>] [-getStoragePolicy -path <path>] [-ls [-R] <path>] [-listPolicies] [-load [-meta] [-data] [-s] [-m] [-R] [-replica value] <path>] [-mkdir <path>] [-mv <src> ... <dst>] [-moveFromLocal <localsrc> ... <dst>] [-moveToLocal <src> <localdst>] [-metaDiff <path>] [-put [-f] <localsrc> ... <dst>] [-rm [-f] [-r|-R] <src> ...] [-rmdir <src> ...] [-renameSnapshot <snapshotDir> <oldName> <newName>] [-restore [-days value] <path>] [-stat [format] <path> ...] [-setStoragePolicy -path <path> -policy <policy>] [-setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>]] [-setfattr {-n name [-v value] | -x name} <path>] [-sync [-update] [-thread thread_num] <localsrc> <dst> [-touchz <path> ...] [-truncate [-w] <length> <path> ...] [-unarchive [-i] [-a] <path>] [-uncache <path>] [-unsetStoragePolicy -path <path>] jindo admin [-mount <mountpoint> <path>] [-report] [-reportMetrics] [-umount <mountpoint>] jindo admin [-addProxyUser -dlsUri <path> -proxyUser <value> -users|-groups <value> -hosts <value>] [-allowSnapshot -dlsUri <path>] [-addUserGroupsMapping -dlsUri <path> -user <value> -groups <value>] [-disallowSnapshot -dlsUri <path>] [-deleteProxyUser -dlsUri <path> -proxyUser <value>] [-deleteUserGroupsMapping -dlsUri <path> -user <value>] [-dumpFile -dlsUri <path>] [-dumpBlockFiles -in <path> -out <path>] [-listProxyUsers -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-listUserGroupsMappings -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-snapshotDiff -dlsUri <path> -fromSnapshot <value> -toSnapshot <value>] jindo benchmark [-io -read -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-memCheck]] [-io -pread -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-seekTimes <value>] [-memCheck]] [-io -write -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-memCheck]] [-meta -mkdir -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-memCheck]] [-meta -listDirectory -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]] [-meta -getFileStatus -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]] [-meta -renameFile -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]] \u4f7f\u7528 jindo \u547d\u4ee4\u8bbf\u95ee\u6807\u51c6 OSS \u6216\u8005 OSS-HDFS \u00a7 \u5168\u5c40\u914d\u7f6e \u00a7 1\u3001\u65b0\u5efa\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg [common] logger.dir = /tmp/jindo-util/ logger.sync = false logger.consolelogger = false logger.level = 0 logger.verbose = 0 logger.cleaner.enable = true hadoopConf.enable = false [jindosdk] fs.oss.endpoint = <ENDPOINT> fs.oss.accessKeyId = <ACCESS_KEYID> fs.oss.accessKeySecret = <ACCESS_KEYSECRET> < ENDPOINT>: \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 endpoint : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeyId : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeySecret 2\u3001\u6dfb\u52a0\u73af\u5883\u53d8\u91cf export JINDOSDK_CONF_DIR=<JINDOSDK_CFG_DIR> : jindosdk.cfg \u914d\u7f6e\u6587\u4ef6\u6240\u5728\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u6bd4\u5982 /etc/ 3\u3001\u4f7f\u7528 jindo \u547d\u4ee4\u8bbf\u95ee OSS ./jindo fs -ls oss://<bucket>/<dir> bucket \u7ea7\u522b\u914d\u7f6e \u00a7 1\u3001\u65b0\u5efa\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg [common] logger.dir = /tmp/jindo-util/ logger.sync = false logger.consolelogger = false logger.level = 0 logger.verbose = 0 logger.cleaner.enable = true hadoopConf.enable = false [jindosdk] fs.oss.bucket.<BUCKET>.accessKeyId = <ACCESS_KEYID> fs.oss.bucket.<BUCKET>.accessKeySecret = <ACCESS_KEYSECRET> fs.oss.bucket.<BUCKET>.endpoint = <ENDPOINT> < ENDPOINT>: \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 endpoint : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeyId : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeySecret 2\u3001\u6dfb\u52a0\u73af\u5883\u53d8\u91cf export JINDOSDK_CONF_DIR=<JINDOSDK_CFG_DIR> : jindosdk.cfg \u914d\u7f6e\u6587\u4ef6\u6240\u5728\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u6bd4\u5982 /etc/ 3\u3001\u4f7f\u7528 jindo \u547d\u4ee4\u8bbf\u95ee OSS-HDFS \u670d\u52a1 ./jindo fs -ls oss://<bucket>/<dir> \u4f7f\u7528 jindo sync \u540c\u6b65\u6570\u636e \u00a7 1\u3001\u65b0\u5efa\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg [common] logger.dir = /tmp/jindo-util/ logger.sync = false logger.consolelogger = false logger.level = 0 logger.verbose = 0 logger.cleaner.enable = true hadoopConf.enable = false [jindosdk] fs.oss.endpoint = <ENDPOINT> fs.oss.accessKeyId = <ACCESS_KEYID> fs.oss.accessKeySecret = <ACCESS_KEYSECRET> < ENDPOINT>: \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 endpoint : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeyId : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeySecret 2\u3001\u6dfb\u52a0\u73af\u5883\u53d8\u91cf export JINDOSDK_CONF_DIR=<JINDOSDK_CFG_DIR> : jindosdk.cfg \u914d\u7f6e\u6587\u4ef6\u6240\u5728\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u6bd4\u5982 /etc/ 3\u3001\u4f7f\u7528 jindo \u547d\u4ee4\u8fdb\u884c\u6570\u636e\u4f20\u8f93 ./jindo fs -sync -thread 10 /local/dir/ oss://<bucket>/<dir> thread: \u4f7f\u7528\u7ebf\u7a0b\u6570\u91cf 4\u3001\u4f7f\u7528\u7aef\u70b9\u7eed\u4f20\u529f\u80fd ./jindo fs -sync -update -thread 10 /local/dir/ oss://<bucket>/<dir> \u56e0\u4e3a\u9700\u8981\u505a\u6587\u4ef6\u6bd4\u8f83\uff0c\u4f7f\u7528\u65ad\u70b9\u7eed\u4f20\u53ef\u80fd\u5bf9\u4f20\u8f93\u6027\u80fd\u6709\u4e00\u5b9a\u7684\u5f71\u54cd","title":"jindo \u547d\u4ee4\u4f7f\u7528\u65b9\u6cd5"},{"location":"backup/jindofs/jindo_util_intro/#jindo","text":"(\u4ece 4.5.0 \u5f00\u59cb\u652f\u6301)","title":"jindo \u547d\u4ee4\u4f7f\u7528\u65b9\u6cd5"},{"location":"backup/jindofs/jindo_util_intro/#jindo-linux","text":"1\u3001\u4e0b\u8f7d jindosdk-4.6.12.tar.gz 2\u3001\u89e3\u538b tar \u5305 tar -zxvf jindosdk-4.6.12.tar.gz 3\u3001\u5728 /jindosdk-4.6.12/bin/ \u76ee\u5f55\u4e0b\u627e\u5230 jindo-util \u4e8c\u8fdb\u5236\u6587\u4ef6 chmod 700 jindo-util mv jindo-util jindo 4\u3001\u67e5\u770b jindo \u6240\u6709\u652f\u6301\u7684\u547d\u4ee4\u548c\u4f7f\u7528\u65b9\u6cd5 # ./jindo fs -help Usage: jindo fs [-appendToFile <localsrc> ... <dst>] [-archive [-i] [-c] <path>] [-cat <src> ...] [-count [-p] [-r] <path> ...] [-cp [-f] <src> ... <dst>] [-checksum <src> ...] [-chgrp [-R] GROUP PATH...] [-chmod [-R] <MODE[,MODE]... | OCTALMODE> PATH...] [-chown [-R] [OWNER][:[GROUP]] PATH...] [-copyFromLocal [-f] <localsrc> ... <dst>] [-copyToLocal [-f] <src> ... <localdst>] [-createSnapshot <snapshotDir> [<snapshotName>]] [-checkStoragePolicy -path <path>] [-deleteSnapshot <snapshotDir> <snapshotName>] [-du [-s] [-p] [-r] <path> ...] [-decommission -hosts <hosts>] [-formatCache] [-get [-f] <src> ... <localdst>] [-getfacl [-R] <path>] [-getfattr [-R] {-n name | -d} <path>] [-getStoragePolicy -path <path>] [-ls [-R] <path>] [-listPolicies] [-load [-meta] [-data] [-s] [-m] [-R] [-replica value] <path>] [-mkdir <path>] [-mv <src> ... <dst>] [-moveFromLocal <localsrc> ... <dst>] [-moveToLocal <src> <localdst>] [-metaDiff <path>] [-put [-f] <localsrc> ... <dst>] [-rm [-f] [-r|-R] <src> ...] [-rmdir <src> ...] [-renameSnapshot <snapshotDir> <oldName> <newName>] [-restore [-days value] <path>] [-stat [format] <path> ...] [-setStoragePolicy -path <path> -policy <policy>] [-setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>]] [-setfattr {-n name [-v value] | -x name} <path>] [-sync [-update] [-thread thread_num] <localsrc> <dst> [-touchz <path> ...] [-truncate [-w] <length> <path> ...] [-unarchive [-i] [-a] <path>] [-uncache <path>] [-unsetStoragePolicy -path <path>] jindo admin [-mount <mountpoint> <path>] [-report] [-reportMetrics] [-umount <mountpoint>] jindo admin [-addProxyUser -dlsUri <path> -proxyUser <value> -users|-groups <value> -hosts <value>] [-allowSnapshot -dlsUri <path>] [-addUserGroupsMapping -dlsUri <path> -user <value> -groups <value>] [-disallowSnapshot -dlsUri <path>] [-deleteProxyUser -dlsUri <path> -proxyUser <value>] [-deleteUserGroupsMapping -dlsUri <path> -user <value>] [-dumpFile -dlsUri <path>] [-dumpBlockFiles -in <path> -out <path>] [-listProxyUsers -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-listUserGroupsMappings -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-snapshotDiff -dlsUri <path> -fromSnapshot <value> -toSnapshot <value>] jindo benchmark [-io -read -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-memCheck]] [-io -pread -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-seekTimes <value>] [-memCheck]] [-io -write -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-memCheck]] [-meta -mkdir -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-memCheck]] [-meta -listDirectory -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]] [-meta -getFileStatus -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]] [-meta -renameFile -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]]","title":"\u4e0b\u8f7d jindo \u547d\u4ee4\u884c\u5de5\u5177\u5305\uff08\u5f53\u524d\u53ea\u652f\u6301 Linux \u73af\u5883\u4f7f\u7528\uff09"},{"location":"backup/jindofs/jindo_util_intro/#jindo-oss-oss-hdfs","text":"","title":"\u4f7f\u7528 jindo \u547d\u4ee4\u8bbf\u95ee\u6807\u51c6 OSS \u6216\u8005 OSS-HDFS"},{"location":"backup/jindofs/jindo_util_intro/#_1","text":"1\u3001\u65b0\u5efa\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg [common] logger.dir = /tmp/jindo-util/ logger.sync = false logger.consolelogger = false logger.level = 0 logger.verbose = 0 logger.cleaner.enable = true hadoopConf.enable = false [jindosdk] fs.oss.endpoint = <ENDPOINT> fs.oss.accessKeyId = <ACCESS_KEYID> fs.oss.accessKeySecret = <ACCESS_KEYSECRET> < ENDPOINT>: \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 endpoint : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeyId : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeySecret 2\u3001\u6dfb\u52a0\u73af\u5883\u53d8\u91cf export JINDOSDK_CONF_DIR=<JINDOSDK_CFG_DIR> : jindosdk.cfg \u914d\u7f6e\u6587\u4ef6\u6240\u5728\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u6bd4\u5982 /etc/ 3\u3001\u4f7f\u7528 jindo \u547d\u4ee4\u8bbf\u95ee OSS ./jindo fs -ls oss://<bucket>/<dir>","title":"\u5168\u5c40\u914d\u7f6e"},{"location":"backup/jindofs/jindo_util_intro/#bucket","text":"1\u3001\u65b0\u5efa\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg [common] logger.dir = /tmp/jindo-util/ logger.sync = false logger.consolelogger = false logger.level = 0 logger.verbose = 0 logger.cleaner.enable = true hadoopConf.enable = false [jindosdk] fs.oss.bucket.<BUCKET>.accessKeyId = <ACCESS_KEYID> fs.oss.bucket.<BUCKET>.accessKeySecret = <ACCESS_KEYSECRET> fs.oss.bucket.<BUCKET>.endpoint = <ENDPOINT> < ENDPOINT>: \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 endpoint : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeyId : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeySecret 2\u3001\u6dfb\u52a0\u73af\u5883\u53d8\u91cf export JINDOSDK_CONF_DIR=<JINDOSDK_CFG_DIR> : jindosdk.cfg \u914d\u7f6e\u6587\u4ef6\u6240\u5728\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u6bd4\u5982 /etc/ 3\u3001\u4f7f\u7528 jindo \u547d\u4ee4\u8bbf\u95ee OSS-HDFS \u670d\u52a1 ./jindo fs -ls oss://<bucket>/<dir>","title":"bucket \u7ea7\u522b\u914d\u7f6e"},{"location":"backup/jindofs/jindo_util_intro/#jindo-sync","text":"1\u3001\u65b0\u5efa\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg [common] logger.dir = /tmp/jindo-util/ logger.sync = false logger.consolelogger = false logger.level = 0 logger.verbose = 0 logger.cleaner.enable = true hadoopConf.enable = false [jindosdk] fs.oss.endpoint = <ENDPOINT> fs.oss.accessKeyId = <ACCESS_KEYID> fs.oss.accessKeySecret = <ACCESS_KEYSECRET> < ENDPOINT>: \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 endpoint : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeyId : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeySecret 2\u3001\u6dfb\u52a0\u73af\u5883\u53d8\u91cf export JINDOSDK_CONF_DIR=<JINDOSDK_CFG_DIR> : jindosdk.cfg \u914d\u7f6e\u6587\u4ef6\u6240\u5728\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u6bd4\u5982 /etc/ 3\u3001\u4f7f\u7528 jindo \u547d\u4ee4\u8fdb\u884c\u6570\u636e\u4f20\u8f93 ./jindo fs -sync -thread 10 /local/dir/ oss://<bucket>/<dir> thread: \u4f7f\u7528\u7ebf\u7a0b\u6570\u91cf 4\u3001\u4f7f\u7528\u7aef\u70b9\u7eed\u4f20\u529f\u80fd ./jindo fs -sync -update -thread 10 /local/dir/ oss://<bucket>/<dir> \u56e0\u4e3a\u9700\u8981\u505a\u6587\u4ef6\u6bd4\u8f83\uff0c\u4f7f\u7528\u65ad\u70b9\u7eed\u4f20\u53ef\u80fd\u5bf9\u4f20\u8f93\u6027\u80fd\u6709\u4e00\u5b9a\u7684\u5f71\u54cd","title":"\u4f7f\u7528 jindo sync \u540c\u6b65\u6570\u636e"},{"location":"backup/jindofs/jindofs_dls_checksum/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c \u00a7 (\u4ece 4.6.1 \u5f00\u59cb\u652f\u6301) \u901a\u8fc7 JindoDistCp \u4ece HDFS \u8fc1\u79fb\u5230 OSS-HDFS \u7684\u6587\u4ef6\uff0c\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u6821\u9a8c\u6587\u4ef6\u5185\u5bb9\u3002 \u6b65\u9aa41. \u00a7 \u8ba1\u7b97\u8f93\u51fa\u901a\u8fc7 JindoDistCp \u8fc1\u79fb\u7684\u76ee\u6807\u76ee\u5f55\u7684\u6587\u4ef6 checksum\u3002 \u547d\u4ee4\u683c\u5f0f \u00a7 jindo distjob -checksum --src <src> --dest <dest> --blockSize <blockSize> --recalculate \u53c2\u6570\u8bf4\u660e\uff1a --src <src> \uff1a\u5f85\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c\u7684 OSS-HDFS \u7684\u8def\u5f84\uff08\u5fc5\u586b\uff09\u3002\u6ce8\uff1asrc \u53ea\u80fd\u4e3a OSS-HDFS \u6216 HDFS \u8def\u5f84\u3002 --dest <dest> \uff1a\u8f93\u51fa checksum \u6587\u4ef6\u7684\u76ee\u6807\u8def\u5f84\uff08\u5fc5\u586b\uff09\u3002 --blockSize <blockSize> \uff1a\u6e90\u6587\u4ef6\u5199\u5165\u7684 blockSize\uff08\u53ef\u9009)\uff0c\u9ed8\u8ba4\u4e3a 134217728\u3002 --recalculate \uff1a\u662f\u5426\u8bfb\u53d6\u6587\u4ef6\u91cd\u65b0\u8ba1\u7b97 checksum\uff08\u53ef\u9009)\uff0c\u9ed8\u8ba4\u4e0d\u4f1a\u91cd\u65b0\u8ba1\u7b97\uff0c\u4ece OSS-HDFS \u8bfb\u53d6\u5199\u5165\u65f6\u8ba1\u7b97\u7684 checksum\u3002\u6ce8\uff1a\u8be5\u53c2\u6570\u53ea\u652f\u6301 --src \u4e3a OSS-HDFS\u3002 \u793a\u4f8b \u00a7 \u5047\u8bbe\u5728 OSS-HDFS \u7684 bucket \u540d\u4e3a dlsbucket\uff0c\u9700\u8981\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c\u7684\u76ee\u5f55\u4e3a oss://dlsbucket/test\uff0c\u8f93\u51fa checksum \u6587\u4ef6\u5230 OSS \u76ee\u5f55 oss://ossbucket/test-dls\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u547d\u4ee4\u4e3a\uff1a jindo distjob -checksum --src oss://dlsbucket/test --dest oss://ossbucket/test-dls --recalculate \u6b65\u9aa42. \u00a7 \u8ba1\u7b97\u8f93\u51fa JindoDistCp \u8fc1\u79fb\u7684\u6e90\u76ee\u5f55\u7684\u6587\u4ef6 checksum\u3002 \u793a\u4f8b \u00a7 \u5047\u8bbe JindoDistCp \u7684\u6e90\u76ee\u5f55\u4e3a hdfs:///test\uff0c\u8f93\u51fa checksum \u6587\u4ef6\u5230 OSS \u76ee\u5f55 oss://ossbucket/test-hdfs\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u547d\u4ee4\u4e3a\uff1a jindo distjob -checksum --src hdfs:///test --dest oss://ossbucket/test-hdfs \u6b65\u9aa43. \u00a7 \u901a\u8fc7 JindoDistCp --diff \u5bf9\u6bd4\u6b65\u9aa41\u548c\u6b65\u9aa42\u8f93\u51fa\u7684\u6587\u4ef6 checksum \u76ee\u5f55 \u793a\u4f8b \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src oss://ossbucket/test-hdfs --dest oss://ossbucket/test-dls --diff","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c"},{"location":"backup/jindofs/jindofs_dls_checksum/#oss-hdfs-jindofs","text":"(\u4ece 4.6.1 \u5f00\u59cb\u652f\u6301) \u901a\u8fc7 JindoDistCp \u4ece HDFS \u8fc1\u79fb\u5230 OSS-HDFS \u7684\u6587\u4ef6\uff0c\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u6821\u9a8c\u6587\u4ef6\u5185\u5bb9\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c"},{"location":"backup/jindofs/jindofs_dls_checksum/#1","text":"\u8ba1\u7b97\u8f93\u51fa\u901a\u8fc7 JindoDistCp \u8fc1\u79fb\u7684\u76ee\u6807\u76ee\u5f55\u7684\u6587\u4ef6 checksum\u3002","title":"\u6b65\u9aa41."},{"location":"backup/jindofs/jindofs_dls_checksum/#_1","text":"jindo distjob -checksum --src <src> --dest <dest> --blockSize <blockSize> --recalculate \u53c2\u6570\u8bf4\u660e\uff1a --src <src> \uff1a\u5f85\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c\u7684 OSS-HDFS \u7684\u8def\u5f84\uff08\u5fc5\u586b\uff09\u3002\u6ce8\uff1asrc \u53ea\u80fd\u4e3a OSS-HDFS \u6216 HDFS \u8def\u5f84\u3002 --dest <dest> \uff1a\u8f93\u51fa checksum \u6587\u4ef6\u7684\u76ee\u6807\u8def\u5f84\uff08\u5fc5\u586b\uff09\u3002 --blockSize <blockSize> \uff1a\u6e90\u6587\u4ef6\u5199\u5165\u7684 blockSize\uff08\u53ef\u9009)\uff0c\u9ed8\u8ba4\u4e3a 134217728\u3002 --recalculate \uff1a\u662f\u5426\u8bfb\u53d6\u6587\u4ef6\u91cd\u65b0\u8ba1\u7b97 checksum\uff08\u53ef\u9009)\uff0c\u9ed8\u8ba4\u4e0d\u4f1a\u91cd\u65b0\u8ba1\u7b97\uff0c\u4ece OSS-HDFS \u8bfb\u53d6\u5199\u5165\u65f6\u8ba1\u7b97\u7684 checksum\u3002\u6ce8\uff1a\u8be5\u53c2\u6570\u53ea\u652f\u6301 --src \u4e3a OSS-HDFS\u3002","title":"\u547d\u4ee4\u683c\u5f0f"},{"location":"backup/jindofs/jindofs_dls_checksum/#_2","text":"\u5047\u8bbe\u5728 OSS-HDFS \u7684 bucket \u540d\u4e3a dlsbucket\uff0c\u9700\u8981\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c\u7684\u76ee\u5f55\u4e3a oss://dlsbucket/test\uff0c\u8f93\u51fa checksum \u6587\u4ef6\u5230 OSS \u76ee\u5f55 oss://ossbucket/test-dls\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u547d\u4ee4\u4e3a\uff1a jindo distjob -checksum --src oss://dlsbucket/test --dest oss://ossbucket/test-dls --recalculate","title":"\u793a\u4f8b"},{"location":"backup/jindofs/jindofs_dls_checksum/#2","text":"\u8ba1\u7b97\u8f93\u51fa JindoDistCp \u8fc1\u79fb\u7684\u6e90\u76ee\u5f55\u7684\u6587\u4ef6 checksum\u3002","title":"\u6b65\u9aa42."},{"location":"backup/jindofs/jindofs_dls_checksum/#_3","text":"\u5047\u8bbe JindoDistCp \u7684\u6e90\u76ee\u5f55\u4e3a hdfs:///test\uff0c\u8f93\u51fa checksum \u6587\u4ef6\u5230 OSS \u76ee\u5f55 oss://ossbucket/test-hdfs\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u547d\u4ee4\u4e3a\uff1a jindo distjob -checksum --src hdfs:///test --dest oss://ossbucket/test-hdfs","title":"\u793a\u4f8b"},{"location":"backup/jindofs/jindofs_dls_checksum/#3","text":"\u901a\u8fc7 JindoDistCp --diff \u5bf9\u6bd4\u6b65\u9aa41\u548c\u6b65\u9aa42\u8f93\u51fa\u7684\u6587\u4ef6 checksum \u76ee\u5f55","title":"\u6b65\u9aa43."},{"location":"backup/jindofs/jindofs_dls_checksum/#_4","text":"hadoop jar jindo-distcp-tool-${version}.jar --src oss://ossbucket/test-hdfs --dest oss://ossbucket/test-dls --diff","title":"\u793a\u4f8b"},{"location":"backup/jindofs/jindofs_dls_dump_inventory_howto/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5143\u6570\u636e\u5bfc\u51fa\u4f7f\u7528\u8bf4\u660e \u00a7 (\u4ece 4.6.0 \u5f00\u59cb\u652f\u6301) \u4ecb\u7ecd \u00a7 \u4f7f\u7528\u5143\u6570\u636e\u5bfc\u51fa\u529f\u80fd\uff0c\u53ef\u4ee5\u5c06\u5f53\u524d OSS-HDFS bucket \u4e0b\u7684\u6587\u4ef6\u5143\u6570\u636e\u6e05\u5355\u5bfc\u51fa\u5230 OSS \u4e0a\u7684\uff0c\u683c\u5f0f\u4e3a json \u6587\u4ef6\uff0c\u65b9\u4fbf\u7528\u6237\u5bf9\u5143\u6570\u636e\u8fdb\u884c\u7edf\u8ba1\u5206\u6790 \u914d\u7f6e jindo \u547d\u4ee4\u884c\u5de5\u5177\uff0c\u914d\u7f6e\u5bf9\u5e94 OSS-HDFS bucket \u7684\u8bbf\u95ee\u5bc6\u94a5\uff0c\u53c2\u8003 jindo \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u8bf4\u660e \u6267\u884c\u5bfc\u51fa\u547d\u4ee4 jindo admin -dumpInventory oss://<hdfs_bucket>/ \u6b64\u65f6\u53ef\u4ee5\u89c2\u5bdf\u5230\u8f93\u51fa\u8def\u5f84 ============Dump Inventory============= Job Id: 2ce40fba-5704-45c4-8720-d92a891d5cfd Data Location: oss://<oss_bucket>/.dlsdata/.sysinfo/meta_analyze/inventory/1666584461201.2ce40fba-5704-45c4-8720-d92a891d5cfd ..................................................................................................................... FINISHED. \u8be5\u547d\u4ee4\u4e3a\u963b\u585e\u547d\u4ee4\uff0c\u8bf7\u8010\u5fc3\u7b49\u5f8510\u79d2\u949f~10\u5206\u949f\uff08\u6839\u636e\u5143\u6570\u636e\u91cf\u5927\u5c0f\uff09\uff0c\u77e5\u9053\u6700\u540e\u8f93\u51fa FINISHED \u8868\u793a\u5bfc\u51fa\u6210\u529f\u3002 \u4e0b\u8f7d\u7ed3\u679c\u6587\u4ef6 \u4f7f\u7528 jindo \u547d\u4ee4\u884c\u5de5\u5177 \uff08\u914d\u7f6e\u666e\u901aOSS\u7684key\u975eOSS-HDFS\uff09 \u3001\u6216 ossutil \u3001\u6216\u4f7f\u7528Hadoop fs \u547d\u4ee4\u3001\u6216\u5728OSS\u63a7\u5236\u53f0\u9875\u9762\uff0c\u4e0b\u8f7d\u7ed3\u679c\u6587\u4ef6\u3002 ossutil cp oss://<oss_bucket>/.dlsdata/.sysinfo/meta_analyze/inventory/1666584461201.2ce40fba-5704-45c4-8720-d92a891d5cfd ./ \u4e0b\u8f7d\u5230\u672c\u5730\uff0c\u4f7f\u7528vi/vim\u6253\u5f00\u5373\u53ef\u3002 \u793a\u4f8b\u7ed3\u679c\u53c2\u8003 {\"id\":16385,\"path\":\"/\",\"type\":\"directory\",\"size\":0,\"user\":\"admin\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1666581702933,\"permission\":511,\"state\":1} {\"id\":6246684106789500068,\"path\":\"/dls-1000326249\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500069,\"path\":\"/dls-1000326249/benchmark\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500070,\"path\":\"/dls-1000326249/benchmark/n1\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500071,\"path\":\"/dls-1000326249/benchmark/n1/490747449\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660895613953,\"permission\":511,\"state\":0}","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5143\u6570\u636e\u5bfc\u51fa\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindofs_dls_dump_inventory_howto/#oss-hdfs-jindofs","text":"(\u4ece 4.6.0 \u5f00\u59cb\u652f\u6301)","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5143\u6570\u636e\u5bfc\u51fa\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindofs_dls_dump_inventory_howto/#_1","text":"\u4f7f\u7528\u5143\u6570\u636e\u5bfc\u51fa\u529f\u80fd\uff0c\u53ef\u4ee5\u5c06\u5f53\u524d OSS-HDFS bucket \u4e0b\u7684\u6587\u4ef6\u5143\u6570\u636e\u6e05\u5355\u5bfc\u51fa\u5230 OSS \u4e0a\u7684\uff0c\u683c\u5f0f\u4e3a json \u6587\u4ef6\uff0c\u65b9\u4fbf\u7528\u6237\u5bf9\u5143\u6570\u636e\u8fdb\u884c\u7edf\u8ba1\u5206\u6790 \u914d\u7f6e jindo \u547d\u4ee4\u884c\u5de5\u5177\uff0c\u914d\u7f6e\u5bf9\u5e94 OSS-HDFS bucket \u7684\u8bbf\u95ee\u5bc6\u94a5\uff0c\u53c2\u8003 jindo \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u8bf4\u660e \u6267\u884c\u5bfc\u51fa\u547d\u4ee4 jindo admin -dumpInventory oss://<hdfs_bucket>/ \u6b64\u65f6\u53ef\u4ee5\u89c2\u5bdf\u5230\u8f93\u51fa\u8def\u5f84 ============Dump Inventory============= Job Id: 2ce40fba-5704-45c4-8720-d92a891d5cfd Data Location: oss://<oss_bucket>/.dlsdata/.sysinfo/meta_analyze/inventory/1666584461201.2ce40fba-5704-45c4-8720-d92a891d5cfd ..................................................................................................................... FINISHED. \u8be5\u547d\u4ee4\u4e3a\u963b\u585e\u547d\u4ee4\uff0c\u8bf7\u8010\u5fc3\u7b49\u5f8510\u79d2\u949f~10\u5206\u949f\uff08\u6839\u636e\u5143\u6570\u636e\u91cf\u5927\u5c0f\uff09\uff0c\u77e5\u9053\u6700\u540e\u8f93\u51fa FINISHED \u8868\u793a\u5bfc\u51fa\u6210\u529f\u3002 \u4e0b\u8f7d\u7ed3\u679c\u6587\u4ef6 \u4f7f\u7528 jindo \u547d\u4ee4\u884c\u5de5\u5177 \uff08\u914d\u7f6e\u666e\u901aOSS\u7684key\u975eOSS-HDFS\uff09 \u3001\u6216 ossutil \u3001\u6216\u4f7f\u7528Hadoop fs \u547d\u4ee4\u3001\u6216\u5728OSS\u63a7\u5236\u53f0\u9875\u9762\uff0c\u4e0b\u8f7d\u7ed3\u679c\u6587\u4ef6\u3002 ossutil cp oss://<oss_bucket>/.dlsdata/.sysinfo/meta_analyze/inventory/1666584461201.2ce40fba-5704-45c4-8720-d92a891d5cfd ./ \u4e0b\u8f7d\u5230\u672c\u5730\uff0c\u4f7f\u7528vi/vim\u6253\u5f00\u5373\u53ef\u3002 \u793a\u4f8b\u7ed3\u679c\u53c2\u8003 {\"id\":16385,\"path\":\"/\",\"type\":\"directory\",\"size\":0,\"user\":\"admin\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1666581702933,\"permission\":511,\"state\":1} {\"id\":6246684106789500068,\"path\":\"/dls-1000326249\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500069,\"path\":\"/dls-1000326249/benchmark\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500070,\"path\":\"/dls-1000326249/benchmark/n1\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500071,\"path\":\"/dls-1000326249/benchmark/n1/490747449\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660895613953,\"permission\":511,\"state\":0}","title":"\u4ecb\u7ecd"},{"location":"backup/jindofs/jindofs_dls_hsm_howto/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5206\u5c42\u5b58\u50a8\u4f7f\u7528\u8bf4\u660e \u00a7 (\u4ece 4.6.2 \u5f00\u59cb\u652f\u6301) \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u662f OSS \u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9HDFS\u63a5\u53e3, \u652f\u6301\u76ee\u5f55\u4ee5\u53ca\u76ee\u5f55\u5c42\u7ea7\uff0c\u901a\u8fc7 Jindo SDK 4.x \u53ef\u4ee5\u517c\u5bb9\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u3002\u5173\u4e8e\u7528\u6237\u5982\u4f55\u521b\u5efa\u548c\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u7684\u57fa\u672c\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 \u94fe\u63a5 \u3002 \u672c\u6587\u4e3b\u8981\u4ecb\u7ecd\u5728 OSS-HDFS \u670d\u52a1\u4e2d\u4f7f\u7528\u5206\u5c42\u5b58\u50a8\u529f\u80fd\u7684\u4e00\u4e9b\u5e38\u89c1\u64cd\u4f5c\u3002 \u4e3a\u4e86\u65b9\u4fbf\u4e0b\u9762\u547d\u4ee4\u7684\u4ecb\u7ecd\uff0c\u6211\u4eec\u5047\u8bbe oss://oss-dfs-test \u8fd9\u4e2aBucket\u5df2\u7ecf\u5f00\u542f OSS-HDFS \u670d\u52a1\u3002\u4e0b\u9762\u6240\u6709\u7684\u4f8b\u5b50\u90fd\u4f1a\u9488\u5bf9\u8fd9\u4e2aBucket\u8fdb\u884c\u64cd\u4f5c\u3002 \u5206\u5c42\u5b58\u50a8\u7b80\u4ecb \u00a7 \u6570\u636e\u6e56\u5b58\u50a8\u7684\u6570\u636e\u91cf\u5e38\u5e38\u4f1a\u968f\u7740\u65f6\u95f4\u800c\u4e0d\u65ad\u589e\u957f\uff0c\u4f46\u5728\u6570\u636e\u6e56\u5185\u7684\u6570\u636e\u5e76\u975e\u540c\u6837\u9891\u7e41\u8bbf\u95ee\u3002\u5bf9\u4e8e\u8f83\u8001\u7684\u6570\u636e\uff0c\u6bd4\u5982\u8d85\u8fc7\u4e00\u5e74\u7684\u6570\u636e\uff0c\u4e1a\u52a1\u4e0a\u53ef\u80fd\u51e0\u4e4e\u4e0d\u518d\u8bbf\u95ee\uff0c\u4f46\u4e3a\u4fdd\u8bc1\u5408\u89c4\u3001\u5b58\u6863\u7b49\u76ee\u7684\uff0c\u8fd9\u4e9b\u6570\u636e\u4ecd\u7136\u9700\u8981\u5b58\u50a8\u4ee5\u5907\u4e0d\u65f6\u4e4b\u9700\u3002\u56e0\u6b64\uff0c\u5206\u5c42\u5b58\u50a8\u7684\u9700\u6c42\u5c31\u51fa\u73b0\u4e86\uff1a\u5bf9\u4e8e\u8f83\u5c11\u8bbf\u95ee\u7684\u6570\u636e\uff0c\u9009\u62e9\u8f83\u4f4e\u6210\u672c\u7684\u5b58\u50a8\u4ecb\u8d28\uff08\u901a\u5e38\u4f1a\u6709\u66f4\u9ad8\u7684\u8bbf\u95ee\u6210\u672c\uff09\u8fdb\u884c\u5b58\u50a8\uff0c\u53ef\u4ee5\u964d\u4f4e\u603b\u5b58\u50a8\u6210\u672c\u3002 \u6807\u51c6 OSS Bucket \u652f\u6301\u591a\u79cd\u5206\u5c42\u5b58\u50a8\u7c7b\u578b\uff0c\u5305\u62ec\u6807\u51c6\u3001\u4f4e\u9891\u3001\u5f52\u6863\u3001\u51b7\u5f52\u6863\u56db\u79cd\uff0c\u800c JindoFS \u670d\u52a1\u4f7f\u7528\u6807\u51c6 OSS \u4f5c\u4e3a\u6570\u636e\u5757\u7684\u5b58\u50a8\u540e\u7aef\uff0c\u56e0\u6b64 JindoFS \u5206\u5c42\u5b58\u50a8\u529f\u80fd\u5373\u8bbe\u7f6e\u540e\u7aef\u6570\u636e\u5757\u5728\u6807\u51c6 OSS \u4e0a\u6240\u4f7f\u7528\u7684\u5b58\u50a8\u7c7b\u578b\u3002 \u4f7f\u7528\u5206\u5c42\u5b58\u50a8 \u00a7 \u5728\u4f7f\u7528\u76f8\u5173\u529f\u80fd\u524d\uff0c\u5e94\u786e\u4fdd JindoSDK \u5df2\u7ecf\u6b63\u786e\u914d\u7f6e\uff0c\u53ef\u4ee5\u6b63\u5e38\u8bbf\u95ee JindoFS \u670d\u52a1\u3002JindoFS \u670d\u52a1\u7684\u5206\u5c42\u5b58\u50a8\u529f\u80fd\u4e0e HCFS API \u517c\u5bb9\u3002\u76f8\u5173 API \u5982\u4e0b\u6240\u793a\uff1a public abstract class FileSystem extends Configured implements Closeable { // ...... public void setStoragePolicy(Path src, String policyName) throws IOException; public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException; public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies() throws IOException; } JindoFS \u4e5f\u652f\u6301\u4f7f\u7528\u4e0b\u5217\u547d\u4ee4\u64cd\u4f5c\u5b58\u50a8\u7b56\u7565\uff1a jindo fs -listPolicies jindo fs -setStoragePolicy -path <path> -policy <policy> jindo fs -getStoragePolicy -path <path> getAllStoragePolicies() \u63a5\u53e3\u548c listPolicies \u547d\u4ee4\u8fd4\u56de\u5f53\u524d\u652f\u6301\u7684\u5b58\u50a8\u7b56\u7565\uff0c\u76ee\u524d\u652f\u6301\u7684\u5b58\u50a8\u7b56\u7565\u6709 CLOUD_STD \uff08\u6807\u51c6\u5b58\u50a8\uff09\u3001 CLOUD_IA \uff08\u4f4e\u9891\u5b58\u50a8\uff09\u3001 CLOUD_AR \uff08\u5f52\u6863\uff09\u548c CLOUD_COLD_AR \uff08\u51b7\u5f52\u6863\uff09\u3002 setStoragePolicy() \u63a5\u53e3\u548c setStoragePolicy \u547d\u4ee4\u53ef\u4ee5\u4e3a\u5177\u4f53\u7684\u8def\u5f84\u8bbe\u7f6e\u5b58\u50a8\u7b56\u7565\uff0cJindoFS \u670d\u52a1\u4f1a\u5728\u540e\u53f0\u6839\u636e\u5b58\u50a8\u7b56\u7565\u6539\u53d8\u5b58\u50a8\u5bf9\u8c61\u7684\u5b58\u50a8\u7ea7\u522b\u3002\u4f8b\u5982\uff0c\u8981\u5c06 oss://oss-dfs-test/dir1 \u8bbe\u7f6e\u4e3a\u5f52\u6863\u7c7b\u578b\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a jindo fs -setStoragePolicy -path oss://oss-dfs-test/dir1 -policy CLOUD_AR \u6ce8\u610f\u6587\u4ef6\u6216\u76ee\u5f55\u7684\u9ed8\u8ba4\u5b58\u50a8\u7b56\u7565\u4e3a\u7a7a\uff0c\u663e\u793a\u4e3a UNSPECIFIED \uff0c\u5f53\u5bf9\u8c61\u5b58\u50a8\u7b56\u7565\u4e3a UNSPECIFIED \u65f6\uff0c\u5bf9\u8c61\u5b9e\u9645\u7684\u5b58\u50a8\u7b56\u7565\u5c06\u7531\u6700\u8fd1\u7684\u8bbe\u7f6e\u4e86\u5b58\u50a8\u7b56\u7565\u7684\u7236\u8282\u70b9\u51b3\u5b9a\u3002 \u4f8b\u5982\u5047\u8bbe oss://oss-dfs-test/dir2 \u7684\u5b58\u50a8\u7b56\u7565\u4e3a CLOUD_STD \uff0c\u5219 oss://oss-dfs-test/dir2/subdir2/subsubdir2 \u7684\u5b58\u50a8\u7b56\u7565\u4e5f\u4e3a CLOUD_STD \uff0c \u9664\u975e\u7528\u6237\u5bf9 oss://oss-dfs-test/dir2/subdir2/ \u6216 oss://oss-dfs-test/dir2/subdir2/subsubdir2 \u5355\u72ec\u8bbe\u7f6e\u8fc7\u5b58\u50a8\u7b56\u7565\u3002 \u7528\u6237\u53ef\u4ee5\u901a\u8fc7 getStoragePolicy() \u63a5\u53e3\u548c getStoragePolicy \u547d\u4ee4\u67e5\u8be2\u67d0\u4e00\u8def\u5f84\u6700\u7ec8\u751f\u6548\u7684\u5b58\u50a8\u7b56\u7565\uff0c\u5982\uff1a jindo fs -getStoragePolicy -path oss://oss-dfs-test/dir1/file1 \u5982\u679c\u6700\u7ec8\u751f\u6548\u7684\u5b58\u50a8\u7b56\u7565\u4e3a UNSPECIFIED \uff08\u5373 getStoragePolicy() \u63a5\u53e3\u8fd4\u56de null \uff09\uff0c\u5219\u5bf9\u8c61\u9ed8\u8ba4\u4f7f\u7528\u6807\u51c6\u5b58\u50a8\u3002 \u76ee\u5f55\u5d4c\u5957 \u00a7 \u76ee\u524d JindoFS \u4e0d\u652f\u6301\u5d4c\u5957\u7b56\u7565 , \u4f8b\u5982\u5bf9 oss://oss-dfs-test/warehouse/dwd.db/dt=20220101 \u4f7f\u7528\u4e86 CLOUD_AR \u7b56\u7565, \u518d\u5bf9 oss://oss-dfs-test/warehouse/dwd.db \u7236\u76ee\u5f55\u4f7f\u7528 CLOUD_STD \u7b56\u7565, \u662f\u4e0d\u4f1a\u6267\u884c\u89e3\u5f52\u6863\u64cd\u4f5c\u7684. \u56e0\u6b64\u5efa\u8bae \u53ea\u5bf9\u4e4b\u524d\u8bbe\u7f6e\u8fc7policy\u7684\u76ee\u5f55\u4fee\u6539policy, \u4e0d\u8981\u518d\u4fee\u6539\u5176\u4e0a\u4e0b\u7ea7\u76ee\u5f55\u7684policy. \u5404\u79cd\u573a\u666f\u4e0b Policy \u4e92\u8f6c\u7684\u4f8b\u5b50 \u00a7 \u6807\u51c6 \u8f6c \u4f4e\u9891IA jindo fs -setStoragePolicy -path <path> -policy CLOUD_IA \u6807\u51c6 \u8f6c \u5f52\u6863CLOUD_AR jindo fs -setStoragePolicy -path <path> -policy CLOUD_AR \u6807\u51c6 \u8f6c \u51b7\u5f52\u6863CLOUD_COLD_AR jindo fs -setStoragePolicy -path <path> -policy CLOUD_COLD_AR \u4f4e\u9891IA \u8f6c \u5f52\u6863CLOUD_AR jindo fs -setStoragePolicy -path <path> -policy CLOUD_AR \u4f4e\u9891IA \u8f6c \u51b7\u5f52\u6863CLOUD_COLD_AR jindo fs -setStoragePolicy -path <path> -policy CLOUD_COLD_AR \u4f4e\u9891IA \u8f6c \u6807\u51c6 jindo fs -setStoragePolicy -path <path> -policy CLOUD_STD \u5f52\u6863 \u8f6c \u51b7\u5f52\u6863 \u4e0d\u652f\u6301 \u51b7\u5f52\u6863 \u8f6c \u5f52\u6863 \u4e0d\u652f\u6301 \u5f52\u6863 \u8f6c \u6807\u51c6 jindo fs -setStoragePolicy -path <path> -policy CLOUD_STD \u51b7\u5f52\u6863 \u8f6c \u6807\u51c6 jindo fs -setStoragePolicy -path <path> -policy CLOUD_STD \u8bbe\u7f6eOSS\u751f\u547d\u5468\u671f \u00a7 \u5728\u4f7f\u7528\u5206\u5c42\u5b58\u50a8\u529f\u80fd\u5c06OSS-HDFS\u7684\u6570\u636e\u8fdb\u884c\u5f52\u6863\u7684\u65f6\u5019\uff0c\u9700\u8981\u4f9d\u8d56\u4e8eOSS\u751f\u547d\u5468\u671f\u529f\u80fd\uff0c\u901a\u8fc7\u4e0b\u9762\u547d\u4ee4 jindo fs -setStoragePolicy -path oss://oss-dfs-test/dir1 -policy CLOUD_AR \u8fdb\u884c\u6570\u636e\u5f52\u6863\u64cd\u4f5c\uff0c\u8be5\u64cd\u4f5c\u53ea\u4f1a\u5c06\u5bf9\u5e94\u76ee\u5f55\u4e0b\u7684\u6570\u636e\u8fdb\u884cTag\u6807\u8bb0, Tag\u5bf9\u5e94\u7684\u952e\u503c\u5bf9\u4e3atransition-storage-class:Archive\uff0c\u9700\u8981\u8bbe\u7f6e\u751f\u547d\u5468\u671f\u8fdb\u884c\u6570\u636e\u7684\u8f6c\u6362\uff0c\u8bbe\u7f6e\u6309\u524d\u7f00\u5339\u914d\u7b56\u7565\u89c4\u5219\uff0c\u524d\u7f00\u4e3a.dlsdata, \u5177\u4f53\u5f52\u6863\u7684\u751f\u547d\u5468\u671f\u7b56\u7565\u8bbe\u7f6e\u5982\u4e0b\uff1a \u4e0a\u8ff0\u751f\u547d\u5468\u671f\u7b56\u7565\u8bbe\u7f6e\u5b8c\u6bd5\u540e\uff0c\u7528\u6237\u9700\u8981\u7b49\u5f85\u6539OSS\u751f\u547d\u5468\u671f\u7b56\u7565\u88ab\u8c03\u5ea6\u6267\u884c\u624d\u80fd\u5b8c\u6210\u6570\u636e\u7684\u5f52\u6863\u3002\u5982\u4f55\u8bbe\u7f6e\u751f\u547d\u5468\u671f\u53ef\u4ee5\u53c2\u8003 \u6587\u6863 \u3002 \u6ce8\u610f\u4e8b\u9879 \u00a7 \u5982\u679c\u9700\u8981\u4f7f\u7528\u5f52\u6863/\u51b7\u5f52\u6863\u7684\u5b58\u50a8\u7b56\u7565\uff0c\u9700\u8981\u5728 OSS \u63a7\u5236\u53f0\u4e0a\u914d\u7f6e\u76f8\u5e94\u7684\u751f\u547d\u5468\u671f\u7b56\u7565\uff0c\u5426\u5219\u5b58\u50a8\u7684\u6570\u636e\u5757\u65e0\u6cd5\u771f\u6b63\u8f6c\u4e3a\u5f52\u6863/\u51b7\u5f52\u6863\u7c7b\u578b\u3002 \u5728 JindoFS \u4e0a\u67e5\u8be2\u5230\u7684\u5b58\u50a8\u72b6\u6001\u4ec5\u8868\u793a JindoFS \u5df2\u6309\u9884\u8bbe\u7684\u751f\u547d\u5468\u671f\u7b56\u7565\u4e0b\u53d1\u5230\u5bf9\u5e94\u7684 OSS \u6570\u636e\u5757\uff0c\u4e0d\u8868\u793a\u6570\u636e\u5757\u5b9e\u9645\u7684\u5b58\u50a8\u7c7b\u578b\u5df2\u8f6c\u53d8\u5b8c\u6210\u3002 OSS \u7684\u751f\u547d\u5468\u671f\u7ba1\u7406\u8fd0\u884c\u5468\u671f\u8f83\u957f\uff0c\u8bbe\u7f6e\u5f52\u6863\u5b58\u50a8\u7b56\u7565\u7684\u5bf9\u8c61\u53ef\u80fd\u9700\u8981\u6700\u957f24\u5c0f\u65f6\u5de6\u53f3\u624d\u4f1a\u771f\u6b63\u8f6c\u4e3a\u5f52\u6863/\u51b7\u5f52\u6863\u7c7b\u578b\u3002 \u5f52\u6863/\u51b7\u5f52\u6863\u7c7b\u578b\u7684\u6570\u636e\u65e0\u6cd5\u8bbf\u95ee\uff0c\u9700\u8981\u8f6c\u56de\u6807\u51c6\u5b58\u50a8\u65b9\u53ef\u8bbf\u95ee\u3002\u8f6c\u56de\u6807\u51c6\u5b58\u50a8\u53ef\u80fd\u9700\u8981\u4e00\u5b9a\u65f6\u95f4\uff0c\u53ef\u4ee5\u901a\u8fc7 checkStoragePolicy \u547d\u4ee4\u67e5\u8be2\u6267\u884c\u72b6\u6001\u3002 \u53d6\u56de\u5f52\u6863/\u51b7\u5f52\u6863\u6570\u636e\u4f1a\u4ea7\u751f\u4e00\u4e9b\u989d\u5916\u8d39\u7528\uff0c\u5e94\u5f53\u5c3d\u91cf\u907f\u514d\u5c06\u8fd8\u9700\u8bbf\u95ee\u7684\u6570\u636e\u8f6c\u4e3a\u5f52\u6863/\u51b7\u5f52\u6863\u7c7b\u578b\u3002 \u5bf9\u4e8e\u5b58\u50a8\u7b56\u7565\u4e0d\u4e00\u81f4\u7684\u76ee\u5f55\u95f4\u7684\u6587\u4ef6 rename \u64cd\u4f5c\uff0crename \u7684\u540c\u65f6\u4f1a\u81ea\u52a8\u751f\u6210\u76f8\u5e94\u7684\u540e\u53f0\u4efb\u52a1\uff0c\u4f7f rename \u540e\u7684\u6587\u4ef6\u6700\u7ec8\u7b26\u5408\u76ee\u6807\u8def\u5f84\u7684\u5b58\u50a8\u7b56\u7565\u3002 \u5f53\u524d\u7248\u672c\u4e0d\u652f\u6301\u5728\u8bbe\u7f6e\u4e3a\u4f4e\u9891/\u5f52\u6863/\u51b7\u5f52\u6863\u5b58\u50a8\u7b56\u7565\u7684\u76ee\u5f55\u4e0b\u521b\u5efa\u6587\u4ef6\u3002\u82e5\u8981\u5728\u4f4e\u9891/\u5f52\u6863/\u51b7\u5f52\u6863\u7684\u76ee\u5f55\u4e0b\u6dfb\u52a0\u6587\u4ef6\uff0c\u53ef\u4ee5\u5728\u6807\u51c6\u76ee\u5f55\u4e0b\u521b\u5efa\u5e76\u5173\u95ed\u6587\u4ef6\u540e\uff0crename \u5230\u76ee\u6807\u7684\u4f4e\u9891/\u5f52\u6863/\u51b7\u5f52\u6863\u76ee\u5f55\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5206\u5c42\u5b58\u50a8\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindofs_dls_hsm_howto/#oss-hdfs-jindofs","text":"(\u4ece 4.6.2 \u5f00\u59cb\u652f\u6301) \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u662f OSS \u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9HDFS\u63a5\u53e3, \u652f\u6301\u76ee\u5f55\u4ee5\u53ca\u76ee\u5f55\u5c42\u7ea7\uff0c\u901a\u8fc7 Jindo SDK 4.x \u53ef\u4ee5\u517c\u5bb9\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u3002\u5173\u4e8e\u7528\u6237\u5982\u4f55\u521b\u5efa\u548c\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u7684\u57fa\u672c\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 \u94fe\u63a5 \u3002 \u672c\u6587\u4e3b\u8981\u4ecb\u7ecd\u5728 OSS-HDFS \u670d\u52a1\u4e2d\u4f7f\u7528\u5206\u5c42\u5b58\u50a8\u529f\u80fd\u7684\u4e00\u4e9b\u5e38\u89c1\u64cd\u4f5c\u3002 \u4e3a\u4e86\u65b9\u4fbf\u4e0b\u9762\u547d\u4ee4\u7684\u4ecb\u7ecd\uff0c\u6211\u4eec\u5047\u8bbe oss://oss-dfs-test \u8fd9\u4e2aBucket\u5df2\u7ecf\u5f00\u542f OSS-HDFS \u670d\u52a1\u3002\u4e0b\u9762\u6240\u6709\u7684\u4f8b\u5b50\u90fd\u4f1a\u9488\u5bf9\u8fd9\u4e2aBucket\u8fdb\u884c\u64cd\u4f5c\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5206\u5c42\u5b58\u50a8\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindofs_dls_hsm_howto/#_1","text":"\u6570\u636e\u6e56\u5b58\u50a8\u7684\u6570\u636e\u91cf\u5e38\u5e38\u4f1a\u968f\u7740\u65f6\u95f4\u800c\u4e0d\u65ad\u589e\u957f\uff0c\u4f46\u5728\u6570\u636e\u6e56\u5185\u7684\u6570\u636e\u5e76\u975e\u540c\u6837\u9891\u7e41\u8bbf\u95ee\u3002\u5bf9\u4e8e\u8f83\u8001\u7684\u6570\u636e\uff0c\u6bd4\u5982\u8d85\u8fc7\u4e00\u5e74\u7684\u6570\u636e\uff0c\u4e1a\u52a1\u4e0a\u53ef\u80fd\u51e0\u4e4e\u4e0d\u518d\u8bbf\u95ee\uff0c\u4f46\u4e3a\u4fdd\u8bc1\u5408\u89c4\u3001\u5b58\u6863\u7b49\u76ee\u7684\uff0c\u8fd9\u4e9b\u6570\u636e\u4ecd\u7136\u9700\u8981\u5b58\u50a8\u4ee5\u5907\u4e0d\u65f6\u4e4b\u9700\u3002\u56e0\u6b64\uff0c\u5206\u5c42\u5b58\u50a8\u7684\u9700\u6c42\u5c31\u51fa\u73b0\u4e86\uff1a\u5bf9\u4e8e\u8f83\u5c11\u8bbf\u95ee\u7684\u6570\u636e\uff0c\u9009\u62e9\u8f83\u4f4e\u6210\u672c\u7684\u5b58\u50a8\u4ecb\u8d28\uff08\u901a\u5e38\u4f1a\u6709\u66f4\u9ad8\u7684\u8bbf\u95ee\u6210\u672c\uff09\u8fdb\u884c\u5b58\u50a8\uff0c\u53ef\u4ee5\u964d\u4f4e\u603b\u5b58\u50a8\u6210\u672c\u3002 \u6807\u51c6 OSS Bucket \u652f\u6301\u591a\u79cd\u5206\u5c42\u5b58\u50a8\u7c7b\u578b\uff0c\u5305\u62ec\u6807\u51c6\u3001\u4f4e\u9891\u3001\u5f52\u6863\u3001\u51b7\u5f52\u6863\u56db\u79cd\uff0c\u800c JindoFS \u670d\u52a1\u4f7f\u7528\u6807\u51c6 OSS \u4f5c\u4e3a\u6570\u636e\u5757\u7684\u5b58\u50a8\u540e\u7aef\uff0c\u56e0\u6b64 JindoFS \u5206\u5c42\u5b58\u50a8\u529f\u80fd\u5373\u8bbe\u7f6e\u540e\u7aef\u6570\u636e\u5757\u5728\u6807\u51c6 OSS \u4e0a\u6240\u4f7f\u7528\u7684\u5b58\u50a8\u7c7b\u578b\u3002","title":"\u5206\u5c42\u5b58\u50a8\u7b80\u4ecb"},{"location":"backup/jindofs/jindofs_dls_hsm_howto/#_2","text":"\u5728\u4f7f\u7528\u76f8\u5173\u529f\u80fd\u524d\uff0c\u5e94\u786e\u4fdd JindoSDK \u5df2\u7ecf\u6b63\u786e\u914d\u7f6e\uff0c\u53ef\u4ee5\u6b63\u5e38\u8bbf\u95ee JindoFS \u670d\u52a1\u3002JindoFS \u670d\u52a1\u7684\u5206\u5c42\u5b58\u50a8\u529f\u80fd\u4e0e HCFS API \u517c\u5bb9\u3002\u76f8\u5173 API \u5982\u4e0b\u6240\u793a\uff1a public abstract class FileSystem extends Configured implements Closeable { // ...... public void setStoragePolicy(Path src, String policyName) throws IOException; public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException; public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies() throws IOException; } JindoFS \u4e5f\u652f\u6301\u4f7f\u7528\u4e0b\u5217\u547d\u4ee4\u64cd\u4f5c\u5b58\u50a8\u7b56\u7565\uff1a jindo fs -listPolicies jindo fs -setStoragePolicy -path <path> -policy <policy> jindo fs -getStoragePolicy -path <path> getAllStoragePolicies() \u63a5\u53e3\u548c listPolicies \u547d\u4ee4\u8fd4\u56de\u5f53\u524d\u652f\u6301\u7684\u5b58\u50a8\u7b56\u7565\uff0c\u76ee\u524d\u652f\u6301\u7684\u5b58\u50a8\u7b56\u7565\u6709 CLOUD_STD \uff08\u6807\u51c6\u5b58\u50a8\uff09\u3001 CLOUD_IA \uff08\u4f4e\u9891\u5b58\u50a8\uff09\u3001 CLOUD_AR \uff08\u5f52\u6863\uff09\u548c CLOUD_COLD_AR \uff08\u51b7\u5f52\u6863\uff09\u3002 setStoragePolicy() \u63a5\u53e3\u548c setStoragePolicy \u547d\u4ee4\u53ef\u4ee5\u4e3a\u5177\u4f53\u7684\u8def\u5f84\u8bbe\u7f6e\u5b58\u50a8\u7b56\u7565\uff0cJindoFS \u670d\u52a1\u4f1a\u5728\u540e\u53f0\u6839\u636e\u5b58\u50a8\u7b56\u7565\u6539\u53d8\u5b58\u50a8\u5bf9\u8c61\u7684\u5b58\u50a8\u7ea7\u522b\u3002\u4f8b\u5982\uff0c\u8981\u5c06 oss://oss-dfs-test/dir1 \u8bbe\u7f6e\u4e3a\u5f52\u6863\u7c7b\u578b\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a jindo fs -setStoragePolicy -path oss://oss-dfs-test/dir1 -policy CLOUD_AR \u6ce8\u610f\u6587\u4ef6\u6216\u76ee\u5f55\u7684\u9ed8\u8ba4\u5b58\u50a8\u7b56\u7565\u4e3a\u7a7a\uff0c\u663e\u793a\u4e3a UNSPECIFIED \uff0c\u5f53\u5bf9\u8c61\u5b58\u50a8\u7b56\u7565\u4e3a UNSPECIFIED \u65f6\uff0c\u5bf9\u8c61\u5b9e\u9645\u7684\u5b58\u50a8\u7b56\u7565\u5c06\u7531\u6700\u8fd1\u7684\u8bbe\u7f6e\u4e86\u5b58\u50a8\u7b56\u7565\u7684\u7236\u8282\u70b9\u51b3\u5b9a\u3002 \u4f8b\u5982\u5047\u8bbe oss://oss-dfs-test/dir2 \u7684\u5b58\u50a8\u7b56\u7565\u4e3a CLOUD_STD \uff0c\u5219 oss://oss-dfs-test/dir2/subdir2/subsubdir2 \u7684\u5b58\u50a8\u7b56\u7565\u4e5f\u4e3a CLOUD_STD \uff0c \u9664\u975e\u7528\u6237\u5bf9 oss://oss-dfs-test/dir2/subdir2/ \u6216 oss://oss-dfs-test/dir2/subdir2/subsubdir2 \u5355\u72ec\u8bbe\u7f6e\u8fc7\u5b58\u50a8\u7b56\u7565\u3002 \u7528\u6237\u53ef\u4ee5\u901a\u8fc7 getStoragePolicy() \u63a5\u53e3\u548c getStoragePolicy \u547d\u4ee4\u67e5\u8be2\u67d0\u4e00\u8def\u5f84\u6700\u7ec8\u751f\u6548\u7684\u5b58\u50a8\u7b56\u7565\uff0c\u5982\uff1a jindo fs -getStoragePolicy -path oss://oss-dfs-test/dir1/file1 \u5982\u679c\u6700\u7ec8\u751f\u6548\u7684\u5b58\u50a8\u7b56\u7565\u4e3a UNSPECIFIED \uff08\u5373 getStoragePolicy() \u63a5\u53e3\u8fd4\u56de null \uff09\uff0c\u5219\u5bf9\u8c61\u9ed8\u8ba4\u4f7f\u7528\u6807\u51c6\u5b58\u50a8\u3002","title":"\u4f7f\u7528\u5206\u5c42\u5b58\u50a8"},{"location":"backup/jindofs/jindofs_dls_hsm_howto/#_3","text":"\u76ee\u524d JindoFS \u4e0d\u652f\u6301\u5d4c\u5957\u7b56\u7565 , \u4f8b\u5982\u5bf9 oss://oss-dfs-test/warehouse/dwd.db/dt=20220101 \u4f7f\u7528\u4e86 CLOUD_AR \u7b56\u7565, \u518d\u5bf9 oss://oss-dfs-test/warehouse/dwd.db \u7236\u76ee\u5f55\u4f7f\u7528 CLOUD_STD \u7b56\u7565, \u662f\u4e0d\u4f1a\u6267\u884c\u89e3\u5f52\u6863\u64cd\u4f5c\u7684. \u56e0\u6b64\u5efa\u8bae \u53ea\u5bf9\u4e4b\u524d\u8bbe\u7f6e\u8fc7policy\u7684\u76ee\u5f55\u4fee\u6539policy, \u4e0d\u8981\u518d\u4fee\u6539\u5176\u4e0a\u4e0b\u7ea7\u76ee\u5f55\u7684policy.","title":"\u76ee\u5f55\u5d4c\u5957"},{"location":"backup/jindofs/jindofs_dls_hsm_howto/#policy","text":"\u6807\u51c6 \u8f6c \u4f4e\u9891IA jindo fs -setStoragePolicy -path <path> -policy CLOUD_IA \u6807\u51c6 \u8f6c \u5f52\u6863CLOUD_AR jindo fs -setStoragePolicy -path <path> -policy CLOUD_AR \u6807\u51c6 \u8f6c \u51b7\u5f52\u6863CLOUD_COLD_AR jindo fs -setStoragePolicy -path <path> -policy CLOUD_COLD_AR \u4f4e\u9891IA \u8f6c \u5f52\u6863CLOUD_AR jindo fs -setStoragePolicy -path <path> -policy CLOUD_AR \u4f4e\u9891IA \u8f6c \u51b7\u5f52\u6863CLOUD_COLD_AR jindo fs -setStoragePolicy -path <path> -policy CLOUD_COLD_AR \u4f4e\u9891IA \u8f6c \u6807\u51c6 jindo fs -setStoragePolicy -path <path> -policy CLOUD_STD \u5f52\u6863 \u8f6c \u51b7\u5f52\u6863 \u4e0d\u652f\u6301 \u51b7\u5f52\u6863 \u8f6c \u5f52\u6863 \u4e0d\u652f\u6301 \u5f52\u6863 \u8f6c \u6807\u51c6 jindo fs -setStoragePolicy -path <path> -policy CLOUD_STD \u51b7\u5f52\u6863 \u8f6c \u6807\u51c6 jindo fs -setStoragePolicy -path <path> -policy CLOUD_STD","title":"\u5404\u79cd\u573a\u666f\u4e0b Policy \u4e92\u8f6c\u7684\u4f8b\u5b50"},{"location":"backup/jindofs/jindofs_dls_hsm_howto/#oss","text":"\u5728\u4f7f\u7528\u5206\u5c42\u5b58\u50a8\u529f\u80fd\u5c06OSS-HDFS\u7684\u6570\u636e\u8fdb\u884c\u5f52\u6863\u7684\u65f6\u5019\uff0c\u9700\u8981\u4f9d\u8d56\u4e8eOSS\u751f\u547d\u5468\u671f\u529f\u80fd\uff0c\u901a\u8fc7\u4e0b\u9762\u547d\u4ee4 jindo fs -setStoragePolicy -path oss://oss-dfs-test/dir1 -policy CLOUD_AR \u8fdb\u884c\u6570\u636e\u5f52\u6863\u64cd\u4f5c\uff0c\u8be5\u64cd\u4f5c\u53ea\u4f1a\u5c06\u5bf9\u5e94\u76ee\u5f55\u4e0b\u7684\u6570\u636e\u8fdb\u884cTag\u6807\u8bb0, Tag\u5bf9\u5e94\u7684\u952e\u503c\u5bf9\u4e3atransition-storage-class:Archive\uff0c\u9700\u8981\u8bbe\u7f6e\u751f\u547d\u5468\u671f\u8fdb\u884c\u6570\u636e\u7684\u8f6c\u6362\uff0c\u8bbe\u7f6e\u6309\u524d\u7f00\u5339\u914d\u7b56\u7565\u89c4\u5219\uff0c\u524d\u7f00\u4e3a.dlsdata, \u5177\u4f53\u5f52\u6863\u7684\u751f\u547d\u5468\u671f\u7b56\u7565\u8bbe\u7f6e\u5982\u4e0b\uff1a \u4e0a\u8ff0\u751f\u547d\u5468\u671f\u7b56\u7565\u8bbe\u7f6e\u5b8c\u6bd5\u540e\uff0c\u7528\u6237\u9700\u8981\u7b49\u5f85\u6539OSS\u751f\u547d\u5468\u671f\u7b56\u7565\u88ab\u8c03\u5ea6\u6267\u884c\u624d\u80fd\u5b8c\u6210\u6570\u636e\u7684\u5f52\u6863\u3002\u5982\u4f55\u8bbe\u7f6e\u751f\u547d\u5468\u671f\u53ef\u4ee5\u53c2\u8003 \u6587\u6863 \u3002","title":"\u8bbe\u7f6eOSS\u751f\u547d\u5468\u671f"},{"location":"backup/jindofs/jindofs_dls_hsm_howto/#_4","text":"\u5982\u679c\u9700\u8981\u4f7f\u7528\u5f52\u6863/\u51b7\u5f52\u6863\u7684\u5b58\u50a8\u7b56\u7565\uff0c\u9700\u8981\u5728 OSS \u63a7\u5236\u53f0\u4e0a\u914d\u7f6e\u76f8\u5e94\u7684\u751f\u547d\u5468\u671f\u7b56\u7565\uff0c\u5426\u5219\u5b58\u50a8\u7684\u6570\u636e\u5757\u65e0\u6cd5\u771f\u6b63\u8f6c\u4e3a\u5f52\u6863/\u51b7\u5f52\u6863\u7c7b\u578b\u3002 \u5728 JindoFS \u4e0a\u67e5\u8be2\u5230\u7684\u5b58\u50a8\u72b6\u6001\u4ec5\u8868\u793a JindoFS \u5df2\u6309\u9884\u8bbe\u7684\u751f\u547d\u5468\u671f\u7b56\u7565\u4e0b\u53d1\u5230\u5bf9\u5e94\u7684 OSS \u6570\u636e\u5757\uff0c\u4e0d\u8868\u793a\u6570\u636e\u5757\u5b9e\u9645\u7684\u5b58\u50a8\u7c7b\u578b\u5df2\u8f6c\u53d8\u5b8c\u6210\u3002 OSS \u7684\u751f\u547d\u5468\u671f\u7ba1\u7406\u8fd0\u884c\u5468\u671f\u8f83\u957f\uff0c\u8bbe\u7f6e\u5f52\u6863\u5b58\u50a8\u7b56\u7565\u7684\u5bf9\u8c61\u53ef\u80fd\u9700\u8981\u6700\u957f24\u5c0f\u65f6\u5de6\u53f3\u624d\u4f1a\u771f\u6b63\u8f6c\u4e3a\u5f52\u6863/\u51b7\u5f52\u6863\u7c7b\u578b\u3002 \u5f52\u6863/\u51b7\u5f52\u6863\u7c7b\u578b\u7684\u6570\u636e\u65e0\u6cd5\u8bbf\u95ee\uff0c\u9700\u8981\u8f6c\u56de\u6807\u51c6\u5b58\u50a8\u65b9\u53ef\u8bbf\u95ee\u3002\u8f6c\u56de\u6807\u51c6\u5b58\u50a8\u53ef\u80fd\u9700\u8981\u4e00\u5b9a\u65f6\u95f4\uff0c\u53ef\u4ee5\u901a\u8fc7 checkStoragePolicy \u547d\u4ee4\u67e5\u8be2\u6267\u884c\u72b6\u6001\u3002 \u53d6\u56de\u5f52\u6863/\u51b7\u5f52\u6863\u6570\u636e\u4f1a\u4ea7\u751f\u4e00\u4e9b\u989d\u5916\u8d39\u7528\uff0c\u5e94\u5f53\u5c3d\u91cf\u907f\u514d\u5c06\u8fd8\u9700\u8bbf\u95ee\u7684\u6570\u636e\u8f6c\u4e3a\u5f52\u6863/\u51b7\u5f52\u6863\u7c7b\u578b\u3002 \u5bf9\u4e8e\u5b58\u50a8\u7b56\u7565\u4e0d\u4e00\u81f4\u7684\u76ee\u5f55\u95f4\u7684\u6587\u4ef6 rename \u64cd\u4f5c\uff0crename \u7684\u540c\u65f6\u4f1a\u81ea\u52a8\u751f\u6210\u76f8\u5e94\u7684\u540e\u53f0\u4efb\u52a1\uff0c\u4f7f rename \u540e\u7684\u6587\u4ef6\u6700\u7ec8\u7b26\u5408\u76ee\u6807\u8def\u5f84\u7684\u5b58\u50a8\u7b56\u7565\u3002 \u5f53\u524d\u7248\u672c\u4e0d\u652f\u6301\u5728\u8bbe\u7f6e\u4e3a\u4f4e\u9891/\u5f52\u6863/\u51b7\u5f52\u6863\u5b58\u50a8\u7b56\u7565\u7684\u76ee\u5f55\u4e0b\u521b\u5efa\u6587\u4ef6\u3002\u82e5\u8981\u5728\u4f4e\u9891/\u5f52\u6863/\u51b7\u5f52\u6863\u7684\u76ee\u5f55\u4e0b\u6dfb\u52a0\u6587\u4ef6\uff0c\u53ef\u4ee5\u5728\u6807\u51c6\u76ee\u5f55\u4e0b\u521b\u5efa\u5e76\u5173\u95ed\u6587\u4ef6\u540e\uff0crename \u5230\u76ee\u6807\u7684\u4f4e\u9891/\u5f52\u6863/\u51b7\u5f52\u6863\u76ee\u5f55\u3002","title":"\u6ce8\u610f\u4e8b\u9879"},{"location":"backup/jindofs/jindofs_dls_migration_import/","text":"\u534a\u6258\u7ba1 JindoFS \u8fc1\u79fb\u5230\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 (\u4ece 4.5.0 \u5f00\u59cb\u652f\u6301) 0.\u80cc\u666f \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u662f OSS \u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9HDFS\u63a5\u53e3, \u652f\u6301\u76ee\u5f55\u4ee5\u53ca\u76ee\u5f55\u5c42\u7ea7\u3002\u5982\u679c\u8981\u4eceJindoFS\u534a\u6258\u7ba1\u96c6\u7fa4\u8fc1\u79fb\u5230OSS-HDFS\u670d\u52a1\u3002\u9996\u5148\u9700\u8981\u786e\u4fdd\u5728\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u5bf9\u5e94\u7684OSS Bucket\u4e0a\u5f00\u901aOSS-HDFS\u670d\u52a1\uff0c\u540c\u65f6\u9700\u8981\u5728\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u5f00\u542fAuditLog\u3002 \u6b65\u9aa41. \u5168\u91cf\u5bfc\u5165 \u00a7 \u5168\u91cf\u5bfc\u5165\u6a21\u5f0f\u4f1a\u8d1f\u8d23\u5c06\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u4e2d\u7684\u4e00\u4e2a\u76ee\u5f55\u4e2d\u7684\u5143\u6570\u636e\u4e00\u6b21\u6027\u5168\u91cf\u8fc1\u79fb\u5bfc\u5165JindoFS\u670d\u52a1\u4e2d\u7684\u67d0\u4e00\u4e2a\u76ee\u5f55\u4e2d\u3002\u76ee\u524d\u53ea\u652f\u6301\u5bfc\u5165\u5230JindoFS\u670d\u52a1\u4e2d\u7684\u4e00\u7ea7\u5b50\u76ee\u5f55\u3002 \u547d\u4ee4\u683c\u5f0f \u00a7 jindo distjob -migrateImport -srcPath <srcPath> -destPath <desPth> -backendLoc <backendLoc> \u53c2\u6570\u8bf4\u660e\uff1a -srcPath \uff1a\u5f85\u8fc1\u79fb\u7684\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u7684\u6e90\u8def\u5f84\uff08\u5fc5\u586b\uff09 -destPath \uff1a\u5bfc\u5165\u7684JindoFS\u670d\u52a1\u7684\u76ee\u6807\u8def\u5f84\uff08\u5fc5\u586b\uff09 -backendLoc \uff1a \u534a\u6258\u7ba1\u96c6\u7fa4\u7684\u6e90\u6570\u636e\u5757\u5bf9\u5e94\u7684OSS\u8def\u5f84\uff08\u5fc5\u586b\uff09 \u793a\u4f8b \u00a7 \u5047\u5982\u5728\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u4e2d\u6709\u4e00\u4e2a\u76ee\u5f55 jfs://my-cluster/foo\u3002\u8981\u5c06\u8fd9\u4e2a\u76ee\u5f55\u8fc1\u79fb\u5230JindoFS\u670d\u52a1\u5316\u96c6\u7fa4\u4e2d\u7684bar\u76ee\u5f55\u4e2d\u3002\u5047\u8bbeJindoFS\u670d\u52a1\u5316\u7684bucket\u540d\u4e3adlsbucket\uff0c\u5219\u76ee\u6807\u76ee\u5f55\u4e3aoss://dlsbucket/bar\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u5168\u91cf\u5bfc\u5165\u7684\u547d\u4ee4\u4e3a\uff1a jindo distjob -migrateImport -srcPath jfs://my-cluster/foo -destPath oss://dlsbucket/bar/ \u6b65\u9aa42. \u751f\u6210Change Log \u00a7 \u5982\u679c\u9700\u8981\u589e\u91cf\u8fc1\u79fb\u5bfc\u5165\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u5230JindoFS\u670d\u52a1\u3002\u9700\u8981\u5148\u8fd0\u884cjindo\u5de5\u5177\u53ef\u4ee5\u5c06\u534a\u6258\u7ba1JindoFS\u7684AuditLog\u8f6c\u6362\u6210\u5bf9\u5e94\u76ee\u5f55\u7684\u53d8\u66f4\u65e5\u5fd7\uff08Change Log\uff09\u3002\u5177\u4f53\u547d\u4ee4\u683c\u5f0f\u4e3a\uff1a \u547d\u4ee4\u683c\u5f0f \u00a7 jindo distjob -mkchangelog -auditLogDir <auditLogDir> -changeLogDir <changeLogDir> -startTime <startTime> \u53c2\u6570\u8bf4\u660e\uff1a -auditLogDir \uff1aAuditlog\u7684\u8def\u5f84 -changeLogDir \uff1a\u53d8\u66f4\u65e5\u5fd7\u7684\u8f93\u51fa\u8def\u5f84 -startTime \uff1a\u5f00\u59cb\u5904\u7406Auditlog\u7684\u8d77\u59cb\u65f6\u95f4 \u793a\u4f8b \u00a7 \u5047\u5982\u5728\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u4e2dAuditLog\u7684\u8def\u5f84\u4e3aoss://samplebuket/sysinfo/auditlog\u3002\u60f3\u8981\u8f93\u51fa\u7684\u76ee\u5f55\u7684\u53d8\u66f4\u65e5\u5fd7\u5b58\u653e\u5728oss://samplebuket/sysinfo/changelog\u4e0b\u3002\u5e76\u4e14\u53ea\u5904\u7406\u4ece2022\u5e741\u67081\u65e5\u5f00\u59cb\u7684AuditLog\u3002\u5219\u547d\u4ee4\u5982\u4e0b\uff1a jindo distjob -mkchangelog -auditLogDir oss://samplebuket/sysinfo/auditlog -changeLogDir oss://samplebuket/sysinfo/changelog -startTime 2022-01-01-12:00:00 \u6b65\u9aa43. \u589e\u91cf\u5bfc\u5165 \u00a7 \u901a\u8fc7\u6b65\u9aa42\u751f\u6210\u4e86Change Log\u4e4b\u540e\uff0c\u5c31\u80fd\u8fd0\u884c\u589e\u91cf\u5bfc\u5165\u547d\u4ee4\u3002\u6b64\u6a21\u5f0f\u4e0b\uff0c\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u4ea7\u751f\u7684\u5143\u6570\u636e\u7684\u589e\u91cf\u66f4\u65b0\uff0c\u4f1a\u901a\u8fc7\u5bf9\u5e94\u7684AuditLog\u8f6c\u6362\u4e3aChangeLog\u518d\u7ecf\u8fc7\u5904\u7406\u8fc1\u79fb\u81f3JindoFS\u670d\u52a1\u4e2d\u3002\u5177\u4f53\u7684\u547d\u4ee4\u683c\u5f0f\u5982\u4e0b\u3002 \u547d\u4ee4\u683c\u5f0f \u00a7 jindo distjob -migrateImport -srcPath <srcPath> -destPath <desPth> -changeLogDir <auditLogDir> -backendLoc <backendLoc> -update \u53c2\u6570\u8bf4\u660e\uff1a -srcPath \uff1a\u5f85\u8fc1\u79fb\u7684JindoFS\u534a\u6258\u7ba1\u96c6\u7fa4\u7684\u6e90\u8def\u5f84\uff08\u5fc5\u586b\uff09 -destPath \uff1a\u5bfc\u5165\u7684\u670d\u52a1\u5316JindoFS\u96c6\u7fa4\u7684\u76ee\u6807\u8def\u5f84\uff08\u5fc5\u586b\uff09 -changeLogDir \uff1a Change Log\u6240\u5728\u7684\u76ee\u5f55\uff0c\u4e0e\u6b65\u9aa42\u7684\u547d\u4ee4\u53c2\u6570\u542b\u4e49\u76f8\u540c\uff08\u5fc5\u586b\uff09 -backendLoc \uff1a \u534a\u6258\u7ba1\u96c6\u7fa4\u7684\u6e90\u6570\u636e\u5757\u5bf9\u5e94\u7684OSS\u8def\u5f84\uff08\u5fc5\u586b\uff09 -update : \u5f00\u542f\u589e\u91cf\u5bfc\u5165\uff08\u9ed8\u8ba4\u6a21\u5f0f\u4e3a\u5168\u91cf\u5bfc\u5165\uff09 \u793a\u4f8b \u00a7 \u5047\u5982\u5728\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u4e2d\u6709\u4e00\u4e2a\u76ee\u5f55 jfs://my-cluster/foo\u3002\u8981\u5c06\u8fd9\u4e2a\u76ee\u5f55\u8fc1\u79fb\u5230DLS\u4e2d\u7684bar\u76ee\u5f55\u4e2d\u3002\u5047\u8bbeJindoFS\u670d\u52a1\u5316\u96c6\u7fa4\u7684bucket\u540d\u4e3adlsbucket\uff0c\u5219\u76ee\u6807\u76ee\u5f55\u4e3aoss://dlsbucket/bar\u3002\u5e76\u4e14Directory Change Log\u7684\u76ee\u5f55\u4e3aoss://logBucket/logDir/\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u589e\u91cf\u5bfc\u5165\u7684\u547d\u4ee4\u4e3a\uff1a jindo distjob -migrateImport -srcPath jfs://my-cluster/foo -destPath oss://dlsbucket/bar/ -changeLogDir oss://logBucket/logDir/ -update \u6b65\u9aa44\u3002\u591a\u6b21\u589e\u91cf\u5bfc\u5165 \u00a7 \u5982\u679c\u9700\u8981\u591a\u6b21\u589e\u91cf\u5bfc\u5165\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u5230JindoFS\u670d\u52a1\u3002\u53ef\u4ee5\u901a\u8fc7\u4fee\u6539\u6b65\u9aa42\u4e2d\u5f00\u59cb\u5904\u7406Auditlog\u7684\u8d77\u59cb\u65f6\u95f4 -startTime \u53c2\u6570\uff0c\u7136\u540e\u591a\u6b21\u8fd0\u884c\u6b65\u9aa42\u548c\u6b65\u9aa43\u7684\u547d\u4ee4\u5b8c\u6210\u3002","title":"\u534a\u6258\u7ba1 JindoFS \u8fc1\u79fb\u5230\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/jindofs_dls_migration_import/#jindofs-oss-hdfs-jindofs","text":"(\u4ece 4.5.0 \u5f00\u59cb\u652f\u6301)","title":"\u534a\u6258\u7ba1 JindoFS \u8fc1\u79fb\u5230\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/jindofs_dls_migration_import/#0","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u662f OSS \u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9HDFS\u63a5\u53e3, \u652f\u6301\u76ee\u5f55\u4ee5\u53ca\u76ee\u5f55\u5c42\u7ea7\u3002\u5982\u679c\u8981\u4eceJindoFS\u534a\u6258\u7ba1\u96c6\u7fa4\u8fc1\u79fb\u5230OSS-HDFS\u670d\u52a1\u3002\u9996\u5148\u9700\u8981\u786e\u4fdd\u5728\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u5bf9\u5e94\u7684OSS Bucket\u4e0a\u5f00\u901aOSS-HDFS\u670d\u52a1\uff0c\u540c\u65f6\u9700\u8981\u5728\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u5f00\u542fAuditLog\u3002","title":"0.\u80cc\u666f"},{"location":"backup/jindofs/jindofs_dls_migration_import/#1","text":"\u5168\u91cf\u5bfc\u5165\u6a21\u5f0f\u4f1a\u8d1f\u8d23\u5c06\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u4e2d\u7684\u4e00\u4e2a\u76ee\u5f55\u4e2d\u7684\u5143\u6570\u636e\u4e00\u6b21\u6027\u5168\u91cf\u8fc1\u79fb\u5bfc\u5165JindoFS\u670d\u52a1\u4e2d\u7684\u67d0\u4e00\u4e2a\u76ee\u5f55\u4e2d\u3002\u76ee\u524d\u53ea\u652f\u6301\u5bfc\u5165\u5230JindoFS\u670d\u52a1\u4e2d\u7684\u4e00\u7ea7\u5b50\u76ee\u5f55\u3002","title":"\u6b65\u9aa41. \u5168\u91cf\u5bfc\u5165"},{"location":"backup/jindofs/jindofs_dls_migration_import/#_1","text":"jindo distjob -migrateImport -srcPath <srcPath> -destPath <desPth> -backendLoc <backendLoc> \u53c2\u6570\u8bf4\u660e\uff1a -srcPath \uff1a\u5f85\u8fc1\u79fb\u7684\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u7684\u6e90\u8def\u5f84\uff08\u5fc5\u586b\uff09 -destPath \uff1a\u5bfc\u5165\u7684JindoFS\u670d\u52a1\u7684\u76ee\u6807\u8def\u5f84\uff08\u5fc5\u586b\uff09 -backendLoc \uff1a \u534a\u6258\u7ba1\u96c6\u7fa4\u7684\u6e90\u6570\u636e\u5757\u5bf9\u5e94\u7684OSS\u8def\u5f84\uff08\u5fc5\u586b\uff09","title":"\u547d\u4ee4\u683c\u5f0f"},{"location":"backup/jindofs/jindofs_dls_migration_import/#_2","text":"\u5047\u5982\u5728\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u4e2d\u6709\u4e00\u4e2a\u76ee\u5f55 jfs://my-cluster/foo\u3002\u8981\u5c06\u8fd9\u4e2a\u76ee\u5f55\u8fc1\u79fb\u5230JindoFS\u670d\u52a1\u5316\u96c6\u7fa4\u4e2d\u7684bar\u76ee\u5f55\u4e2d\u3002\u5047\u8bbeJindoFS\u670d\u52a1\u5316\u7684bucket\u540d\u4e3adlsbucket\uff0c\u5219\u76ee\u6807\u76ee\u5f55\u4e3aoss://dlsbucket/bar\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u5168\u91cf\u5bfc\u5165\u7684\u547d\u4ee4\u4e3a\uff1a jindo distjob -migrateImport -srcPath jfs://my-cluster/foo -destPath oss://dlsbucket/bar/","title":"\u793a\u4f8b"},{"location":"backup/jindofs/jindofs_dls_migration_import/#2-change-log","text":"\u5982\u679c\u9700\u8981\u589e\u91cf\u8fc1\u79fb\u5bfc\u5165\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u5230JindoFS\u670d\u52a1\u3002\u9700\u8981\u5148\u8fd0\u884cjindo\u5de5\u5177\u53ef\u4ee5\u5c06\u534a\u6258\u7ba1JindoFS\u7684AuditLog\u8f6c\u6362\u6210\u5bf9\u5e94\u76ee\u5f55\u7684\u53d8\u66f4\u65e5\u5fd7\uff08Change Log\uff09\u3002\u5177\u4f53\u547d\u4ee4\u683c\u5f0f\u4e3a\uff1a","title":"\u6b65\u9aa42. \u751f\u6210Change Log"},{"location":"backup/jindofs/jindofs_dls_migration_import/#_3","text":"jindo distjob -mkchangelog -auditLogDir <auditLogDir> -changeLogDir <changeLogDir> -startTime <startTime> \u53c2\u6570\u8bf4\u660e\uff1a -auditLogDir \uff1aAuditlog\u7684\u8def\u5f84 -changeLogDir \uff1a\u53d8\u66f4\u65e5\u5fd7\u7684\u8f93\u51fa\u8def\u5f84 -startTime \uff1a\u5f00\u59cb\u5904\u7406Auditlog\u7684\u8d77\u59cb\u65f6\u95f4","title":"\u547d\u4ee4\u683c\u5f0f"},{"location":"backup/jindofs/jindofs_dls_migration_import/#_4","text":"\u5047\u5982\u5728\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u4e2dAuditLog\u7684\u8def\u5f84\u4e3aoss://samplebuket/sysinfo/auditlog\u3002\u60f3\u8981\u8f93\u51fa\u7684\u76ee\u5f55\u7684\u53d8\u66f4\u65e5\u5fd7\u5b58\u653e\u5728oss://samplebuket/sysinfo/changelog\u4e0b\u3002\u5e76\u4e14\u53ea\u5904\u7406\u4ece2022\u5e741\u67081\u65e5\u5f00\u59cb\u7684AuditLog\u3002\u5219\u547d\u4ee4\u5982\u4e0b\uff1a jindo distjob -mkchangelog -auditLogDir oss://samplebuket/sysinfo/auditlog -changeLogDir oss://samplebuket/sysinfo/changelog -startTime 2022-01-01-12:00:00","title":"\u793a\u4f8b"},{"location":"backup/jindofs/jindofs_dls_migration_import/#3","text":"\u901a\u8fc7\u6b65\u9aa42\u751f\u6210\u4e86Change Log\u4e4b\u540e\uff0c\u5c31\u80fd\u8fd0\u884c\u589e\u91cf\u5bfc\u5165\u547d\u4ee4\u3002\u6b64\u6a21\u5f0f\u4e0b\uff0c\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u4ea7\u751f\u7684\u5143\u6570\u636e\u7684\u589e\u91cf\u66f4\u65b0\uff0c\u4f1a\u901a\u8fc7\u5bf9\u5e94\u7684AuditLog\u8f6c\u6362\u4e3aChangeLog\u518d\u7ecf\u8fc7\u5904\u7406\u8fc1\u79fb\u81f3JindoFS\u670d\u52a1\u4e2d\u3002\u5177\u4f53\u7684\u547d\u4ee4\u683c\u5f0f\u5982\u4e0b\u3002","title":"\u6b65\u9aa43. \u589e\u91cf\u5bfc\u5165"},{"location":"backup/jindofs/jindofs_dls_migration_import/#_5","text":"jindo distjob -migrateImport -srcPath <srcPath> -destPath <desPth> -changeLogDir <auditLogDir> -backendLoc <backendLoc> -update \u53c2\u6570\u8bf4\u660e\uff1a -srcPath \uff1a\u5f85\u8fc1\u79fb\u7684JindoFS\u534a\u6258\u7ba1\u96c6\u7fa4\u7684\u6e90\u8def\u5f84\uff08\u5fc5\u586b\uff09 -destPath \uff1a\u5bfc\u5165\u7684\u670d\u52a1\u5316JindoFS\u96c6\u7fa4\u7684\u76ee\u6807\u8def\u5f84\uff08\u5fc5\u586b\uff09 -changeLogDir \uff1a Change Log\u6240\u5728\u7684\u76ee\u5f55\uff0c\u4e0e\u6b65\u9aa42\u7684\u547d\u4ee4\u53c2\u6570\u542b\u4e49\u76f8\u540c\uff08\u5fc5\u586b\uff09 -backendLoc \uff1a \u534a\u6258\u7ba1\u96c6\u7fa4\u7684\u6e90\u6570\u636e\u5757\u5bf9\u5e94\u7684OSS\u8def\u5f84\uff08\u5fc5\u586b\uff09 -update : \u5f00\u542f\u589e\u91cf\u5bfc\u5165\uff08\u9ed8\u8ba4\u6a21\u5f0f\u4e3a\u5168\u91cf\u5bfc\u5165\uff09","title":"\u547d\u4ee4\u683c\u5f0f"},{"location":"backup/jindofs/jindofs_dls_migration_import/#_6","text":"\u5047\u5982\u5728\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u4e2d\u6709\u4e00\u4e2a\u76ee\u5f55 jfs://my-cluster/foo\u3002\u8981\u5c06\u8fd9\u4e2a\u76ee\u5f55\u8fc1\u79fb\u5230DLS\u4e2d\u7684bar\u76ee\u5f55\u4e2d\u3002\u5047\u8bbeJindoFS\u670d\u52a1\u5316\u96c6\u7fa4\u7684bucket\u540d\u4e3adlsbucket\uff0c\u5219\u76ee\u6807\u76ee\u5f55\u4e3aoss://dlsbucket/bar\u3002\u5e76\u4e14Directory Change Log\u7684\u76ee\u5f55\u4e3aoss://logBucket/logDir/\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u589e\u91cf\u5bfc\u5165\u7684\u547d\u4ee4\u4e3a\uff1a jindo distjob -migrateImport -srcPath jfs://my-cluster/foo -destPath oss://dlsbucket/bar/ -changeLogDir oss://logBucket/logDir/ -update","title":"\u793a\u4f8b"},{"location":"backup/jindofs/jindofs_dls_migration_import/#4","text":"\u5982\u679c\u9700\u8981\u591a\u6b21\u589e\u91cf\u5bfc\u5165\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u5230JindoFS\u670d\u52a1\u3002\u53ef\u4ee5\u901a\u8fc7\u4fee\u6539\u6b65\u9aa42\u4e2d\u5f00\u59cb\u5904\u7406Auditlog\u7684\u8d77\u59cb\u65f6\u95f4 -startTime \u53c2\u6570\uff0c\u7136\u540e\u591a\u6b21\u8fd0\u884c\u6b65\u9aa42\u548c\u6b65\u9aa43\u7684\u547d\u4ee4\u5b8c\u6210\u3002","title":"\u6b65\u9aa44\u3002\u591a\u6b21\u589e\u91cf\u5bfc\u5165"},{"location":"backup/jindofs/jindofs_dls_root_policy/","text":"\u4f7f\u7528 RootPolicy \u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 (\u4ece 4.6.0 \u5f00\u59cb\u652f\u6301) \u80cc\u666f \u00a7 JindoFS \u652f\u6301 RootPolicy \u89c4\u5219\uff0c\u901a\u8fc7 RootPolicy \u53ef\u4ee5\u4e3a OSS-HDFS\uff08JindoFS \u670d\u52a1\uff09\u8bbe\u7f6e\u81ea\u5b9a\u4e49\u524d\u7f00\uff0c\u8fd9\u6837\u7528\u6237\u539f\u6709\u7684\u8bbf\u95ee hdfs:// \u524d\u7f00\u7684\u4f5c\u4e1a\uff0c\u53ef\u4ee5\u4e0d\u7ecf\u4fee\u6539\u76f4\u63a5\u8fd0\u884c\u5728 JindoFS \u670d\u52a1\u4e0a\u3002 1\u3001\u8bbe\u7f6e RootPolicy \u00a7 \u6211\u4eec\u63d0\u4f9b\u4e86\u4e00\u7ec4 Shell \u547d\u4ee4\uff0c\u5e2e\u52a9\u7528\u6237\u5411\u7279\u5b9a\u7684bucket\u6ce8\u518c\u81ea\u5b9a\u4e49\u5730\u5740\u3002\u901a\u8fc7 SetRootPolicy\u547d\u4ee4 \uff0c\u53ef\u4ee5\u4e3a\u7279\u5b9a bucket \u6ce8\u518c\u81ea\u5b9a\u4e49\u524d\u7f00\u7684\u8bbf\u95ee\u5730\u5740\u3002\u5982\uff1a jindo admin -setRootPolicy oss://<bucket_name>.<dls_endpoint>/ hdfs://<your_ns_name>/ : OSS-HDFS \u670d\u52a1 bucket \u7684\u540d\u5b57\uff0c\u5f53\u524d\u7248\u672c\u4ec5\u652f\u6301\u6839\u76ee\u5f55\u3002 : OSS-HDFS \u670d\u52a1\u7684 endpoint\uff0c\u5982\u676d\u5dderegion\u4e3a cn-hangzhou.oss-dls.aliyuncs.com \u3002\u5982\u679c Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u6709\u5982\u4e0b\u914d\u7f6e\u9879 <configuration> <property> <name>fs.oss.endpoint</name> <value><dls_endpoint></value> </property> \u6216 <property> <name>fs.oss.bucket.<bucket_name>.endpoint</name> <value><dls_endpoint></value> </property> </configuration> \u4e0a\u9762\u4f8b\u5b50\u4e2d\u7684 oss://<bucket_name>.<dls_endpoint>/ \u53ef\u4ee5\u5199\u6210 oss://<bucket_name>/ \u7684\u5f62\u5f0f\uff0c\u4e0b\u6587\u4e2d\u9700\u8981 endpoint \u7684\u5730\u65b9\u90fd\u53ef\u7701\u7565\u4e0d\u586b\u3002 * : \u81ea\u5b9a\u4e49\u8bbf\u95ee hdfs \u670d\u52a1\u7684 nsname\uff0c\u4e3a\u4efb\u610f\u975e\u7a7a\u5b57\u7b26\u4e32\uff0c\u5982 test \uff0c\u5f53\u524d\u7248\u672c\u4ec5\u652f\u6301\u6839\u76ee\u5f55\u3002 2\u3001\u914d\u7f6e access policy \u53d1\u73b0\u5730\u5740\u548c scheme \u5b9e\u73b0\u7c7b \u00a7 \u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u6dfb\u52a0\u5982\u4e0b\u914d\u7f6e\u9879 <configuration> <property> <name>fs.accessPolicies.discovery</name> <value>oss://<bucket_name>.<dls_endpoint>/</value> </property> <property> <name>fs.AbstractFileSystem.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.v28.HDFS</value> </property> <property> <name>fs.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.v28.JindoDistributedFileSystem</value> </property> </configuration> \u5982\u679c\u662f\u591a\u4e2abucket\uff0c\u7528 , \u5206\u9694\u3002 3\u3001\u9a8c\u8bc1\u548c\u4f7f\u7528 \u00a7 \u914d\u7f6e\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u5bf9\u914d\u7f6e\u8fdb\u884c\u7b80\u5355\u9a8c\u8bc1\u3002 hadoop fs -ls hdfs://<your_ns_name>/ \u9a8c\u8bc1\u901a\u8fc7\u540e\uff0c\u91cd\u542f\u76f8\u5173\u670d\u52a1\uff0c\u5982 Hive/Spark \u7b49\uff0c\u5373\u53ef\u4f7f\u7528\u81ea\u5b9a\u4e49\u524d\u7f00\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u3002 4\u3001\u5176\u4ed6\u529f\u80fd \u00a7 4.1\u3001\u5220\u9664 RootPolicy \u89c4\u5219 \u00a7 \u901a\u8fc7 UnsetRootPolicy\u547d\u4ee4 \u53ef\u4ee5\u5220\u9664\u76f8\u5e94\u5730\u5740\u3002 jindo admin -unsetRootPolicy oss://<bucket_name>.<dls_endpoint>/ hdfs://<your_ns_name>/ 4.2\u3001\u5217\u4e3e\u6240\u6709 RootPolicy \u89c4\u5219 \u00a7 \u4f7f\u7528 ListAccessPolicies\u547d\u4ee4 \uff0c\u53ef\u4ee5\u5217\u51fa\u7279\u5b9a bucket \u5f53\u524d\u6ce8\u518c\u7684\u6240\u6709\u524d\u7f00\u5730\u5740\u3002 jindo admin -listAccessPolicies oss://<bucket_name>.<dls_endpoint>/","title":"\u4f7f\u7528 RootPolicy \u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/jindofs_dls_root_policy/#rootpolicy-oss-hdfs-jindofs","text":"(\u4ece 4.6.0 \u5f00\u59cb\u652f\u6301)","title":"\u4f7f\u7528 RootPolicy \u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/jindofs_dls_root_policy/#_1","text":"JindoFS \u652f\u6301 RootPolicy \u89c4\u5219\uff0c\u901a\u8fc7 RootPolicy \u53ef\u4ee5\u4e3a OSS-HDFS\uff08JindoFS \u670d\u52a1\uff09\u8bbe\u7f6e\u81ea\u5b9a\u4e49\u524d\u7f00\uff0c\u8fd9\u6837\u7528\u6237\u539f\u6709\u7684\u8bbf\u95ee hdfs:// \u524d\u7f00\u7684\u4f5c\u4e1a\uff0c\u53ef\u4ee5\u4e0d\u7ecf\u4fee\u6539\u76f4\u63a5\u8fd0\u884c\u5728 JindoFS \u670d\u52a1\u4e0a\u3002","title":"\u80cc\u666f"},{"location":"backup/jindofs/jindofs_dls_root_policy/#1-rootpolicy","text":"\u6211\u4eec\u63d0\u4f9b\u4e86\u4e00\u7ec4 Shell \u547d\u4ee4\uff0c\u5e2e\u52a9\u7528\u6237\u5411\u7279\u5b9a\u7684bucket\u6ce8\u518c\u81ea\u5b9a\u4e49\u5730\u5740\u3002\u901a\u8fc7 SetRootPolicy\u547d\u4ee4 \uff0c\u53ef\u4ee5\u4e3a\u7279\u5b9a bucket \u6ce8\u518c\u81ea\u5b9a\u4e49\u524d\u7f00\u7684\u8bbf\u95ee\u5730\u5740\u3002\u5982\uff1a jindo admin -setRootPolicy oss://<bucket_name>.<dls_endpoint>/ hdfs://<your_ns_name>/ : OSS-HDFS \u670d\u52a1 bucket \u7684\u540d\u5b57\uff0c\u5f53\u524d\u7248\u672c\u4ec5\u652f\u6301\u6839\u76ee\u5f55\u3002 : OSS-HDFS \u670d\u52a1\u7684 endpoint\uff0c\u5982\u676d\u5dderegion\u4e3a cn-hangzhou.oss-dls.aliyuncs.com \u3002\u5982\u679c Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u6709\u5982\u4e0b\u914d\u7f6e\u9879 <configuration> <property> <name>fs.oss.endpoint</name> <value><dls_endpoint></value> </property> \u6216 <property> <name>fs.oss.bucket.<bucket_name>.endpoint</name> <value><dls_endpoint></value> </property> </configuration> \u4e0a\u9762\u4f8b\u5b50\u4e2d\u7684 oss://<bucket_name>.<dls_endpoint>/ \u53ef\u4ee5\u5199\u6210 oss://<bucket_name>/ \u7684\u5f62\u5f0f\uff0c\u4e0b\u6587\u4e2d\u9700\u8981 endpoint \u7684\u5730\u65b9\u90fd\u53ef\u7701\u7565\u4e0d\u586b\u3002 * : \u81ea\u5b9a\u4e49\u8bbf\u95ee hdfs \u670d\u52a1\u7684 nsname\uff0c\u4e3a\u4efb\u610f\u975e\u7a7a\u5b57\u7b26\u4e32\uff0c\u5982 test \uff0c\u5f53\u524d\u7248\u672c\u4ec5\u652f\u6301\u6839\u76ee\u5f55\u3002","title":"1\u3001\u8bbe\u7f6e RootPolicy"},{"location":"backup/jindofs/jindofs_dls_root_policy/#2-access-policy-scheme","text":"\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u6dfb\u52a0\u5982\u4e0b\u914d\u7f6e\u9879 <configuration> <property> <name>fs.accessPolicies.discovery</name> <value>oss://<bucket_name>.<dls_endpoint>/</value> </property> <property> <name>fs.AbstractFileSystem.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.v28.HDFS</value> </property> <property> <name>fs.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.v28.JindoDistributedFileSystem</value> </property> </configuration> \u5982\u679c\u662f\u591a\u4e2abucket\uff0c\u7528 , \u5206\u9694\u3002","title":"2\u3001\u914d\u7f6e access policy \u53d1\u73b0\u5730\u5740\u548c scheme \u5b9e\u73b0\u7c7b"},{"location":"backup/jindofs/jindofs_dls_root_policy/#3","text":"\u914d\u7f6e\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u5bf9\u914d\u7f6e\u8fdb\u884c\u7b80\u5355\u9a8c\u8bc1\u3002 hadoop fs -ls hdfs://<your_ns_name>/ \u9a8c\u8bc1\u901a\u8fc7\u540e\uff0c\u91cd\u542f\u76f8\u5173\u670d\u52a1\uff0c\u5982 Hive/Spark \u7b49\uff0c\u5373\u53ef\u4f7f\u7528\u81ea\u5b9a\u4e49\u524d\u7f00\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u3002","title":"3\u3001\u9a8c\u8bc1\u548c\u4f7f\u7528"},{"location":"backup/jindofs/jindofs_dls_root_policy/#4","text":"","title":"4\u3001\u5176\u4ed6\u529f\u80fd"},{"location":"backup/jindofs/jindofs_dls_root_policy/#41-rootpolicy","text":"\u901a\u8fc7 UnsetRootPolicy\u547d\u4ee4 \u53ef\u4ee5\u5220\u9664\u76f8\u5e94\u5730\u5740\u3002 jindo admin -unsetRootPolicy oss://<bucket_name>.<dls_endpoint>/ hdfs://<your_ns_name>/","title":"4.1\u3001\u5220\u9664 RootPolicy \u89c4\u5219"},{"location":"backup/jindofs/jindofs_dls_root_policy/#42-rootpolicy","text":"\u4f7f\u7528 ListAccessPolicies\u547d\u4ee4 \uff0c\u53ef\u4ee5\u5217\u51fa\u7279\u5b9a bucket \u5f53\u524d\u6ce8\u518c\u7684\u6240\u6709\u524d\u7f00\u5730\u5740\u3002 jindo admin -listAccessPolicies oss://<bucket_name>.<dls_endpoint>/","title":"4.2\u3001\u5217\u4e3e\u6240\u6709 RootPolicy \u89c4\u5219"},{"location":"backup/jindofs/jindofs_dls_shell_howto/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Shell \u547d\u4ee4\u4f7f\u7528\u8bf4\u660e \u00a7 (\u4ece 4.5.0 \u5f00\u59cb\u652f\u6301) \u8bbe\u7f6e Hadoop Proxy User \u547d\u4ee4 \u00a7 AddProxyUser \u547d\u4ee4 \u00a7 AddProxyUser \u547d\u4ee4\u53ef\u4ee5\u6dfb\u52a0\u4ee3\u7406\u7528\u6237\u3002 jindo admin -addProxyUser [-dlsUri <uri>] [-proxyUser <proxyUser>] [-users <user1,user2...>]|[-groups <group1,group2...>] [-hosts <host1,host2...>] \u5728\u540c\u4e00\u6761\u547d\u4ee4\u4e2d\uff0c\u53c2\u6570 users \u548c groups \u4e0d\u53ef\u540c\u65f6\u4f7f\u7528\uff0c\u4efb\u9009\u5176\u4e00\u3002 \u4f8b\uff1a jindo admin -addProxyUser -dlsUri oss://jindosdk-dls-unit-test/ -proxyUser hive -groups group1,group2 -hosts host1,host2 \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u7528\u6237 hive \u53ef\u4ee5\u4ee3\u7406\u5c5e\u4e8e group1 \u6216 group2 \u4e14\u4ece host1 \u6216 host2 \u53d1\u9001\u8bf7\u6c42\u7684\u7528\u6237\u3002 DeleteProxyUser \u547d\u4ee4 \u00a7 DeleteProxyUser \u547d\u4ee4\u53ef\u4ee5\u5220\u9664\u4ee3\u7406\u7528\u6237\u3002 jindo admin -deleteProxyUser [-dlsUri <uri>] [-proxyUser <proxyUser>] \u4f8b\uff1a jindo admin -deleteProxyUser -dlsUri oss://jindosdk-dls-unit-test/ -proxyUser hive \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u7528\u6237 hive \u4e0d\u53ef\u4ee5\u4ee3\u7406\u4efb\u4f55\u7528\u6237\u3002 ListProxyUsers \u547d\u4ee4 \u00a7 ListProxyUsers \u53ef\u4ee5\u67e5\u770b\u6240\u6709\u7684\u4ee3\u7406\u7528\u6237\u548c\u4ee3\u7406\u4fe1\u606f\u3002 jindo admin -listProxyUsers [-dlsUri <uri>] [-maxKeys <maxKeys>] [-marker <marker>] maxKeys \u6307\u5b9a\u67e5\u770b\u7684\u4ee3\u7406\u7528\u6237\u4e2a\u6570\uff0cmarker \u7528\u6765\u7b5b\u9009\u540d\u79f0\u4e2d\u542b\u6709\u7279\u6b8a\u6807\u8bb0\u7684\u4ee3\u7406\u7528\u6237\uff0c\u8fd9\u4e24\u9879\u53c2\u6570\u5747\u53ef\u7701\u7565\u3002 \u4f8b\uff1a jindo admin -listProxyUsers -dlsUri oss://jindosdk-dls-unit-test/ -maxKeys 10 -marker hive \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u5c06\u4f18\u5148\u663e\u793a oss://jindosdk-dls-unit-test/ \u4e0b\u540d\u79f0\u4e2d\u5e26 hive \u6807\u8bb0\u768410\u4e2a\u4ee3\u7406\u7528\u6237\u4fe1\u606f\u3002 \u8bbe\u7f6e User Groups Mapping \u547d\u4ee4 \u00a7 AddUserGroupsMapping \u547d\u4ee4 \u00a7 AddUserGroupsMapping \u547d\u4ee4\u6620\u5c04\u7528\u6237\u548c\u7ec4\u7684\u5173\u7cfb\u3002 jindo admin -addUserGroupsMapping [-dlsUri <uri>] [-user <user>] [-groups <group1,group2...>] \u4f8b\uff1a jindo admin -addUserGroupsMapping -dlsUri oss://jindosdk-dls-unit-test/ -user user1 -groups group1,group2 \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u7528\u6237 user1 \u5c06\u5c5e\u4e8e group1 \u6216 group2\u3002 DeleteUserGroupsMapping \u547d\u4ee4 \u00a7 DeleteUserGroupsMapping \u547d\u4ee4\u53ef\u4ee5\u5220\u9664\u7528\u6237\u548c\u7ec4\u7684\u6620\u5c04\u5173\u7cfb\u3002 jindo admin -deleteUserGroupsMapping [-dlsUri <uri>] [-user <user>] \u4f8b\uff1a jindo admin -deleteUserGroupsMapping -dlsUri oss://jindosdk-dls-unit-test/ -user user1 \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u5c06\u5220\u9664 user1 \u4e0e\u5bf9\u5e94\u7ec4\u7684\u6620\u5c04\u5173\u7cfb\u3002 ListUserGroupsMappings \u547d\u4ee4 \u00a7 ListUserGroupsMappings \u53ef\u4ee5\u67e5\u770b\u6240\u6709\u7684\u7528\u6237\u7684\u7ec4\u3002 jindo admin -listUserGroupsMappings [-dlsUri <dlsUri>] [-maxKeys <maxKeys>] [-marker <marker>] maxKeys \u6307\u5b9a\u67e5\u770b\u7684\u7528\u6237\u4e2a\u6570\uff0cmarker \u7528\u6765\u7b5b\u9009\u540d\u79f0\u4e2d\u542b\u6709\u7279\u6b8a\u6807\u8bb0\u7684\u7528\u6237\uff0c\u8fd9\u4e24\u9879\u53c2\u6570\u5747\u53ef\u7701\u7565\u3002 \u4f8b\uff1a jindo admin -listUserGroupsMappings -dlsUri oss://jindosdk-dls-unit-test/ -maxKeys 10 -marker user1 \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u5c06\u4f18\u5148\u663e\u793a oss://jindosdk-dls-unit-test/ \u4e0b\u540d\u79f0\u4e2d\u5e26 user1 \u6807\u8bb0\u768410\u4e2a\u7528\u6237\u7684\u7ec4\u4fe1\u606f\u3002 \u7ba1\u7406 Snapshot \u547d\u4ee4 \u00a7 AllowSnapshot \u547d\u4ee4 \u00a7 AllowSnapshot \u547d\u4ee4\u5141\u8bb8\u521b\u5efa\u76ee\u5f55\u5feb\u7167\u3002 jindo admin -allowSnapshot [-dlsUri <dlsSnapshotDir>] \u4f8b\uff1a jindo admin -allowSnapshot -dlsUri oss://jindosdk-dls-unit-test/testSnapshot/snapshot_allow \u6267\u884c\u4e0a\u9762\u7684\u547d\u4ee4\u540e\uff0c\u5c06\u5141\u8bb8 oss://jindosdk-dls-unit-test/testSnapshot/snapshot_allow \u76ee\u5f55\u521b\u5efa\u5feb\u7167\u3002 DisallowSnapshot \u547d\u4ee4 \u00a7 DisallowSnapshot \u547d\u4ee4\u53ef\u4ee5\u7981\u6b62\u521b\u5efa\u76ee\u5f55\u5feb\u7167\u3002 jindo admin -disallowSnapshot [-dlsUri <dlsSnapshotDir>] \u4f8b\uff1a jindo admin -disallowSnapshot -dlsUri oss://jindosdk-dls-unit-test/testSnapshot/snapshot_allow \u6267\u884c\u4e0a\u9762\u7684\u547d\u4ee4\u540e\uff0c\u5c06\u4e0d\u5141\u8bb8 oss://jindosdk-dls-unit-test/testSnapshot/snapshot_allow \u76ee\u5f55\u521b\u5efa\u5feb\u7167\u3002 SnapshotDiff \u547d\u4ee4 \u00a7 SnapshotDiff \u547d\u4ee4\u7528\u6765\u6bd4\u8f83\u540c\u4e00\u76ee\u5f55\u4e24\u4efd\u5feb\u7167\u7684\u4e0d\u540c\u3002 jindo admin -snapshotDiff [-dlsUri <dlsSnapshotDir>] [-fromSnapshot <fromSnapshot>] [-toSnapshot <toSnapshot>] \u4f8b\uff1a jindo admin -snapshotDiff -dlsUri oss://jindosdk-dls-unit-test/testSnapshotDir/ -fromSnapshot S1 -toSnapshot S2 S1 \u548c S1 \u662f\u4e8b\u5148\u901a\u8fc7 HDFS \u7684 Shell \u547d\u4ee4\u884c\u5de5\u5177\u4e3a\u6d4b\u8bd5\u76ee\u5f55\u521b\u5efa\u7684\u4e24\u4efd\u5feb\u7167\u3002 \u901a\u8fc7 AccessPolicy \u547d\u4ee4\u5b9e\u73b0\u8def\u5f84\u6539\u5199 \u00a7 SetRootPolicy \u547d\u4ee4 \u00a7 SetRootPolicy \u547d\u4ee4\u5141\u8bb8\u4e3a bucket \u8bbe\u7f6e\u4efb\u610f\u524d\u7f00\u7684\u8bbf\u95ee\u8def\u5f84\u3002 jindo admin -setRootPolicy [<dlsRootPath>] [<accessRootPath>] \u4f8b\uff1a jindo admin -setRootPolicy oss://jindosdk-dls-unit-test/ hdfs://myufs/ \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\u540e\uff0c\u652f\u6301\u4f7f\u7528 hdfs://myufs/ \u8bbf\u95ee oss://jindosdk-dls-unit-test/\u3002 UnsetRootPolicy \u547d\u4ee4 \u00a7 UnsetRootPolicy \u547d\u4ee4\u4e3a bucket \u53d6\u6d88\u8bbf\u95ee\u8def\u5f84\u3002 jindo admin -unsetRootPolicy [<dlsRootPath>] [<accessRootPath>] \u4f8b\uff1a jindo admin -unsetRootPolicy oss://jindosdk-dls-unit-test/ hdfs://myufs/ \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\u540e\uff0c\u4e0d\u518d\u652f\u6301\u7528 hdfs://myufs/ \u8bbf\u95ee oss://jindosdk-dls-unit-test/\u3002 ListAccessPolicies \u547d\u4ee4 \u00a7 ListAccessPolicies \u53ef\u67e5\u770b\u5f53\u524d bucket \u652f\u6301\u7684\u6240\u6709\u8bbf\u95ee\u8def\u5f84\u3002 jindo admin -listAccessPolicies [<dlsRootPath>] \u4f8b\uff1a jindo admin -listAccessPolicies oss://jindosdk-dls-unit-test/ \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u67e5\u770b oss://jindosdk-dls-unit-test/ \u7684\u6240\u6709\u8bbf\u95ee\u8def\u5f84\u3002 \u5176\u4ed6 \u00a7 DumpFile \u547d\u4ee4 \u00a7 DumpFile \u547d\u4ee4\u53ef\u4ee5\u67e5\u770b\u6587\u4ef6\u7684\u8be6\u7ec6\u5b58\u50a8\u4fe1\u606f\u3002 jindo admin -dumpFile [-dlsUri <dlsUri>] \u4f8b\uff1a jindo admin -dumpFile -dlsUri oss://jindosdk-dls-unit-test/testDumpFile/dumpfile_simple_file \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\u540e\uff0c\u5c06\u663e\u793a\u7c7b\u4f3c\u5982\u4e0b\u7ed3\u679c\uff1a =============Dump result============= File /testDumpFile/dumpfile_simple_file length: 11 blockSize: 33554432 modification time: 1657626266711 access time: 1657626266481 permission: 420 owner: root group: supergroup file id: 4180702376518228901 =============Block 0 block id: 8792388394945610760 offset: 0 numBytes: 11 gs: 1032 type: CLOUD loc: .dlsdata underConstruction: 0 isLastBlockComplete: 1","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Shell \u547d\u4ee4\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#oss-hdfs-jindofs-shell","text":"(\u4ece 4.5.0 \u5f00\u59cb\u652f\u6301)","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Shell \u547d\u4ee4\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#hadoop-proxy-user","text":"","title":"\u8bbe\u7f6e Hadoop Proxy User \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#addproxyuser","text":"AddProxyUser \u547d\u4ee4\u53ef\u4ee5\u6dfb\u52a0\u4ee3\u7406\u7528\u6237\u3002 jindo admin -addProxyUser [-dlsUri <uri>] [-proxyUser <proxyUser>] [-users <user1,user2...>]|[-groups <group1,group2...>] [-hosts <host1,host2...>] \u5728\u540c\u4e00\u6761\u547d\u4ee4\u4e2d\uff0c\u53c2\u6570 users \u548c groups \u4e0d\u53ef\u540c\u65f6\u4f7f\u7528\uff0c\u4efb\u9009\u5176\u4e00\u3002 \u4f8b\uff1a jindo admin -addProxyUser -dlsUri oss://jindosdk-dls-unit-test/ -proxyUser hive -groups group1,group2 -hosts host1,host2 \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u7528\u6237 hive \u53ef\u4ee5\u4ee3\u7406\u5c5e\u4e8e group1 \u6216 group2 \u4e14\u4ece host1 \u6216 host2 \u53d1\u9001\u8bf7\u6c42\u7684\u7528\u6237\u3002","title":"AddProxyUser \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#deleteproxyuser","text":"DeleteProxyUser \u547d\u4ee4\u53ef\u4ee5\u5220\u9664\u4ee3\u7406\u7528\u6237\u3002 jindo admin -deleteProxyUser [-dlsUri <uri>] [-proxyUser <proxyUser>] \u4f8b\uff1a jindo admin -deleteProxyUser -dlsUri oss://jindosdk-dls-unit-test/ -proxyUser hive \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u7528\u6237 hive \u4e0d\u53ef\u4ee5\u4ee3\u7406\u4efb\u4f55\u7528\u6237\u3002","title":"DeleteProxyUser \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#listproxyusers","text":"ListProxyUsers \u53ef\u4ee5\u67e5\u770b\u6240\u6709\u7684\u4ee3\u7406\u7528\u6237\u548c\u4ee3\u7406\u4fe1\u606f\u3002 jindo admin -listProxyUsers [-dlsUri <uri>] [-maxKeys <maxKeys>] [-marker <marker>] maxKeys \u6307\u5b9a\u67e5\u770b\u7684\u4ee3\u7406\u7528\u6237\u4e2a\u6570\uff0cmarker \u7528\u6765\u7b5b\u9009\u540d\u79f0\u4e2d\u542b\u6709\u7279\u6b8a\u6807\u8bb0\u7684\u4ee3\u7406\u7528\u6237\uff0c\u8fd9\u4e24\u9879\u53c2\u6570\u5747\u53ef\u7701\u7565\u3002 \u4f8b\uff1a jindo admin -listProxyUsers -dlsUri oss://jindosdk-dls-unit-test/ -maxKeys 10 -marker hive \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u5c06\u4f18\u5148\u663e\u793a oss://jindosdk-dls-unit-test/ \u4e0b\u540d\u79f0\u4e2d\u5e26 hive \u6807\u8bb0\u768410\u4e2a\u4ee3\u7406\u7528\u6237\u4fe1\u606f\u3002","title":"ListProxyUsers \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#user-groups-mapping","text":"","title":"\u8bbe\u7f6e User Groups Mapping \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#addusergroupsmapping","text":"AddUserGroupsMapping \u547d\u4ee4\u6620\u5c04\u7528\u6237\u548c\u7ec4\u7684\u5173\u7cfb\u3002 jindo admin -addUserGroupsMapping [-dlsUri <uri>] [-user <user>] [-groups <group1,group2...>] \u4f8b\uff1a jindo admin -addUserGroupsMapping -dlsUri oss://jindosdk-dls-unit-test/ -user user1 -groups group1,group2 \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u7528\u6237 user1 \u5c06\u5c5e\u4e8e group1 \u6216 group2\u3002","title":"AddUserGroupsMapping \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#deleteusergroupsmapping","text":"DeleteUserGroupsMapping \u547d\u4ee4\u53ef\u4ee5\u5220\u9664\u7528\u6237\u548c\u7ec4\u7684\u6620\u5c04\u5173\u7cfb\u3002 jindo admin -deleteUserGroupsMapping [-dlsUri <uri>] [-user <user>] \u4f8b\uff1a jindo admin -deleteUserGroupsMapping -dlsUri oss://jindosdk-dls-unit-test/ -user user1 \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u5c06\u5220\u9664 user1 \u4e0e\u5bf9\u5e94\u7ec4\u7684\u6620\u5c04\u5173\u7cfb\u3002","title":"DeleteUserGroupsMapping \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#listusergroupsmappings","text":"ListUserGroupsMappings \u53ef\u4ee5\u67e5\u770b\u6240\u6709\u7684\u7528\u6237\u7684\u7ec4\u3002 jindo admin -listUserGroupsMappings [-dlsUri <dlsUri>] [-maxKeys <maxKeys>] [-marker <marker>] maxKeys \u6307\u5b9a\u67e5\u770b\u7684\u7528\u6237\u4e2a\u6570\uff0cmarker \u7528\u6765\u7b5b\u9009\u540d\u79f0\u4e2d\u542b\u6709\u7279\u6b8a\u6807\u8bb0\u7684\u7528\u6237\uff0c\u8fd9\u4e24\u9879\u53c2\u6570\u5747\u53ef\u7701\u7565\u3002 \u4f8b\uff1a jindo admin -listUserGroupsMappings -dlsUri oss://jindosdk-dls-unit-test/ -maxKeys 10 -marker user1 \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u5c06\u4f18\u5148\u663e\u793a oss://jindosdk-dls-unit-test/ \u4e0b\u540d\u79f0\u4e2d\u5e26 user1 \u6807\u8bb0\u768410\u4e2a\u7528\u6237\u7684\u7ec4\u4fe1\u606f\u3002","title":"ListUserGroupsMappings \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#snapshot","text":"","title":"\u7ba1\u7406 Snapshot \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#allowsnapshot","text":"AllowSnapshot \u547d\u4ee4\u5141\u8bb8\u521b\u5efa\u76ee\u5f55\u5feb\u7167\u3002 jindo admin -allowSnapshot [-dlsUri <dlsSnapshotDir>] \u4f8b\uff1a jindo admin -allowSnapshot -dlsUri oss://jindosdk-dls-unit-test/testSnapshot/snapshot_allow \u6267\u884c\u4e0a\u9762\u7684\u547d\u4ee4\u540e\uff0c\u5c06\u5141\u8bb8 oss://jindosdk-dls-unit-test/testSnapshot/snapshot_allow \u76ee\u5f55\u521b\u5efa\u5feb\u7167\u3002","title":"AllowSnapshot \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#disallowsnapshot","text":"DisallowSnapshot \u547d\u4ee4\u53ef\u4ee5\u7981\u6b62\u521b\u5efa\u76ee\u5f55\u5feb\u7167\u3002 jindo admin -disallowSnapshot [-dlsUri <dlsSnapshotDir>] \u4f8b\uff1a jindo admin -disallowSnapshot -dlsUri oss://jindosdk-dls-unit-test/testSnapshot/snapshot_allow \u6267\u884c\u4e0a\u9762\u7684\u547d\u4ee4\u540e\uff0c\u5c06\u4e0d\u5141\u8bb8 oss://jindosdk-dls-unit-test/testSnapshot/snapshot_allow \u76ee\u5f55\u521b\u5efa\u5feb\u7167\u3002","title":"DisallowSnapshot \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#snapshotdiff","text":"SnapshotDiff \u547d\u4ee4\u7528\u6765\u6bd4\u8f83\u540c\u4e00\u76ee\u5f55\u4e24\u4efd\u5feb\u7167\u7684\u4e0d\u540c\u3002 jindo admin -snapshotDiff [-dlsUri <dlsSnapshotDir>] [-fromSnapshot <fromSnapshot>] [-toSnapshot <toSnapshot>] \u4f8b\uff1a jindo admin -snapshotDiff -dlsUri oss://jindosdk-dls-unit-test/testSnapshotDir/ -fromSnapshot S1 -toSnapshot S2 S1 \u548c S1 \u662f\u4e8b\u5148\u901a\u8fc7 HDFS \u7684 Shell \u547d\u4ee4\u884c\u5de5\u5177\u4e3a\u6d4b\u8bd5\u76ee\u5f55\u521b\u5efa\u7684\u4e24\u4efd\u5feb\u7167\u3002","title":"SnapshotDiff \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#accesspolicy","text":"","title":"\u901a\u8fc7 AccessPolicy \u547d\u4ee4\u5b9e\u73b0\u8def\u5f84\u6539\u5199"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#setrootpolicy","text":"SetRootPolicy \u547d\u4ee4\u5141\u8bb8\u4e3a bucket \u8bbe\u7f6e\u4efb\u610f\u524d\u7f00\u7684\u8bbf\u95ee\u8def\u5f84\u3002 jindo admin -setRootPolicy [<dlsRootPath>] [<accessRootPath>] \u4f8b\uff1a jindo admin -setRootPolicy oss://jindosdk-dls-unit-test/ hdfs://myufs/ \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\u540e\uff0c\u652f\u6301\u4f7f\u7528 hdfs://myufs/ \u8bbf\u95ee oss://jindosdk-dls-unit-test/\u3002","title":"SetRootPolicy \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#unsetrootpolicy","text":"UnsetRootPolicy \u547d\u4ee4\u4e3a bucket \u53d6\u6d88\u8bbf\u95ee\u8def\u5f84\u3002 jindo admin -unsetRootPolicy [<dlsRootPath>] [<accessRootPath>] \u4f8b\uff1a jindo admin -unsetRootPolicy oss://jindosdk-dls-unit-test/ hdfs://myufs/ \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\u540e\uff0c\u4e0d\u518d\u652f\u6301\u7528 hdfs://myufs/ \u8bbf\u95ee oss://jindosdk-dls-unit-test/\u3002","title":"UnsetRootPolicy \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#listaccesspolicies","text":"ListAccessPolicies \u53ef\u67e5\u770b\u5f53\u524d bucket \u652f\u6301\u7684\u6240\u6709\u8bbf\u95ee\u8def\u5f84\u3002 jindo admin -listAccessPolicies [<dlsRootPath>] \u4f8b\uff1a jindo admin -listAccessPolicies oss://jindosdk-dls-unit-test/ \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u67e5\u770b oss://jindosdk-dls-unit-test/ \u7684\u6240\u6709\u8bbf\u95ee\u8def\u5f84\u3002","title":"ListAccessPolicies \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#_1","text":"","title":"\u5176\u4ed6"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#dumpfile","text":"DumpFile \u547d\u4ee4\u53ef\u4ee5\u67e5\u770b\u6587\u4ef6\u7684\u8be6\u7ec6\u5b58\u50a8\u4fe1\u606f\u3002 jindo admin -dumpFile [-dlsUri <dlsUri>] \u4f8b\uff1a jindo admin -dumpFile -dlsUri oss://jindosdk-dls-unit-test/testDumpFile/dumpfile_simple_file \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\u540e\uff0c\u5c06\u663e\u793a\u7c7b\u4f3c\u5982\u4e0b\u7ed3\u679c\uff1a =============Dump result============= File /testDumpFile/dumpfile_simple_file length: 11 blockSize: 33554432 modification time: 1657626266711 access time: 1657626266481 permission: 420 owner: root group: supergroup file id: 4180702376518228901 =============Block 0 block id: 8792388394945610760 offset: 0 numBytes: 11 gs: 1032 type: CLOUD loc: .dlsdata underConstruction: 0 isLastBlockComplete: 1","title":"DumpFile \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u7167\u4f7f\u7528\u8bf4\u660e \u00a7 (\u4ece 4.0.0 \u5f00\u59cb\u652f\u6301) \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u662fOSS\u65b0\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9HDFS\u63a5\u53e3, \u652f\u6301\u76ee\u5f55\u4ee5\u53ca\u76ee\u5f55\u5c42\u7ea7\uff0c\u901a\u8fc7Jindo SDK 4.x \u53ef\u4ee5\u517c\u5bb9\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u3002\u5173\u4e8e\u7528\u6237\u5982\u4f55\u521b\u5efa\u548c\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u7684\u57fa\u672c\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 \u94fe\u63a5 \u3002 OSS-HDFS \u670d\u52a1\u7684\u5feb\u7167\u529f\u80fd\u5728\u4f7f\u7528\u4e0a\u4e0eHDFS\u7684\u5feb\u7167\u529f\u80fd\u5b8c\u5168\u517c\u5bb9\u3002\u672c\u6587\u4e3b\u8981\u4ecb\u7ecd\u5728 OSS-HDFS \u670d\u52a1\u4e2d\u4f7f\u7528\u5feb\u7167\u529f\u80fd\u7684\u4e00\u4e9b\u5e38\u89c1\u64cd\u4f5c\u3002 \u4e3a\u4e86\u65b9\u4fbf\u4e0b\u9762\u547d\u4ee4\u7684\u4ecb\u7ecd\uff0c\u6211\u4eec\u5047\u8bbe oss://oss-dfs-test \u8fd9\u4e2aBucket\u5df2\u7ecf\u5f00\u542f OSS-HDFS \u670d\u52a1\u3002\u4e0b\u9762\u6240\u6709\u7684\u4f8b\u5b50\u90fd\u4f1a\u9488\u5bf9\u8fd9\u4e2aBucket\u8fdb\u884c\u64cd\u4f5c\u3002 \u5f00\u542f\u5feb\u7167\u529f\u80fd \u00a7 \u5047\u8bbe\u6211\u4eec\u5148\u5728 oss://oss-dfs-test \u4e0b\u521b\u5efa\u4e00\u4e2a\u76ee\u5f55TestSnapshot\u3002 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u662f\u5173\u95ed\u7684\u3002\u9700\u8981\u5f00\u542f\u548c\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\uff0c\u9700\u8981\u4f7f\u7528Jindo SDK\u7684shell\u547d\u4ee4\u884c\u3002\u5177\u4f53\u7684\u5f00\u542f\u5feb\u7167\u7684\u683c\u5f0f\u4e3a jindo admin -allowSnapshot -dlsUri <path> \u5982\u679c\u6211\u4eec\u8981\u5f00\u542f\u524d\u9762\u521b\u5efa\u7684TestSnapshot\u7684\u5feb\u7167\u529f\u80fd\u3002\u53ef\u4ee5\u8f93\u5165\u5982\u4e0b\u7684\u547d\u4ee4\uff1a jindo admin -allowSnapshot -dlsUri oss://oss-dfs-test.<Endpoint>/TestSnapshot \u521b\u5efa\u5feb\u7167 \u00a7 \u5728\u4e00\u4e2a\u76ee\u5f55\u5f00\u542f\u5feb\u7167\u529f\u80fd\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7HDFS\u7684shell\u547d\u4ee4\u3002\u5177\u4f53\u683c\u5f0f\u4e3a\uff1a hdfs dfs -createSnapshot <path> [<snapshotName>] \u4f8b\u5982\u4e0a\u9762\u4f8b\u5b50\u7684\u4e2d\u7684TestSnapshot\uff0c\u6211\u4eec\u8fdb\u884c\u4e86\u4e00\u4e9b\u76ee\u5f55\u548c\u6587\u4ef6\u7684\u64cd\u4f5c\u3002\u7136\u540e\u53ef\u4ee5\u521b\u5efa\u4e00\u4e2a\u5feb\u7167\u5bf9\u6b64\u523b\u7684\u5feb\u7167\u7684\u72b6\u6001\u8fdb\u884c\u4fdd\u5b58\u3002\u5047\u8bbe\u6211\u4eec\u547d\u540d\u5feb\u7167\u540d\u4e3aS1\u3002 #\u4e0b\u9762\u8fd9\u4e9b\u4e3a\u521b\u5efa\u6d4b\u8bd5\u7684\u6587\u4ef6\u548c\u76ee\u5f55 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir2 hdfs dfs -touchz oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1/file1 hdfs dfs -touchz oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir3/file2 #\u901a\u8fc7HDFS\u7684shell\u547d\u4ee4\u884c\u5de5\u5177\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u521b\u5efa\u5feb\u7167\uff0c\u5e76\u5c06\u5feb\u7167\u547d\u540d\u4e3aS1\uff1a hdfs dfs -createSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S1 \u8bbf\u95ee\u5feb\u7167\u4e2d\u7684\u76ee\u5f55\u548c\u6587\u4ef6 \u00a7 \u5feb\u7167\u76ee\u5f55\u7684\u683c\u5f0f \u00a7 \u4e3a\u4e86\u533a\u522b\u4e8e\u6587\u4ef6\u7cfb\u7edf\u73b0\u6709\u7684\u76ee\u5f55\u548c\u6587\u4ef6\uff0c\u8bbf\u95ee\u5feb\u7167\u4e2d\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u9700\u8981\u6dfb\u52a0\u5feb\u7167\u7684\u4fe1\u606f\u6765\u6307\u51fa\u81ea\u5df1\u8981\u8bbf\u95ee\u7684\u5feb\u7167\u540d\u3002\u5bf9\u4e8e\u4e00\u4e2a\u5f00\u542f\u4e86\u5feb\u7167\u7684\u76ee\u5f55\uff0c\u6211\u4eec\u8981\u8bbf\u95ee\u5b83\u4e0b\u9762\u67d0\u4e2a\u5feb\u7167\u7684\u76ee\u5f55\u548c\u6587\u4ef6\uff0c\u6211\u4eec\u53ef\u4ee5\u4e0b\u9762\u7684\u683c\u5f0f\u6765\u8bbf\u95ee\uff1a <snapshotRoot>/.snapshot/<snapshotName>/<actual subPath> \u5176\u4e2dsnapshotRoot\u4e3a\u5feb\u7167\u7684\u6839\u76ee\u5f55\uff0c\u4e5f\u5c31\u662f\u5f00\u542f\u5feb\u7167\u547d\u4ee4\u4e2ddlsUri\u53c2\u6570\u5236\u5b9a\u7684\u8def\u5f84\u3002snapshotName\u662f\u4e4b\u524d\u521b\u5efa\u7684\u5feb\u7167\u540d\u3002\u5236\u5b9a\u4e86\u5feb\u7167\u540d\u4e4b\u540e\uff0c\u4e4b\u540e\u7684\u8def\u5f84\u5c31\u662f\u539f\u6765\u5feb\u7167\u6839\u76ee\u5f55\uff08snapshotRoot\uff09\u4e0b\u8981\u8bbf\u95ee\u7684\u8def\u5f84\u3002 \u6bd4\u5982\u4e0a\u9762\u7684\u4f8b\u5b50TestSnapshot\u76ee\u5f55\uff0c\u6211\u4eec\u5982\u679c\u8981\u5217\u51fa oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u4e0b\u7684\u6587\u4ef6\u3002\u53ef\u4ee5\u7528\u5e38\u89c4\u7684ls\u547d\u4ee4\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u7531\u4e8e\u6211\u4eec\u5728TestSnapshot\u4e0b\u5f00\u542f\u7684\u5feb\u7167\uff0c\u5e76\u521b\u5efa\u4e86\u5feb\u7167S1\u3002\u6211\u4eec\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u8def\u5f84\u6765\u5217\u51fa\u5feb\u7167S1\u4e0b\u7684\u76ee\u5f55\u548c\u6587\u4ef6\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/.snapshot/S1/dir1 \u5176\u4e2d.snapshot/S1 \u5c31\u662f\u7528\u6765\u8868\u660e\u76ee\u5f55\u7684\u5feb\u7167\u7684\u4fe1\u606f\u3002 \u5229\u7528\u5feb\u7167\u6062\u590d\u6570\u636e \u00a7 \u5feb\u7167\u7684\u4e00\u4e2a\u5e38\u89c1\u4f5c\u7528\u662f\u8fdb\u884c\u6570\u636e\u5907\u4efd\u548c\u6062\u590d\u3002\u5229\u7528\u5feb\u7167\u53ef\u4ee5\u5bf9\u91cd\u8981\u6570\u636e\u8fdb\u884c\u6062\u590d\uff0c\u9632\u6b62\u7528\u6237\u9519\u8bef\u6027\u7684\u64cd\u4f5c\u3002\u53ea\u8981\u6839\u636e\u4e0a\u9762\u8bb2\u7684 \u5feb\u7167\u76ee\u5f55\u7684\u683c\u5f0f \u5c31\u53ef\u4ee5\u8bbf\u95ee\u5230\u53ea\u8bfb\u526f\u672c\u4e2d\u7684\u6570\u636e\uff0c\u7136\u540e\u53ef\u4ee5\u6839\u636e\u9700\u6c42\u8fdb\u884c\u6062\u590d\u64cd\u4f5c\u3002 \u5728\u4e0a\u9762\u7684\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u5047\u8bbe\u5728\u540e\u9762\u7684\u67d0\u6b21\u4efb\u52a1\u4e2d\u8bef\u5220\u4e86 oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 hdfs dfs -rm -r oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u7531\u4e8e\u6211\u4eec\u5df2\u7ecf\u5bf9/TestSnapshot\u6dfb\u52a0\u4e86\u5feb\u7167S1\u3002\u6211\u4eec\u53ef\u4ee5\u7528\u5982\u4e0b\u7684\u547d\u4ee4\u6062\u590d\u6570\u636e\u3002 hdfs dfs -cp oss://oss-dfs-test.<Endpoint>/TestSnapshot/.snapshot/S1/dir1 oss://oss-dfs-test.<Endpoint>/TestSnapshot \u8fd9\u4e2a\u65f6\u5019\u6211\u4eec\u518d\u8bbf\u95ee/TestSnapshot/dir1\uff0c\u5c31\u4f1a\u53d1\u73b0\u539f\u6765\u7684\u8bef\u5220\u7684\u6587\u4ef6\u5939\u548c\u6587\u4ef6\u90fd\u6062\u590d\u4e86\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u91cd\u547d\u540d\u5feb\u7167 \u00a7 \u5bf9\u4e8e\u4e00\u4e2a\u5df2\u7ecf\u521b\u5efa\u7684\u5feb\u7167\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u91cd\u547d\u540d hdfs dfs -renameSnapshot <path> <oldName> <newName> \u4f8b\u5982\u4e0a\u9762\u4f8b\u5b50\u7684\u4e2d\u7684TestSnapshot\u7684\u5feb\u7167S1\uff0c\u6211\u4eec\u53ef\u4ee5\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u91cd\u547d\u540d\u4e3aS100 hdfs dfs -renameSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S1 S100 \u5220\u9664\u5feb\u7167 \u00a7 \u5f53\u4e0d\u9700\u8981\u5feb\u7167\u7684\u65f6\u5019\uff0c\u53ef\u4ee5\u8fd0\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5220\u9664\u5236\u5b9a\u7684\u5feb\u7167\u3002 hdfs dfs -deleteSnapshot <path> <snapshotName> \u5728\u4e0a\u9762\u7684\u4f8b\u5b50\u4e2d\uff0c\u6211\u4eec\u53ef\u4ee5\u8fd0\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5220\u9664\u4e4b\u524d\u521b\u5efa\u5e76\u91cd\u547d\u540d\u7684\u5feb\u7167S100 hdfs dfs -deleteSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S100 \u5173\u95ed\u5feb\u7167\u529f\u80fd \u00a7 \u5f53\u6211\u4eec\u9700\u8981\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u7684\u65f6\u5019\uff0c\u6211\u4eec\u540c\u6837\u9700\u8981\u4f7f\u7528Jindo SDK\u7684shell\u547d\u4ee4\u884c\u3002\u5177\u4f53\u7684\u5173\u95ed\u5feb\u7167\u7684\u683c\u5f0f\u4e3a jindo admin -disallowSnapshot -dlsUri <path> \u67e5\u770b\u4e24\u4e2a\u5feb\u7167\u4e4b\u95f4\u7684\u5dee\u5f02 \u00a7 \u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u5f53\u547d\u4ee4\u67e5\u770b\u4e24\u4e2a\u5feb\u7167\u4e4b\u95f4\u7684\u5dee\u5f02 jindo dls -snapshotDiff -dlsUri <uri> -fromSnapshot <fromSnapshot> -toSnapshot <toSnapshot> \u6ce8\u610f\u4e8b\u9879 \u00a7 \u5982\u679c\u9700\u8981\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\uff0c\u9996\u5148\u8981\u786e\u4fdd\u8be5\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u5feb\u7167\u5df2\u7ecf\u88ab\u5220\u9664\u3002\u5220\u9664\u5feb\u7167\u7684\u547d\u4ee4\u53ef\u4ee5\u53c2\u8003\u524d\u9762\u7684\u5220\u9664\u5feb\u7167\u547d\u4ee4\u3002\u5982\u679c\u8be5\u76ee\u5f55\u4e0b\u8fd8\u5b58\u5728\u5feb\u7167\uff0c\u5173\u95ed\u5feb\u7167\u529f\u80fd\u4f1a\u62a5\u9519\u3002 \u5bf9\u4e8e\u6211\u4eec\u524d\u9762\u7684\u4f8b\u5b50TestSnapshot\uff0c\u5047\u8bbe\u6211\u4eec\u5df2\u7ecf\u5220\u9664\u4e86\u6240\u6709\u5feb\u7167\u3002\u6211\u4eec\u53ef\u4ee5\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u3002 jindo admin -disallowSnapshot -dlsUri oss://oss-dfs-test.<Endpoint>/TestSnapshot","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u7167\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#oss-hdfs-jindofs","text":"(\u4ece 4.0.0 \u5f00\u59cb\u652f\u6301) \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u662fOSS\u65b0\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9HDFS\u63a5\u53e3, \u652f\u6301\u76ee\u5f55\u4ee5\u53ca\u76ee\u5f55\u5c42\u7ea7\uff0c\u901a\u8fc7Jindo SDK 4.x \u53ef\u4ee5\u517c\u5bb9\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u3002\u5173\u4e8e\u7528\u6237\u5982\u4f55\u521b\u5efa\u548c\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u7684\u57fa\u672c\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 \u94fe\u63a5 \u3002 OSS-HDFS \u670d\u52a1\u7684\u5feb\u7167\u529f\u80fd\u5728\u4f7f\u7528\u4e0a\u4e0eHDFS\u7684\u5feb\u7167\u529f\u80fd\u5b8c\u5168\u517c\u5bb9\u3002\u672c\u6587\u4e3b\u8981\u4ecb\u7ecd\u5728 OSS-HDFS \u670d\u52a1\u4e2d\u4f7f\u7528\u5feb\u7167\u529f\u80fd\u7684\u4e00\u4e9b\u5e38\u89c1\u64cd\u4f5c\u3002 \u4e3a\u4e86\u65b9\u4fbf\u4e0b\u9762\u547d\u4ee4\u7684\u4ecb\u7ecd\uff0c\u6211\u4eec\u5047\u8bbe oss://oss-dfs-test \u8fd9\u4e2aBucket\u5df2\u7ecf\u5f00\u542f OSS-HDFS \u670d\u52a1\u3002\u4e0b\u9762\u6240\u6709\u7684\u4f8b\u5b50\u90fd\u4f1a\u9488\u5bf9\u8fd9\u4e2aBucket\u8fdb\u884c\u64cd\u4f5c\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u7167\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#_1","text":"\u5047\u8bbe\u6211\u4eec\u5148\u5728 oss://oss-dfs-test \u4e0b\u521b\u5efa\u4e00\u4e2a\u76ee\u5f55TestSnapshot\u3002 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u662f\u5173\u95ed\u7684\u3002\u9700\u8981\u5f00\u542f\u548c\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\uff0c\u9700\u8981\u4f7f\u7528Jindo SDK\u7684shell\u547d\u4ee4\u884c\u3002\u5177\u4f53\u7684\u5f00\u542f\u5feb\u7167\u7684\u683c\u5f0f\u4e3a jindo admin -allowSnapshot -dlsUri <path> \u5982\u679c\u6211\u4eec\u8981\u5f00\u542f\u524d\u9762\u521b\u5efa\u7684TestSnapshot\u7684\u5feb\u7167\u529f\u80fd\u3002\u53ef\u4ee5\u8f93\u5165\u5982\u4e0b\u7684\u547d\u4ee4\uff1a jindo admin -allowSnapshot -dlsUri oss://oss-dfs-test.<Endpoint>/TestSnapshot","title":"\u5f00\u542f\u5feb\u7167\u529f\u80fd"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#_2","text":"\u5728\u4e00\u4e2a\u76ee\u5f55\u5f00\u542f\u5feb\u7167\u529f\u80fd\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7HDFS\u7684shell\u547d\u4ee4\u3002\u5177\u4f53\u683c\u5f0f\u4e3a\uff1a hdfs dfs -createSnapshot <path> [<snapshotName>] \u4f8b\u5982\u4e0a\u9762\u4f8b\u5b50\u7684\u4e2d\u7684TestSnapshot\uff0c\u6211\u4eec\u8fdb\u884c\u4e86\u4e00\u4e9b\u76ee\u5f55\u548c\u6587\u4ef6\u7684\u64cd\u4f5c\u3002\u7136\u540e\u53ef\u4ee5\u521b\u5efa\u4e00\u4e2a\u5feb\u7167\u5bf9\u6b64\u523b\u7684\u5feb\u7167\u7684\u72b6\u6001\u8fdb\u884c\u4fdd\u5b58\u3002\u5047\u8bbe\u6211\u4eec\u547d\u540d\u5feb\u7167\u540d\u4e3aS1\u3002 #\u4e0b\u9762\u8fd9\u4e9b\u4e3a\u521b\u5efa\u6d4b\u8bd5\u7684\u6587\u4ef6\u548c\u76ee\u5f55 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir2 hdfs dfs -touchz oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1/file1 hdfs dfs -touchz oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir3/file2 #\u901a\u8fc7HDFS\u7684shell\u547d\u4ee4\u884c\u5de5\u5177\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u521b\u5efa\u5feb\u7167\uff0c\u5e76\u5c06\u5feb\u7167\u547d\u540d\u4e3aS1\uff1a hdfs dfs -createSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S1","title":"\u521b\u5efa\u5feb\u7167"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#_3","text":"","title":"\u8bbf\u95ee\u5feb\u7167\u4e2d\u7684\u76ee\u5f55\u548c\u6587\u4ef6"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#_4","text":"\u4e3a\u4e86\u533a\u522b\u4e8e\u6587\u4ef6\u7cfb\u7edf\u73b0\u6709\u7684\u76ee\u5f55\u548c\u6587\u4ef6\uff0c\u8bbf\u95ee\u5feb\u7167\u4e2d\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u9700\u8981\u6dfb\u52a0\u5feb\u7167\u7684\u4fe1\u606f\u6765\u6307\u51fa\u81ea\u5df1\u8981\u8bbf\u95ee\u7684\u5feb\u7167\u540d\u3002\u5bf9\u4e8e\u4e00\u4e2a\u5f00\u542f\u4e86\u5feb\u7167\u7684\u76ee\u5f55\uff0c\u6211\u4eec\u8981\u8bbf\u95ee\u5b83\u4e0b\u9762\u67d0\u4e2a\u5feb\u7167\u7684\u76ee\u5f55\u548c\u6587\u4ef6\uff0c\u6211\u4eec\u53ef\u4ee5\u4e0b\u9762\u7684\u683c\u5f0f\u6765\u8bbf\u95ee\uff1a <snapshotRoot>/.snapshot/<snapshotName>/<actual subPath> \u5176\u4e2dsnapshotRoot\u4e3a\u5feb\u7167\u7684\u6839\u76ee\u5f55\uff0c\u4e5f\u5c31\u662f\u5f00\u542f\u5feb\u7167\u547d\u4ee4\u4e2ddlsUri\u53c2\u6570\u5236\u5b9a\u7684\u8def\u5f84\u3002snapshotName\u662f\u4e4b\u524d\u521b\u5efa\u7684\u5feb\u7167\u540d\u3002\u5236\u5b9a\u4e86\u5feb\u7167\u540d\u4e4b\u540e\uff0c\u4e4b\u540e\u7684\u8def\u5f84\u5c31\u662f\u539f\u6765\u5feb\u7167\u6839\u76ee\u5f55\uff08snapshotRoot\uff09\u4e0b\u8981\u8bbf\u95ee\u7684\u8def\u5f84\u3002 \u6bd4\u5982\u4e0a\u9762\u7684\u4f8b\u5b50TestSnapshot\u76ee\u5f55\uff0c\u6211\u4eec\u5982\u679c\u8981\u5217\u51fa oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u4e0b\u7684\u6587\u4ef6\u3002\u53ef\u4ee5\u7528\u5e38\u89c4\u7684ls\u547d\u4ee4\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u7531\u4e8e\u6211\u4eec\u5728TestSnapshot\u4e0b\u5f00\u542f\u7684\u5feb\u7167\uff0c\u5e76\u521b\u5efa\u4e86\u5feb\u7167S1\u3002\u6211\u4eec\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u8def\u5f84\u6765\u5217\u51fa\u5feb\u7167S1\u4e0b\u7684\u76ee\u5f55\u548c\u6587\u4ef6\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/.snapshot/S1/dir1 \u5176\u4e2d.snapshot/S1 \u5c31\u662f\u7528\u6765\u8868\u660e\u76ee\u5f55\u7684\u5feb\u7167\u7684\u4fe1\u606f\u3002","title":"\u5feb\u7167\u76ee\u5f55\u7684\u683c\u5f0f"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#_5","text":"\u5feb\u7167\u7684\u4e00\u4e2a\u5e38\u89c1\u4f5c\u7528\u662f\u8fdb\u884c\u6570\u636e\u5907\u4efd\u548c\u6062\u590d\u3002\u5229\u7528\u5feb\u7167\u53ef\u4ee5\u5bf9\u91cd\u8981\u6570\u636e\u8fdb\u884c\u6062\u590d\uff0c\u9632\u6b62\u7528\u6237\u9519\u8bef\u6027\u7684\u64cd\u4f5c\u3002\u53ea\u8981\u6839\u636e\u4e0a\u9762\u8bb2\u7684 \u5feb\u7167\u76ee\u5f55\u7684\u683c\u5f0f \u5c31\u53ef\u4ee5\u8bbf\u95ee\u5230\u53ea\u8bfb\u526f\u672c\u4e2d\u7684\u6570\u636e\uff0c\u7136\u540e\u53ef\u4ee5\u6839\u636e\u9700\u6c42\u8fdb\u884c\u6062\u590d\u64cd\u4f5c\u3002 \u5728\u4e0a\u9762\u7684\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u5047\u8bbe\u5728\u540e\u9762\u7684\u67d0\u6b21\u4efb\u52a1\u4e2d\u8bef\u5220\u4e86 oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 hdfs dfs -rm -r oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u7531\u4e8e\u6211\u4eec\u5df2\u7ecf\u5bf9/TestSnapshot\u6dfb\u52a0\u4e86\u5feb\u7167S1\u3002\u6211\u4eec\u53ef\u4ee5\u7528\u5982\u4e0b\u7684\u547d\u4ee4\u6062\u590d\u6570\u636e\u3002 hdfs dfs -cp oss://oss-dfs-test.<Endpoint>/TestSnapshot/.snapshot/S1/dir1 oss://oss-dfs-test.<Endpoint>/TestSnapshot \u8fd9\u4e2a\u65f6\u5019\u6211\u4eec\u518d\u8bbf\u95ee/TestSnapshot/dir1\uff0c\u5c31\u4f1a\u53d1\u73b0\u539f\u6765\u7684\u8bef\u5220\u7684\u6587\u4ef6\u5939\u548c\u6587\u4ef6\u90fd\u6062\u590d\u4e86\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1","title":"\u5229\u7528\u5feb\u7167\u6062\u590d\u6570\u636e"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#_6","text":"\u5bf9\u4e8e\u4e00\u4e2a\u5df2\u7ecf\u521b\u5efa\u7684\u5feb\u7167\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u91cd\u547d\u540d hdfs dfs -renameSnapshot <path> <oldName> <newName> \u4f8b\u5982\u4e0a\u9762\u4f8b\u5b50\u7684\u4e2d\u7684TestSnapshot\u7684\u5feb\u7167S1\uff0c\u6211\u4eec\u53ef\u4ee5\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u91cd\u547d\u540d\u4e3aS100 hdfs dfs -renameSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S1 S100","title":"\u91cd\u547d\u540d\u5feb\u7167"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#_7","text":"\u5f53\u4e0d\u9700\u8981\u5feb\u7167\u7684\u65f6\u5019\uff0c\u53ef\u4ee5\u8fd0\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5220\u9664\u5236\u5b9a\u7684\u5feb\u7167\u3002 hdfs dfs -deleteSnapshot <path> <snapshotName> \u5728\u4e0a\u9762\u7684\u4f8b\u5b50\u4e2d\uff0c\u6211\u4eec\u53ef\u4ee5\u8fd0\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5220\u9664\u4e4b\u524d\u521b\u5efa\u5e76\u91cd\u547d\u540d\u7684\u5feb\u7167S100 hdfs dfs -deleteSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S100","title":"\u5220\u9664\u5feb\u7167"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#_8","text":"\u5f53\u6211\u4eec\u9700\u8981\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u7684\u65f6\u5019\uff0c\u6211\u4eec\u540c\u6837\u9700\u8981\u4f7f\u7528Jindo SDK\u7684shell\u547d\u4ee4\u884c\u3002\u5177\u4f53\u7684\u5173\u95ed\u5feb\u7167\u7684\u683c\u5f0f\u4e3a jindo admin -disallowSnapshot -dlsUri <path>","title":"\u5173\u95ed\u5feb\u7167\u529f\u80fd"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#_9","text":"\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u5f53\u547d\u4ee4\u67e5\u770b\u4e24\u4e2a\u5feb\u7167\u4e4b\u95f4\u7684\u5dee\u5f02 jindo dls -snapshotDiff -dlsUri <uri> -fromSnapshot <fromSnapshot> -toSnapshot <toSnapshot>","title":"\u67e5\u770b\u4e24\u4e2a\u5feb\u7167\u4e4b\u95f4\u7684\u5dee\u5f02"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#_10","text":"\u5982\u679c\u9700\u8981\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\uff0c\u9996\u5148\u8981\u786e\u4fdd\u8be5\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u5feb\u7167\u5df2\u7ecf\u88ab\u5220\u9664\u3002\u5220\u9664\u5feb\u7167\u7684\u547d\u4ee4\u53ef\u4ee5\u53c2\u8003\u524d\u9762\u7684\u5220\u9664\u5feb\u7167\u547d\u4ee4\u3002\u5982\u679c\u8be5\u76ee\u5f55\u4e0b\u8fd8\u5b58\u5728\u5feb\u7167\uff0c\u5173\u95ed\u5feb\u7167\u529f\u80fd\u4f1a\u62a5\u9519\u3002 \u5bf9\u4e8e\u6211\u4eec\u524d\u9762\u7684\u4f8b\u5b50TestSnapshot\uff0c\u5047\u8bbe\u6211\u4eec\u5df2\u7ecf\u5220\u9664\u4e86\u6240\u6709\u5feb\u7167\u3002\u6211\u4eec\u53ef\u4ee5\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u3002 jindo admin -disallowSnapshot -dlsUri oss://oss-dfs-test.<Endpoint>/TestSnapshot","title":"\u6ce8\u610f\u4e8b\u9879"},{"location":"backup/jindofs/jindofs_dls_trash_howto/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u56de\u6536\u7ad9\u4f7f\u7528\u8bf4\u660e \u00a7 (\u4ece 4.5.1 \u5f00\u59cb\u652f\u6301) \u4ecb\u7ecd \u00a7 \u5f53\u4eceJindoFS\u5220\u9664\u6587\u4ef6\u65f6\uff0c\u6587\u4ef6\u4e0d\u662f\u7acb\u5373\u88ab\u7acb\u5373\u5220\u9664\u3002\u88ab\u5220\u9664\u7684\u6587\u4ef6\u88ab\u632a\u5230\u4e86 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\u3002\u5f53\u7ecf\u8fc730\u5206\u949f\u540e\uff0cCurrent\u76ee\u5f55\u4f1a\u88ab\u632a\u5230 ```bash /user/ /.Trash/ \u4e5f\u5c31\u662f\u8bf4\u67d0\u4e00\u65f6\u95f4\u6bb5\u5185\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u4f1a\u88ab\u5f52\u7c7b\u5230\u4e00\u4e2a\u5e26\u65f6\u95f4\u6233\u7684\u76ee\u5f55\u4e0b\uff0c\u8868\u793a\u662f\u5728\u8fd9\u4e2a\u65f6\u95f4\u5185\u88ab\u5220\u9664\u7684\uff0c\u76f8\u5f53\u4e8e\u4e00\u6b21checkpoint\u3002\u5f53\u7ecf\u8fc7**7\u5929**\u540e\uff0c\u8fd9\u4e2a<timestamp>\u76ee\u5f55\u5c06\u4f1a\u88ab\u6c38\u8fdc\u5220\u9664\u3002\u56e0\u6b64\u57287\u5929\u5185\uff0c\u60a8\u6709\u673a\u4f1a\u4ece.Trash\u76ee\u5f55\u4e0b\u627e\u5230\u5bf9\u5e94\u65f6\u523b\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u5c06\u5176\u4ece.Trash\u632a\u51fa\uff0c\u4ece\u800c\u6062\u590d\u5b83\u3002 \u6ce8\u610f\uff1a\u8fd9\u9879\u529f\u80fd\uff0c\u662f\u7531\u5ba2\u6237\u7aef\u548c\u670d\u52a1\u7aef\u914d\u5408\uff0c\u65b9\u80fd\u5f62\u6210\u56de\u6536\u7ad9\u7684\u529f\u80fd\u3002\u670d\u52a1\u7aef\u53ea\u8d1f\u8d23\u7ef4\u62a4 ```/user/<username>/.Trash``` \u7684\u5b9a\u65f6\u6e05\u7406\uff0c\u670d\u52a1\u7aef\u7684\u5b9a\u65f6\u6e05\u7406\u662f\u9ed8\u8ba4\u5f00\u542f\u7684\u3002\u5ba2\u6237\u7aef\u8d1f\u8d23\u5c06\u5f85\u5220\u9664\u6587\u4ef6\u632a\u5230.Trash\u76ee\u5f55\u4e0b\u3002 ## Hadoop FileSystem Shell \u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd ```bash hadoop fs -rm oss://bucket/a/b/c \u5ba2\u6237\u7aef\u7684Hadoop Shell\u547d\u4ee4\u9ed8\u8ba4\u4e0d\u5f00\u542fTrash\u529f\u80fd\uff0c\u56e0\u6b64\u9700\u8981\u5728core-site.xml\u91cc\u6dfb\u52a0\u4e00\u6761\u914d\u7f6e <property> <name>fs.trash.interval</name> <value>1440</value> </property> (\u8be5\u503c\u53ea\u9700\u5927\u4e8e0\u5373\u53ef) \u6b64\u65f6\u5728\u5ba2\u6237\u7aef\u4f1a\u81ea\u52a8\u5c06rm\u547d\u4ee4\u8f6c\u6362\u4e3a\u4e00\u6761 hadoop fs -mv oss://bucket/a/b/c /user/<username>/.Trash/Current/a/b/c \u547d\u4ee4\uff0c\u56e0\u6b64\uff0c\u60a8\u4e0d\u9700\u8981\u611f\u77e5\u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u670d\u52a1\u7aef\u4f1a\u8d1f\u8d23\u6e05\u7406\u3002 \u5982\u679c\u60a8\u60f3\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\uff0c\u91ca\u653e\u7a7a\u95f4\uff0c\u53ef\u4ee5\u6dfb\u52a0 -skipTrash \u53c2\u6570\uff0c\u6b64\u65f6\u5c06\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\u3002 Hadoop \u751f\u6001\u7ec4\u4ef6\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd \u00a7 Hive/Spark/Flink \u7b49\u7ec4\u4ef6\u5e76\u4e0d\u611f\u77e5 JindoFS \u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u4f7f\u7528 FileSystem(HCFS) \u7684 delete \u63a5\u53e3\u610f\u5473\u7740\u7acb\u5373\u5220\u9664\u3002\u56e0\u6b64\u5982\u679c\u60f3\u8981\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd\uff0c\u9700\u8981\u663e\u5f0f\u5730\u8c03\u7528 FileSystem \u7684 rename \u63a5\u53e3\uff0c\u5c06\u76ee\u6807\u6587\u4ef6\u624b\u52a8\u632a\u5230 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\uff0c\u7531 JindoFS \u670d\u52a1\u7aef\u8d1f\u8d23\u5b9a\u671f\u5220\u9664\u3002\u8fd9\u4e00\u70b9\u4e0a JindoFS \u91c7\u53d6\u4e86\u8ddf\u5f00\u6e90 Hadoop \u76f8\u4f3c\u7684\u7b56\u7565\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u56de\u6536\u7ad9\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindofs_dls_trash_howto/#oss-hdfs-jindofs","text":"(\u4ece 4.5.1 \u5f00\u59cb\u652f\u6301)","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u56de\u6536\u7ad9\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindofs_dls_trash_howto/#_1","text":"\u5f53\u4eceJindoFS\u5220\u9664\u6587\u4ef6\u65f6\uff0c\u6587\u4ef6\u4e0d\u662f\u7acb\u5373\u88ab\u7acb\u5373\u5220\u9664\u3002\u88ab\u5220\u9664\u7684\u6587\u4ef6\u88ab\u632a\u5230\u4e86 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\u3002\u5f53\u7ecf\u8fc730\u5206\u949f\u540e\uff0cCurrent\u76ee\u5f55\u4f1a\u88ab\u632a\u5230 ```bash /user/ /.Trash/ \u4e5f\u5c31\u662f\u8bf4\u67d0\u4e00\u65f6\u95f4\u6bb5\u5185\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u4f1a\u88ab\u5f52\u7c7b\u5230\u4e00\u4e2a\u5e26\u65f6\u95f4\u6233\u7684\u76ee\u5f55\u4e0b\uff0c\u8868\u793a\u662f\u5728\u8fd9\u4e2a\u65f6\u95f4\u5185\u88ab\u5220\u9664\u7684\uff0c\u76f8\u5f53\u4e8e\u4e00\u6b21checkpoint\u3002\u5f53\u7ecf\u8fc7**7\u5929**\u540e\uff0c\u8fd9\u4e2a<timestamp>\u76ee\u5f55\u5c06\u4f1a\u88ab\u6c38\u8fdc\u5220\u9664\u3002\u56e0\u6b64\u57287\u5929\u5185\uff0c\u60a8\u6709\u673a\u4f1a\u4ece.Trash\u76ee\u5f55\u4e0b\u627e\u5230\u5bf9\u5e94\u65f6\u523b\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u5c06\u5176\u4ece.Trash\u632a\u51fa\uff0c\u4ece\u800c\u6062\u590d\u5b83\u3002 \u6ce8\u610f\uff1a\u8fd9\u9879\u529f\u80fd\uff0c\u662f\u7531\u5ba2\u6237\u7aef\u548c\u670d\u52a1\u7aef\u914d\u5408\uff0c\u65b9\u80fd\u5f62\u6210\u56de\u6536\u7ad9\u7684\u529f\u80fd\u3002\u670d\u52a1\u7aef\u53ea\u8d1f\u8d23\u7ef4\u62a4 ```/user/<username>/.Trash``` \u7684\u5b9a\u65f6\u6e05\u7406\uff0c\u670d\u52a1\u7aef\u7684\u5b9a\u65f6\u6e05\u7406\u662f\u9ed8\u8ba4\u5f00\u542f\u7684\u3002\u5ba2\u6237\u7aef\u8d1f\u8d23\u5c06\u5f85\u5220\u9664\u6587\u4ef6\u632a\u5230.Trash\u76ee\u5f55\u4e0b\u3002 ## Hadoop FileSystem Shell \u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd ```bash hadoop fs -rm oss://bucket/a/b/c \u5ba2\u6237\u7aef\u7684Hadoop Shell\u547d\u4ee4\u9ed8\u8ba4\u4e0d\u5f00\u542fTrash\u529f\u80fd\uff0c\u56e0\u6b64\u9700\u8981\u5728core-site.xml\u91cc\u6dfb\u52a0\u4e00\u6761\u914d\u7f6e <property> <name>fs.trash.interval</name> <value>1440</value> </property> (\u8be5\u503c\u53ea\u9700\u5927\u4e8e0\u5373\u53ef) \u6b64\u65f6\u5728\u5ba2\u6237\u7aef\u4f1a\u81ea\u52a8\u5c06rm\u547d\u4ee4\u8f6c\u6362\u4e3a\u4e00\u6761 hadoop fs -mv oss://bucket/a/b/c /user/<username>/.Trash/Current/a/b/c \u547d\u4ee4\uff0c\u56e0\u6b64\uff0c\u60a8\u4e0d\u9700\u8981\u611f\u77e5\u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u670d\u52a1\u7aef\u4f1a\u8d1f\u8d23\u6e05\u7406\u3002 \u5982\u679c\u60a8\u60f3\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\uff0c\u91ca\u653e\u7a7a\u95f4\uff0c\u53ef\u4ee5\u6dfb\u52a0 -skipTrash \u53c2\u6570\uff0c\u6b64\u65f6\u5c06\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\u3002","title":"\u4ecb\u7ecd"},{"location":"backup/jindofs/jindofs_dls_trash_howto/#hadoop","text":"Hive/Spark/Flink \u7b49\u7ec4\u4ef6\u5e76\u4e0d\u611f\u77e5 JindoFS \u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u4f7f\u7528 FileSystem(HCFS) \u7684 delete \u63a5\u53e3\u610f\u5473\u7740\u7acb\u5373\u5220\u9664\u3002\u56e0\u6b64\u5982\u679c\u60f3\u8981\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd\uff0c\u9700\u8981\u663e\u5f0f\u5730\u8c03\u7528 FileSystem \u7684 rename \u63a5\u53e3\uff0c\u5c06\u76ee\u6807\u6587\u4ef6\u624b\u52a8\u632a\u5230 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\uff0c\u7531 JindoFS \u670d\u52a1\u7aef\u8d1f\u8d23\u5b9a\u671f\u5220\u9664\u3002\u8fd9\u4e00\u70b9\u4e0a JindoFS \u91c7\u53d6\u4e86\u8ddf\u5f00\u6e90 Hadoop \u76f8\u4f3c\u7684\u7b56\u7565\u3002","title":"Hadoop \u751f\u6001\u7ec4\u4ef6\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd"},{"location":"backup/jindofs/outline/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7528\u6237\u6587\u6863 \u00a7 \u5feb\u901f\u5165\u95e8 \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8 Hadoop \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 \u57fa\u672c\u529f\u80fd \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u7167\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5206\u5c42\u5b58\u50a8\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Shell \u547d\u4ee4\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u56de\u6536\u7ad9\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5143\u6570\u636e\u5bfc\u51fa\u4f7f\u7528\u8bf4\u660e \u5927\u6570\u636e\u751f\u6001 \u00a7 Hadoop \u7ec4\u4ef6 \u00a7 Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 Spark \u7ec4\u4ef6 \u00a7 Spark \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e Hive \u7ec4\u4ef6 \u00a7 Hive \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e Presto \u7ec4\u4ef6 \u00a7 Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e Impala \u7ec4\u4ef6 \u00a7 Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e HBase \u7ec4\u4ef6 \u00a7 HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 Flink \u7ec4\u4ef6 \u00a7 Flink \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e Flume \u7ec4\u4ef6 \u00a7 Flume \u4f7f\u7528 JindoSDK \u5199\u5165\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 AI \u751f\u6001 \u00a7 JindoFuse \u4f7f\u7528 \u00a7 \u4f7f\u7528 JindoFuse \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u6700\u4f73\u5b9e\u8df5 \u00a7 JindoDistCp \u6570\u636e\u8fc1\u79fb \u00a7 JindoDistCp \u6570\u636e\u8fc1\u79fb JindoTable \u6570\u4ed3\u8fc1\u79fb \u00a7 \u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1) \u534a\u6258\u7ba1 JindoFS \u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 \u534a\u6258\u7ba1 JindoFS \u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u4f7f\u7528 RootPolicy \u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 \u4f7f\u7528 RootPolicy \u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u5e38\u89c1\u95ee\u9898 \u00a7 \u5e38\u89c1\u95ee\u9898 \u5386\u53f2\u7248\u672c \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.6.0 \u7528\u6237\u6587\u6863 \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.5.1 \u7528\u6237\u6587\u6863 \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.5.0 \u7528\u6237\u6587\u6863 \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.4.0 \u7528\u6237\u6587\u6863 \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.3.0 \u7528\u6237\u6587\u6863 \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.2.0 \u7528\u6237\u6587\u6863 \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.1.0 \u7528\u6237\u6587\u6863 \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.0.0 \u7528\u6237\u6587\u6863 \u00a7","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7528\u6237\u6587\u6863"},{"location":"backup/jindofs/outline/#oss-hdfs-jindofs","text":"","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7528\u6237\u6587\u6863"},{"location":"backup/jindofs/outline/#_1","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8 Hadoop \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5","title":"\u5feb\u901f\u5165\u95e8"},{"location":"backup/jindofs/outline/#_2","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u7167\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5206\u5c42\u5b58\u50a8\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Shell \u547d\u4ee4\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u56de\u6536\u7ad9\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5143\u6570\u636e\u5bfc\u51fa\u4f7f\u7528\u8bf4\u660e","title":"\u57fa\u672c\u529f\u80fd"},{"location":"backup/jindofs/outline/#_3","text":"","title":"\u5927\u6570\u636e\u751f\u6001"},{"location":"backup/jindofs/outline/#hadoop","text":"Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09","title":"Hadoop \u7ec4\u4ef6"},{"location":"backup/jindofs/outline/#spark","text":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e","title":"Spark \u7ec4\u4ef6"},{"location":"backup/jindofs/outline/#hive","text":"Hive \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e","title":"Hive \u7ec4\u4ef6"},{"location":"backup/jindofs/outline/#presto","text":"Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e","title":"Presto \u7ec4\u4ef6"},{"location":"backup/jindofs/outline/#impala","text":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e","title":"Impala \u7ec4\u4ef6"},{"location":"backup/jindofs/outline/#hbase","text":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8","title":"HBase \u7ec4\u4ef6"},{"location":"backup/jindofs/outline/#flink","text":"Flink \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e","title":"Flink \u7ec4\u4ef6"},{"location":"backup/jindofs/outline/#flume","text":"Flume \u4f7f\u7528 JindoSDK \u5199\u5165\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09","title":"Flume \u7ec4\u4ef6"},{"location":"backup/jindofs/outline/#ai","text":"","title":"AI \u751f\u6001"},{"location":"backup/jindofs/outline/#jindofuse","text":"\u4f7f\u7528 JindoFuse \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09","title":"JindoFuse \u4f7f\u7528"},{"location":"backup/jindofs/outline/#_4","text":"","title":"\u6700\u4f73\u5b9e\u8df5"},{"location":"backup/jindofs/outline/#jindodistcp","text":"JindoDistCp \u6570\u636e\u8fc1\u79fb","title":"JindoDistCp \u6570\u636e\u8fc1\u79fb"},{"location":"backup/jindofs/outline/#jindotable","text":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1)","title":"JindoTable \u6570\u4ed3\u8fc1\u79fb"},{"location":"backup/jindofs/outline/#jindofs-oss-hdfs-jindofs","text":"\u534a\u6258\u7ba1 JindoFS \u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09","title":"\u534a\u6258\u7ba1 JindoFS \u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/outline/#rootpolicy-oss-hdfs-jindofs","text":"\u4f7f\u7528 RootPolicy \u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09","title":"\u4f7f\u7528 RootPolicy \u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/outline/#_5","text":"\u5e38\u89c1\u95ee\u9898","title":"\u5e38\u89c1\u95ee\u9898"},{"location":"backup/jindofs/outline/#_6","text":"","title":"\u5386\u53f2\u7248\u672c"},{"location":"backup/jindofs/outline/#oss-hdfs-jindofs-460","text":"","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.6.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofs/outline/#oss-hdfs-jindofs-451","text":"","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.5.1 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofs/outline/#oss-hdfs-jindofs-450","text":"","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.5.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofs/outline/#oss-hdfs-jindofs-440","text":"","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.4.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofs/outline/#oss-hdfs-jindofs-430","text":"","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.3.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofs/outline/#oss-hdfs-jindofs-420","text":"","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.2.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofs/outline/#oss-hdfs-jindofs-410","text":"","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.1.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofs/outline/#oss-hdfs-jindofs-400","text":"","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.0.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofs/configuration/jindosdk_configuration_list/","text":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e \u00a7 SDK\u914d\u7f6e\u9879\u5217\u8868 \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u5ba2\u6237\u7aef\u5199\u5165\u65f6\u7684\u4e34\u65f6\u6587\u4ef6\u76ee\u5f55\uff0c\u53ef\u914d\u7f6e\u591a\u4e2a\uff08\u9017\u53f7\u9694\u5f00\uff09\uff0c\u4f1a\u8f6e\u6d41\u5199\u5165\uff0c\u591a\u7528\u6237\u73af\u5883\u9700\u914d\u7f6e\u53ef\u8bfb\u5199\u6743\u9650 --> <name>fs.oss.tmp.data.dirs</name> <value>/tmp/</value> </property> <property> <!-- \u4e34\u65f6\u6587\u4ef6\u81ea\u6e05\u7406\u670d\u52a1 --> <name>fs.oss.tmp.data.cleaner.enable</name> <value>true</value> </property> <property> <!-- \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u6b21\u6570 --> <name>fs.oss.retry.count</name> <value>5</value> </property> <property> <!-- \u8bf7\u6c42 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 --> <name>fs.oss.timeout.millisecond</name> <value>30000</value> </property> <property> <!-- \u8fde\u63a5 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 --> <name>fs.oss.connection.timeout.millisecond</name> <value>3000</value> </property> <property> <!-- \u5355\u4e2a\u6587\u4ef6 oss \u5e76\u53d1\u4e0a\u4f20\u7ebf\u7a0b\u6570 --> <name>fs.oss.upload.thread.concurrency</name> <value>5</value> </property> <property> <!-- oss \u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u961f\u5217\u5927\u5c0f --> <name>fs.oss.upload.queue.size</name> <value>5</value> </property> <property> <!-- \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u6570 --> <name>fs.oss.upload.max.pending.tasks.per.stream</name> <value>16</value> </property> <property> <!-- oss \u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u961f\u5217\u5927\u5c0f --> <name>fs.oss.download.queue.size</name> <value>5</value> </property> <property> <!-- \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u6570 --> <name>fs.oss.download.thread.concurrency</name> <value>16</value> </property> <property> <!-- \u9884\u8bfb oss \u7684 buffer \u5927\u5c0f --> <name>fs.oss.read.readahead.buffer.size</name> <value>1048576</value> </property> <property> <!-- \u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 --> <name>fs.oss.read.readahead.buffer.count</name> <value>4</value> </property> <property> <!--oss \u5237\u65b0\u7f13\u51b2\u533a\u95f4\u9694\uff08\u6beb\u79d2\uff09\uff0c\u5c0f\u4e8e 0 \u65f6\u4e0d\u751f\u6548 --> <name>fs.oss.flush.interval.millisecond</name> <value>-1</value> </property> <property> <!-- \u6587\u4ef6\u7ea7\u522b crc64 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u6253\u5f00 --> <name>fs.oss.checksum.crc64.enable</name> <value>true</value> </property> <property> <!--\u8bf7\u6c42\u7ea7\u522b md5 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u5173\u95ed --> <name>fs.oss.checksum.md5.enable</name> <value>false</value> </property> </configuration> \u3010\u6ce8\uff1a4.5\u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u65b9\u5f0f\u3011","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/jindofs/configuration/jindosdk_configuration_list/#jindosdk","text":"","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/jindofs/configuration/jindosdk_configuration_list/#sdk","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u5ba2\u6237\u7aef\u5199\u5165\u65f6\u7684\u4e34\u65f6\u6587\u4ef6\u76ee\u5f55\uff0c\u53ef\u914d\u7f6e\u591a\u4e2a\uff08\u9017\u53f7\u9694\u5f00\uff09\uff0c\u4f1a\u8f6e\u6d41\u5199\u5165\uff0c\u591a\u7528\u6237\u73af\u5883\u9700\u914d\u7f6e\u53ef\u8bfb\u5199\u6743\u9650 --> <name>fs.oss.tmp.data.dirs</name> <value>/tmp/</value> </property> <property> <!-- \u4e34\u65f6\u6587\u4ef6\u81ea\u6e05\u7406\u670d\u52a1 --> <name>fs.oss.tmp.data.cleaner.enable</name> <value>true</value> </property> <property> <!-- \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u6b21\u6570 --> <name>fs.oss.retry.count</name> <value>5</value> </property> <property> <!-- \u8bf7\u6c42 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 --> <name>fs.oss.timeout.millisecond</name> <value>30000</value> </property> <property> <!-- \u8fde\u63a5 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 --> <name>fs.oss.connection.timeout.millisecond</name> <value>3000</value> </property> <property> <!-- \u5355\u4e2a\u6587\u4ef6 oss \u5e76\u53d1\u4e0a\u4f20\u7ebf\u7a0b\u6570 --> <name>fs.oss.upload.thread.concurrency</name> <value>5</value> </property> <property> <!-- oss \u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u961f\u5217\u5927\u5c0f --> <name>fs.oss.upload.queue.size</name> <value>5</value> </property> <property> <!-- \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u6570 --> <name>fs.oss.upload.max.pending.tasks.per.stream</name> <value>16</value> </property> <property> <!-- oss \u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u961f\u5217\u5927\u5c0f --> <name>fs.oss.download.queue.size</name> <value>5</value> </property> <property> <!-- \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u6570 --> <name>fs.oss.download.thread.concurrency</name> <value>16</value> </property> <property> <!-- \u9884\u8bfb oss \u7684 buffer \u5927\u5c0f --> <name>fs.oss.read.readahead.buffer.size</name> <value>1048576</value> </property> <property> <!-- \u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 --> <name>fs.oss.read.readahead.buffer.count</name> <value>4</value> </property> <property> <!--oss \u5237\u65b0\u7f13\u51b2\u533a\u95f4\u9694\uff08\u6beb\u79d2\uff09\uff0c\u5c0f\u4e8e 0 \u65f6\u4e0d\u751f\u6548 --> <name>fs.oss.flush.interval.millisecond</name> <value>-1</value> </property> <property> <!-- \u6587\u4ef6\u7ea7\u522b crc64 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u6253\u5f00 --> <name>fs.oss.checksum.crc64.enable</name> <value>true</value> </property> <property> <!--\u8bf7\u6c42\u7ea7\u522b md5 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u5173\u95ed --> <name>fs.oss.checksum.md5.enable</name> <value>false</value> </property> </configuration> \u3010\u6ce8\uff1a4.5\u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u65b9\u5f0f\u3011","title":"SDK\u914d\u7f6e\u9879\u5217\u8868"},{"location":"backup/jindofs/configuration/jindosdk_configuration_list_ini/","text":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e \u00a7 SDK\u914d\u7f6e\u9879\u5217\u8868 \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 jindosdk.cfg \u4e2d\u3002 [jindosdk] # \u5ba2\u6237\u7aef\u5199\u5165\u65f6\u7684\u4e34\u65f6\u6587\u4ef6\u76ee\u5f55\uff0c\u53ef\u914d\u7f6e\u591a\u4e2a\uff08\u9017\u53f7\u9694\u5f00\uff09\uff0c\u4f1a\u8f6e\u6d41\u5199\u5165\uff0c\u591a\u7528\u6237\u73af\u5883\u9700\u914d\u7f6e\u53ef\u8bfb\u5199\u6743\u9650 fs.oss.tmp.data.dirs=/tmp/ # \u4e34\u65f6\u6587\u4ef6\u81ea\u6e05\u7406\u670d\u52a1 fs.oss.tmp.data.cleaner.enable=true # \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u6b21\u6570 fs.oss.retry.count=5 # \u8bf7\u6c42 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 fs.oss.timeout.millisecond=30000 # \u8fde\u63a5 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 fs.oss.connection.timeout.millisecond=3000 # \u5355\u4e2a\u6587\u4ef6 oss \u5e76\u53d1\u4e0a\u4f20\u7ebf\u7a0b\u6570 fs.oss.upload.thread.concurrency=5 # oss \u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u961f\u5217\u5927\u5c0f fs.oss.upload.queue.size=5 # \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u6570 fs.oss.upload.max.pending.tasks.per.stream=16 # oss \u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u961f\u5217\u5927\u5c0f fs.oss.download.queue.size=5 # \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u6570 fs.oss.download.thread.concurrency=16 # \u9884\u8bfb oss \u7684 buffer \u5927\u5c0f fs.oss.read.readahead.buffer.size=1048576 # \u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 fs.oss.read.readahead.buffer.count=4 # oss \u5237\u65b0\u7f13\u51b2\u533a\u95f4\u9694\uff08\u6beb\u79d2\uff09\uff0c\u5c0f\u4e8e 0 \u65f6\u4e0d\u751f\u6548 fs.oss.flush.interval.millisecond=-1 # \u6587\u4ef6\u7ea7\u522b crc64 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u6253\u5f00 fs.oss.checksum.crc64.enable=true # \u8bf7\u6c42\u7ea7\u522b md5 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u5173\u95ed fs.oss.checksum.md5.enable=false \u3010\u6ce8\uff1a4.5\u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u65b9\u5f0f\u3011","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/jindofs/configuration/jindosdk_configuration_list_ini/#jindosdk","text":"","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/jindofs/configuration/jindosdk_configuration_list_ini/#sdk","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 jindosdk.cfg \u4e2d\u3002 [jindosdk] # \u5ba2\u6237\u7aef\u5199\u5165\u65f6\u7684\u4e34\u65f6\u6587\u4ef6\u76ee\u5f55\uff0c\u53ef\u914d\u7f6e\u591a\u4e2a\uff08\u9017\u53f7\u9694\u5f00\uff09\uff0c\u4f1a\u8f6e\u6d41\u5199\u5165\uff0c\u591a\u7528\u6237\u73af\u5883\u9700\u914d\u7f6e\u53ef\u8bfb\u5199\u6743\u9650 fs.oss.tmp.data.dirs=/tmp/ # \u4e34\u65f6\u6587\u4ef6\u81ea\u6e05\u7406\u670d\u52a1 fs.oss.tmp.data.cleaner.enable=true # \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u6b21\u6570 fs.oss.retry.count=5 # \u8bf7\u6c42 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 fs.oss.timeout.millisecond=30000 # \u8fde\u63a5 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 fs.oss.connection.timeout.millisecond=3000 # \u5355\u4e2a\u6587\u4ef6 oss \u5e76\u53d1\u4e0a\u4f20\u7ebf\u7a0b\u6570 fs.oss.upload.thread.concurrency=5 # oss \u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u961f\u5217\u5927\u5c0f fs.oss.upload.queue.size=5 # \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u6570 fs.oss.upload.max.pending.tasks.per.stream=16 # oss \u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u961f\u5217\u5927\u5c0f fs.oss.download.queue.size=5 # \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u6570 fs.oss.download.thread.concurrency=16 # \u9884\u8bfb oss \u7684 buffer \u5927\u5c0f fs.oss.read.readahead.buffer.size=1048576 # \u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 fs.oss.read.readahead.buffer.count=4 # oss \u5237\u65b0\u7f13\u51b2\u533a\u95f4\u9694\uff08\u6beb\u79d2\uff09\uff0c\u5c0f\u4e8e 0 \u65f6\u4e0d\u751f\u6548 fs.oss.flush.interval.millisecond=-1 # \u6587\u4ef6\u7ea7\u522b crc64 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u6253\u5f00 fs.oss.checksum.crc64.enable=true # \u8bf7\u6c42\u7ea7\u522b md5 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u5173\u95ed fs.oss.checksum.md5.enable=false \u3010\u6ce8\uff1a4.5\u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u65b9\u5f0f\u3011","title":"SDK\u914d\u7f6e\u9879\u5217\u8868"},{"location":"backup/jindofs/configuration/jindosdk_endpoint_configuration/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Endpoint \u914d\u7f6e \u00a7 \u6709\u4e09\u79cd\u65b9\u5f0f\u53ef\u4ee5\u6307\u5b9a Endpoint\uff0cJindoSDK \u4f1a\u4ee5\u4e0b\u4f18\u5148\u7ea7\u67e5\u627e Endpoint\uff1a \u65b9\u5f0f\u4e00\uff1a \u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u6307\u5b9a Endpoint\u3002\uff08\u63a8\u8350\uff09 \u00a7 \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u4f7f\u7528\u8fd9\u79cd\u683c\u5f0f\u7684\u8bbf\u95ee\u8def\u5f84 oss://<Bucket>.<Endpoint>/<Object> , \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u63a5\u53e3\u3002 \u65b9\u5f0f\u4e8c\uff1a \u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint\u3002 \u00a7 \u5982\u679c\u4f7f\u7528 oss://<Bucket>/<Object> \u8fd9\u79cd\u8bbf\u95ee\u8def\u5f84\uff0c\u5373\u8bbf\u95ee\u8def\u5f84\u4e2d\u6ca1\u6709 Endpoint, JindoSDK \u5219\u4f1a\u5728\u914d\u7f6e\u4e2d\u67e5\u627e Bucket \u7ea7\u522b\u7684 Endpoint\u3002 \u53ef\u5728 Hadoop \u7684 core-site.xml \u4e2d\u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint \u6307\u5411 OSS-HDFS \u670d\u52a1\u7684 Endpoint\u3002 <configuration> <property> <name>fs.oss.bucket.XXX.endpoint</name> <value>cn-xxx.oss-dls.aliyuncs.com</value> </property> </configuration> \u8bf4\u660e: XXX \u4e3a OSS-HDFS \u670d\u52a1 Bucket \u540d\u79f0\u3002 \u65b9\u5f0f\u4e09\uff1a\u914d\u7f6e\u5168\u5c40\u9ed8\u8ba4 Endpoint\u3002 \u00a7 \u5982\u679c\u4f7f\u7528 oss://<Bucket>/<Object> \u8fd9\u79cd\u8bbf\u95ee\u8def\u5f84\u800c\u4e14\u6ca1\u6709\u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint\uff0c\u5219\u4f1a\u7528\u5168\u5c40 Endpoint \u8bbf\u95ee\uff0c\u5982\u679c\u8981\u9ed8\u8ba4\u4f7f\u7528\u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff0c\u5219\u53ef\u7528\u8be5\u65b9\u6cd5\u3002 \u53ef\u5728 Hadoop \u7684 core-site.xml \u4e2d\u914d\u7f6e\u5168\u5c40\u9ed8\u8ba4 Endpoint\u3002 <configuration> <property> <name>fs.oss.endpoint</name> <value>cn-xxx.oss-dls.aliyuncs.com</value> </property> </configuration>","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Endpoint \u914d\u7f6e"},{"location":"backup/jindofs/configuration/jindosdk_endpoint_configuration/#oss-hdfs-jindofs-endpoint","text":"\u6709\u4e09\u79cd\u65b9\u5f0f\u53ef\u4ee5\u6307\u5b9a Endpoint\uff0cJindoSDK \u4f1a\u4ee5\u4e0b\u4f18\u5148\u7ea7\u67e5\u627e Endpoint\uff1a","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Endpoint \u914d\u7f6e"},{"location":"backup/jindofs/configuration/jindosdk_endpoint_configuration/#endpoint","text":"\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u4f7f\u7528\u8fd9\u79cd\u683c\u5f0f\u7684\u8bbf\u95ee\u8def\u5f84 oss://<Bucket>.<Endpoint>/<Object> , \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u63a5\u53e3\u3002","title":"\u65b9\u5f0f\u4e00\uff1a \u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u6307\u5b9a Endpoint\u3002\uff08\u63a8\u8350\uff09"},{"location":"backup/jindofs/configuration/jindosdk_endpoint_configuration/#bucket-endpoint","text":"\u5982\u679c\u4f7f\u7528 oss://<Bucket>/<Object> \u8fd9\u79cd\u8bbf\u95ee\u8def\u5f84\uff0c\u5373\u8bbf\u95ee\u8def\u5f84\u4e2d\u6ca1\u6709 Endpoint, JindoSDK \u5219\u4f1a\u5728\u914d\u7f6e\u4e2d\u67e5\u627e Bucket \u7ea7\u522b\u7684 Endpoint\u3002 \u53ef\u5728 Hadoop \u7684 core-site.xml \u4e2d\u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint \u6307\u5411 OSS-HDFS \u670d\u52a1\u7684 Endpoint\u3002 <configuration> <property> <name>fs.oss.bucket.XXX.endpoint</name> <value>cn-xxx.oss-dls.aliyuncs.com</value> </property> </configuration> \u8bf4\u660e: XXX \u4e3a OSS-HDFS \u670d\u52a1 Bucket \u540d\u79f0\u3002","title":"\u65b9\u5f0f\u4e8c\uff1a \u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint\u3002"},{"location":"backup/jindofs/configuration/jindosdk_endpoint_configuration/#endpoint_1","text":"\u5982\u679c\u4f7f\u7528 oss://<Bucket>/<Object> \u8fd9\u79cd\u8bbf\u95ee\u8def\u5f84\u800c\u4e14\u6ca1\u6709\u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint\uff0c\u5219\u4f1a\u7528\u5168\u5c40 Endpoint \u8bbf\u95ee\uff0c\u5982\u679c\u8981\u9ed8\u8ba4\u4f7f\u7528\u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff0c\u5219\u53ef\u7528\u8be5\u65b9\u6cd5\u3002 \u53ef\u5728 Hadoop \u7684 core-site.xml \u4e2d\u914d\u7f6e\u5168\u5c40\u9ed8\u8ba4 Endpoint\u3002 <configuration> <property> <name>fs.oss.endpoint</name> <value>cn-xxx.oss-dls.aliyuncs.com</value> </property> </configuration>","title":"\u65b9\u5f0f\u4e09\uff1a\u914d\u7f6e\u5168\u5c40\u9ed8\u8ba4 Endpoint\u3002"},{"location":"backup/jindofs/flink/jindosdk_on_flink/","text":"Flink \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 \u5728\u96c6\u7fa4\u4e0a\u6709\u5f00\u6e90\u7248\u672c Flink \u8f6f\u4ef6\uff0c\u7248\u672c\u4e0d\u4f4e\u4e8e 1.10.1\u3002 \u6ce8\uff1aFlink 1.16.0 \u53ca\u66f4\u9ad8\u7248\u672c\u7684\u53ef\u7528\u6027\u5c1a\u672a\u5f97\u5230\u9a8c\u8bc1\u3002 \u4e3a\u4ec0\u4e48 Flink \u9700\u8981\u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u00a7 \u76ee\u524d\uff0c\u5f00\u6e90\u7248\u672c Flink \u5bf9\u6d41\u5f0f\u5199\u5165 OSS-HDFS \u670d\u52a1\u5e76\u4e0d\u652f\u6301\uff0c\u66f4\u4e0d\u80fd\u652f\u6301 \u201c\u4e25\u683c\u4e00\u6b21\u201d\uff08EXACTLY_ONCE\uff09\u8bed\u4e49\uff0c\u5982\u6709\u8be5\u9700\u6c42\u5219\u9700\u8981\u4f7f\u7528 JindoSDK\u3002 \u6ce8\uff1aApache Flink \u81ea 1.16 \u7248\u672c\u8d77\u539f\u751f\u63d0\u4f9b\u4e86\u6d41\u5f0f\u5199\u5165 Aliyun OSS \u5bf9\u8c61\u5b58\u50a8\u7684\u652f\u6301\uff0c\u4f46\u4e0d\u80fd\u652f\u6301\u5199\u5165 OSS-HDFS \u670d\u52a1\u3002\u82e5\u914d\u7f6e\u8fc7 Apache Flink \u7684\u81ea\u6709\u5b9e\u73b0\uff0c\u9700\u79fb\u9664\uff0c\u89c1\u4e0b\u6587\u3002 JindoSDK \u914d\u7f6e \u00a7 \u9700\u8981\u5728\u6240\u6709 Flink \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flink \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e\u4e0b\u5217 .jar \u6587\u4ef6\uff1a * jindo-flink-${version}-full.jar \u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 plugins/flink/ \u76ee\u5f55\u4e0b\u627e\u5230\u3002 \u53e6\u5916\uff0c\u5982\u679c\u96c6\u7fa4\u5185\u5b58\u5728 Apache Flink \u63d0\u4f9b\u7684\u81ea\u5e26 Flink OSS Connector\uff0c\u9700\u79fb\u9664\u3002\u79fb\u9664\u65b9\u6cd5\u4e3a\uff0c\u4ece Flink \u96c6\u7fa4\u7684 lib \u76ee\u5f55\u6216 plugins/oss-fs-hadoop \u79fb\u8d70\u5982\u4e0b jar \u5305\uff1a * flink-oss-fs-hadoop-${flink-version}.jar \u6ce8\uff1aJindoSDK \u76ee\u524d\u63d0\u4f9b\u591a\u5e73\u53f0\u652f\u6301\uff0c\u5f53\u524d\u652f\u6301\u7684\u5e73\u53f0\u4ee5\u53ca\u9700\u8981\u989d\u5916\u653e\u7f6e\u7684\u8d44\u6e90\uff0c\u53c2\u8003 JindoData 4.6.4 \u7248\u672c\u8bf4\u660e \u5982\u4f55\u4f7f\u7528 \u00a7 \u914d\u7f6e\u597d SDK \u540e\uff0c\u65e0\u9700\u989d\u5916\u914d\u7f6e\uff0c\u4ee5\u5e38\u89c4 Flink \u6d41\u5f0f\u4f5c\u4e1a\u65b9\u6cd5\u4f7f\u7528\u5373\u53ef\uff0c\u6ce8\u610f\u4f7f\u7528\u6b63\u786e\u7684\u8def\u5f84\u3002\u5199\u5165 OSS-HDFS \u670d\u52a1\u987b\u4ee5 oss:// \u4e3a\u524d\u7f00\u3002JindoFS \u670d\u52a1\u4e0e\u963f\u91cc\u4e91 OSS \u4f7f\u7528\u76f8\u540c\u7684\u524d\u7f00\uff0cJindoSDK \u5185\u4f1a\u81ea\u52a8\u8fdb\u884c\u8bc6\u522b\u3002\u4e8c\u8005\u7684\u8054\u7cfb\uff0c\u53c2\u8003 JindoFS \u670d\u52a1\u4ecb\u7ecd \u3002 \u793a\u4f8b \u00a7 \u4e0b\u9762\u662f\u4e00\u4e2a\u7b80\u5355\u7684\u793a\u4f8b\u7a0b\u5e8f\uff0c\u5982\u679c\u5df2\u7ecf\u6309\u7167\u4e0a\u4e00\u8282\u914d\u7f6e\u597d SDK\uff0c\u5219\u4e0b\u5217\u7a0b\u5e8f\u5e94\u80fd\u6b63\u786e\u8fd0\u884c\uff1a (1) \u901a\u7528\u914d\u7f6e \u6253\u5f00 Flink \u7684 \u201c\u68c0\u67e5\u70b9\u201d\uff08Checkpoint\uff09\u3002\u4f8b\u5982\uff0cenv \u662f\u4e00\u4e2a\u5df2\u7ecf\u5efa\u7acb\u7684 StreamExecutionEnvironment\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u65b9\u5f0f\u5efa\u7acb\uff1a StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); \u5219\u53ef\u4ee5\u8fd9\u6837\u542f\u7528 checkpoint\uff1a env.enableCheckpointing(<userDefinedCheckpointInterval>, CheckpointingMode.EXACTLY_ONCE); \u4f7f\u7528\u53ef\u91cd\u53d1\u7684\u6570\u636e\u6e90\uff0c\u4f8b\u5982 Kafka (2) \u793a\u4f8b\u7a0b\u5e8f \u4e0b\u6587\u4e2d\uff0coutputStream \u662f\u4e00\u4e2a\u9884\u5148\u5f62\u6210\u7684 DataStream<String> \u5bf9\u8c61\uff0c\u5e76\u671f\u671b\u5c06\u5176\u5199\u5165 OSS\uff0c\u90a3\u4e48\u53ef\u4ee5\u8fd9\u6837\u6dfb\u52a0 sink\uff1a String outputPath = \"oss://<user-defined-bucket.dls-endpoint>/<user-defined-dir>\" StreamingFileSink<String> sink = StreamingFileSink.forRowFormat( new Path(outputPath), new SimpleStringEncoder<String>(\"UTF-8\") ).build(); outputStream.addSink(sink); \u6700\u540e\u7528 env.execute() \u6267\u884c Flink \u4f5c\u4e1a\u5373\u53ef\u3002 \u9ad8\u7ea7\u4f7f\u7528\uff1a\u81ea\u5b9a\u4e49\u53c2\u6570 \u00a7 \u7528\u6237\u5728\u63d0\u4ea4 Flink \u4f5c\u4e1a\u65f6\uff0c\u53ef\u4ee5\u81ea\u5b9a\u4e49\u914d\u7f6e\u4e00\u4e9b\u53c2\u6570\uff0c\u4ee5\u5f00\u542f\u6216\u63a7\u5236\u7279\u5b9a\u529f\u80fd\u3002\u4f8b\u5982\uff0c\u4ee5 yarn-cluster \u6a21\u5f0f\u63d0\u4ea4\u65f6\uff0c\u53ef\u4ee5\u901a\u8fc7 -yD \u8fdb\u884c\u914d\u7f6e\uff0c\u65b9\u5f0f\u7c7b\u4f3c\u4e8e\uff1a <flink_home>/bin/flink run -m yarn-cluster -yD key1=value1 -yD key2=value2 ... \u76ee\u524d\u652f\u6301\u901a\u8fc7\u914d\u7f6e\u5f00\u542f \u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\u3002 \u71b5\u6ce8\u5165 \u00a7 \u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\u7684\u76ee\u7684\u662f\uff0c\u5c06\u5199\u5165\u8def\u5f84\u7684\u4e00\u6bb5\u7279\u5b9a\u5b57\u7b26\u4e32\u5339\u914d\u51fa\u6765\uff0c\u7528\u4e00\u6bb5\u968f\u673a\u7684\u5b57\u7b26\u4e32\u8fdb\u884c\u66ff\u6362\uff0c\u4ee5\u524a\u5f31\u6240\u8c13 \u201c\u7247\u533a\u201d (sharding) \u6548\u5e94\uff0c\u63d0\u9ad8\u5199\u5165\u6548\u7387\u3002\u914d\u7f6e\u53c2\u6570\uff1a oss.entropy.key=<user-defined-key> oss.entropy.length=<user-defined-length> \u5982\u6b64\uff0c\u6bcf\u6b21\u6253\u5f00\u5199\u4e00\u4e2a\u65b0\u6587\u4ef6\u65f6\uff0c\u8def\u5f84\u4e2d\u4e0e <user-defined-key> \u76f8\u540c\u7684\u5b57\u7b26\u4e32\u4f1a\u88ab\u66ff\u6362\u4e3a\u4e00\u4e2a\u968f\u673a\u5b57\u7b26\u4e32\uff0c\u968f\u673a\u4e32\u7684\u957f\u5ea6\u4e3a <user-defined-length> \u3002\u914d\u7f6e\u8be5\u53c2\u6570\u65f6\uff0c <user-defined-length> \u5fc5\u987b\u5927\u4e8e\u96f6\u3002","title":"Flink \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofs/flink/jindosdk_on_flink/#flink-jindosdk-oss-hdfs-jindofs","text":"","title":"Flink \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofs/flink/jindosdk_on_flink/#_1","text":"\u5728\u96c6\u7fa4\u4e0a\u6709\u5f00\u6e90\u7248\u672c Flink \u8f6f\u4ef6\uff0c\u7248\u672c\u4e0d\u4f4e\u4e8e 1.10.1\u3002 \u6ce8\uff1aFlink 1.16.0 \u53ca\u66f4\u9ad8\u7248\u672c\u7684\u53ef\u7528\u6027\u5c1a\u672a\u5f97\u5230\u9a8c\u8bc1\u3002","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/jindofs/flink/jindosdk_on_flink/#flink-jindosdk-oss-hdfs","text":"\u76ee\u524d\uff0c\u5f00\u6e90\u7248\u672c Flink \u5bf9\u6d41\u5f0f\u5199\u5165 OSS-HDFS \u670d\u52a1\u5e76\u4e0d\u652f\u6301\uff0c\u66f4\u4e0d\u80fd\u652f\u6301 \u201c\u4e25\u683c\u4e00\u6b21\u201d\uff08EXACTLY_ONCE\uff09\u8bed\u4e49\uff0c\u5982\u6709\u8be5\u9700\u6c42\u5219\u9700\u8981\u4f7f\u7528 JindoSDK\u3002 \u6ce8\uff1aApache Flink \u81ea 1.16 \u7248\u672c\u8d77\u539f\u751f\u63d0\u4f9b\u4e86\u6d41\u5f0f\u5199\u5165 Aliyun OSS \u5bf9\u8c61\u5b58\u50a8\u7684\u652f\u6301\uff0c\u4f46\u4e0d\u80fd\u652f\u6301\u5199\u5165 OSS-HDFS \u670d\u52a1\u3002\u82e5\u914d\u7f6e\u8fc7 Apache Flink \u7684\u81ea\u6709\u5b9e\u73b0\uff0c\u9700\u79fb\u9664\uff0c\u89c1\u4e0b\u6587\u3002","title":"\u4e3a\u4ec0\u4e48 Flink \u9700\u8981\u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS-HDFS \u670d\u52a1"},{"location":"backup/jindofs/flink/jindosdk_on_flink/#jindosdk","text":"\u9700\u8981\u5728\u6240\u6709 Flink \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flink \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e\u4e0b\u5217 .jar \u6587\u4ef6\uff1a * jindo-flink-${version}-full.jar \u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 plugins/flink/ \u76ee\u5f55\u4e0b\u627e\u5230\u3002 \u53e6\u5916\uff0c\u5982\u679c\u96c6\u7fa4\u5185\u5b58\u5728 Apache Flink \u63d0\u4f9b\u7684\u81ea\u5e26 Flink OSS Connector\uff0c\u9700\u79fb\u9664\u3002\u79fb\u9664\u65b9\u6cd5\u4e3a\uff0c\u4ece Flink \u96c6\u7fa4\u7684 lib \u76ee\u5f55\u6216 plugins/oss-fs-hadoop \u79fb\u8d70\u5982\u4e0b jar \u5305\uff1a * flink-oss-fs-hadoop-${flink-version}.jar \u6ce8\uff1aJindoSDK \u76ee\u524d\u63d0\u4f9b\u591a\u5e73\u53f0\u652f\u6301\uff0c\u5f53\u524d\u652f\u6301\u7684\u5e73\u53f0\u4ee5\u53ca\u9700\u8981\u989d\u5916\u653e\u7f6e\u7684\u8d44\u6e90\uff0c\u53c2\u8003 JindoData 4.6.4 \u7248\u672c\u8bf4\u660e","title":"JindoSDK \u914d\u7f6e"},{"location":"backup/jindofs/flink/jindosdk_on_flink/#_2","text":"\u914d\u7f6e\u597d SDK \u540e\uff0c\u65e0\u9700\u989d\u5916\u914d\u7f6e\uff0c\u4ee5\u5e38\u89c4 Flink \u6d41\u5f0f\u4f5c\u4e1a\u65b9\u6cd5\u4f7f\u7528\u5373\u53ef\uff0c\u6ce8\u610f\u4f7f\u7528\u6b63\u786e\u7684\u8def\u5f84\u3002\u5199\u5165 OSS-HDFS \u670d\u52a1\u987b\u4ee5 oss:// \u4e3a\u524d\u7f00\u3002JindoFS \u670d\u52a1\u4e0e\u963f\u91cc\u4e91 OSS \u4f7f\u7528\u76f8\u540c\u7684\u524d\u7f00\uff0cJindoSDK \u5185\u4f1a\u81ea\u52a8\u8fdb\u884c\u8bc6\u522b\u3002\u4e8c\u8005\u7684\u8054\u7cfb\uff0c\u53c2\u8003 JindoFS \u670d\u52a1\u4ecb\u7ecd \u3002","title":"\u5982\u4f55\u4f7f\u7528"},{"location":"backup/jindofs/flink/jindosdk_on_flink/#_3","text":"\u4e0b\u9762\u662f\u4e00\u4e2a\u7b80\u5355\u7684\u793a\u4f8b\u7a0b\u5e8f\uff0c\u5982\u679c\u5df2\u7ecf\u6309\u7167\u4e0a\u4e00\u8282\u914d\u7f6e\u597d SDK\uff0c\u5219\u4e0b\u5217\u7a0b\u5e8f\u5e94\u80fd\u6b63\u786e\u8fd0\u884c\uff1a (1) \u901a\u7528\u914d\u7f6e \u6253\u5f00 Flink \u7684 \u201c\u68c0\u67e5\u70b9\u201d\uff08Checkpoint\uff09\u3002\u4f8b\u5982\uff0cenv \u662f\u4e00\u4e2a\u5df2\u7ecf\u5efa\u7acb\u7684 StreamExecutionEnvironment\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u65b9\u5f0f\u5efa\u7acb\uff1a StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); \u5219\u53ef\u4ee5\u8fd9\u6837\u542f\u7528 checkpoint\uff1a env.enableCheckpointing(<userDefinedCheckpointInterval>, CheckpointingMode.EXACTLY_ONCE); \u4f7f\u7528\u53ef\u91cd\u53d1\u7684\u6570\u636e\u6e90\uff0c\u4f8b\u5982 Kafka (2) \u793a\u4f8b\u7a0b\u5e8f \u4e0b\u6587\u4e2d\uff0coutputStream \u662f\u4e00\u4e2a\u9884\u5148\u5f62\u6210\u7684 DataStream<String> \u5bf9\u8c61\uff0c\u5e76\u671f\u671b\u5c06\u5176\u5199\u5165 OSS\uff0c\u90a3\u4e48\u53ef\u4ee5\u8fd9\u6837\u6dfb\u52a0 sink\uff1a String outputPath = \"oss://<user-defined-bucket.dls-endpoint>/<user-defined-dir>\" StreamingFileSink<String> sink = StreamingFileSink.forRowFormat( new Path(outputPath), new SimpleStringEncoder<String>(\"UTF-8\") ).build(); outputStream.addSink(sink); \u6700\u540e\u7528 env.execute() \u6267\u884c Flink \u4f5c\u4e1a\u5373\u53ef\u3002","title":"\u793a\u4f8b"},{"location":"backup/jindofs/flink/jindosdk_on_flink/#_4","text":"\u7528\u6237\u5728\u63d0\u4ea4 Flink \u4f5c\u4e1a\u65f6\uff0c\u53ef\u4ee5\u81ea\u5b9a\u4e49\u914d\u7f6e\u4e00\u4e9b\u53c2\u6570\uff0c\u4ee5\u5f00\u542f\u6216\u63a7\u5236\u7279\u5b9a\u529f\u80fd\u3002\u4f8b\u5982\uff0c\u4ee5 yarn-cluster \u6a21\u5f0f\u63d0\u4ea4\u65f6\uff0c\u53ef\u4ee5\u901a\u8fc7 -yD \u8fdb\u884c\u914d\u7f6e\uff0c\u65b9\u5f0f\u7c7b\u4f3c\u4e8e\uff1a <flink_home>/bin/flink run -m yarn-cluster -yD key1=value1 -yD key2=value2 ... \u76ee\u524d\u652f\u6301\u901a\u8fc7\u914d\u7f6e\u5f00\u542f \u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\u3002","title":"\u9ad8\u7ea7\u4f7f\u7528\uff1a\u81ea\u5b9a\u4e49\u53c2\u6570"},{"location":"backup/jindofs/flink/jindosdk_on_flink/#_5","text":"\u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\u7684\u76ee\u7684\u662f\uff0c\u5c06\u5199\u5165\u8def\u5f84\u7684\u4e00\u6bb5\u7279\u5b9a\u5b57\u7b26\u4e32\u5339\u914d\u51fa\u6765\uff0c\u7528\u4e00\u6bb5\u968f\u673a\u7684\u5b57\u7b26\u4e32\u8fdb\u884c\u66ff\u6362\uff0c\u4ee5\u524a\u5f31\u6240\u8c13 \u201c\u7247\u533a\u201d (sharding) \u6548\u5e94\uff0c\u63d0\u9ad8\u5199\u5165\u6548\u7387\u3002\u914d\u7f6e\u53c2\u6570\uff1a oss.entropy.key=<user-defined-key> oss.entropy.length=<user-defined-length> \u5982\u6b64\uff0c\u6bcf\u6b21\u6253\u5f00\u5199\u4e00\u4e2a\u65b0\u6587\u4ef6\u65f6\uff0c\u8def\u5f84\u4e2d\u4e0e <user-defined-key> \u76f8\u540c\u7684\u5b57\u7b26\u4e32\u4f1a\u88ab\u66ff\u6362\u4e3a\u4e00\u4e2a\u968f\u673a\u5b57\u7b26\u4e32\uff0c\u968f\u673a\u4e32\u7684\u957f\u5ea6\u4e3a <user-defined-length> \u3002\u914d\u7f6e\u8be5\u53c2\u6570\u65f6\uff0c <user-defined-length> \u5fc5\u987b\u5927\u4e8e\u96f6\u3002","title":"\u71b5\u6ce8\u5165"},{"location":"backup/jindofs/flume/jindosdk_on_flume/","text":"Flume \u4f7f\u7528 JindoSDK \u5199\u5165\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 \u5728\u96c6\u7fa4\u4e0a\u5df2\u7ecf\u90e8\u7f72 Flume\uff0c\u5df2\u90e8\u7f72 JindoSDK 4.0 \u4ee5\u4e0a\u7248\u672c\u3002 \u4e3a\u4ec0\u4e48 Flume \u9700\u8981\u4f7f\u7528 JindoSDK \u5199\u5165 OSS-HDFS \u670d\u52a1 \u00a7 Flume \u901a\u8fc7 flush() \u8c03\u7528\u4fdd\u8bc1\u4e8b\u52a1\u6027\u5199\u5165\uff0c\u901a\u8fc7 JindoSDK \u5199\u5165 OSS-HDFS \u670d\u52a1\uff0c\u53ef\u4ee5\u8ba9 flush \u540e\u7684\u6570\u636e\u7acb\u523b\u53ef\u89c1\uff0c\u4fdd\u8bc1\u6570\u636e\u4e0d\u4e22\u5931\u3002 SDK \u914d\u7f6e \u00a7 \u9700\u8981\u5728 Flume \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flume \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e JindoSDK\u3002 \u4e0b\u8f7d\u5e76\u89e3\u538b\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 ) cp jindosdk-x.x.x/lib/*.jar $FLUME_HOME/jars/ Sink \u914d\u7f6e\u793a\u4f8b \u00a7 # \u914d\u7f6e OSS Sink xxx.sinks.oss_sink.hdfs.path = oss://${your_bucket}/flume_dir/%Y-%m-%d/%H # Sink\u53c2\u6570\uff0cbatchSize \u9700\u8981\u8bbe\u7f6e\u5927\u4e00\u4e9b\uff0c\u63a8\u8350\u6bcf\u6b21 Flush \u7684\u91cf\u5728 32MB\u4ee5\u4e0a\uff0c\u5426\u5219\u4f1a\u5f71\u54cd\u6027\u80fd xxx.sinks.oss_sink.hdfs.batchSize = 100000 ... xxx.sinks.oss_sink.hdfs.round = true xxx.sinks.oss_sink.hdfs.roundValue = 15 xxx.sinks.oss_sink.hdfs.Unit = minute xxx.sinks.oss_sink.hdfs.filePrefix = your_topic xxx.sinks.oss_sink.rollSize = 3600 xxx.sinks.oss_sink.threadsPoolSize = 30 ...","title":"Flume \u4f7f\u7528 JindoSDK \u5199\u5165\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/flume/jindosdk_on_flume/#flume-jindosdk-oss-hdfs-jindofs","text":"","title":"Flume \u4f7f\u7528 JindoSDK \u5199\u5165\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/flume/jindosdk_on_flume/#_1","text":"\u5728\u96c6\u7fa4\u4e0a\u5df2\u7ecf\u90e8\u7f72 Flume\uff0c\u5df2\u90e8\u7f72 JindoSDK 4.0 \u4ee5\u4e0a\u7248\u672c\u3002","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/jindofs/flume/jindosdk_on_flume/#flume-jindosdk-oss-hdfs","text":"Flume \u901a\u8fc7 flush() \u8c03\u7528\u4fdd\u8bc1\u4e8b\u52a1\u6027\u5199\u5165\uff0c\u901a\u8fc7 JindoSDK \u5199\u5165 OSS-HDFS \u670d\u52a1\uff0c\u53ef\u4ee5\u8ba9 flush \u540e\u7684\u6570\u636e\u7acb\u523b\u53ef\u89c1\uff0c\u4fdd\u8bc1\u6570\u636e\u4e0d\u4e22\u5931\u3002","title":"\u4e3a\u4ec0\u4e48 Flume \u9700\u8981\u4f7f\u7528 JindoSDK \u5199\u5165 OSS-HDFS \u670d\u52a1"},{"location":"backup/jindofs/flume/jindosdk_on_flume/#sdk","text":"\u9700\u8981\u5728 Flume \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flume \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e JindoSDK\u3002 \u4e0b\u8f7d\u5e76\u89e3\u538b\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 ) cp jindosdk-x.x.x/lib/*.jar $FLUME_HOME/jars/","title":"SDK \u914d\u7f6e"},{"location":"backup/jindofs/flume/jindosdk_on_flume/#sink","text":"# \u914d\u7f6e OSS Sink xxx.sinks.oss_sink.hdfs.path = oss://${your_bucket}/flume_dir/%Y-%m-%d/%H # Sink\u53c2\u6570\uff0cbatchSize \u9700\u8981\u8bbe\u7f6e\u5927\u4e00\u4e9b\uff0c\u63a8\u8350\u6bcf\u6b21 Flush \u7684\u91cf\u5728 32MB\u4ee5\u4e0a\uff0c\u5426\u5219\u4f1a\u5f71\u54cd\u6027\u80fd xxx.sinks.oss_sink.hdfs.batchSize = 100000 ... xxx.sinks.oss_sink.hdfs.round = true xxx.sinks.oss_sink.hdfs.roundValue = 15 xxx.sinks.oss_sink.hdfs.Unit = minute xxx.sinks.oss_sink.hdfs.filePrefix = your_topic xxx.sinks.oss_sink.rollSize = 3600 xxx.sinks.oss_sink.threadsPoolSize = 30 ...","title":"Sink \u914d\u7f6e\u793a\u4f8b"},{"location":"backup/jindofs/hadoop/jindosdk_ide_hadoop/","text":"Hadoop \u4f7f\u7528 JindoSDK \u96c6\u6210\u5f00\u53d1 \u00a7 \u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindosdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.net.URI; public class TestJindoSDK { public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. FileSystem fs = FileSystem.get(URI.create(\"oss://<Bucket>.<HDFS_Endpoint>/\"), conf); FSDataInputStream in = fs.open(new Path(\"/uttest/file1\")); in.read(); in.close(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency>","title":"Hadoop \u4f7f\u7528 JindoSDK \u96c6\u6210\u5f00\u53d1"},{"location":"backup/jindofs/hadoop/jindosdk_ide_hadoop/#hadoop-jindosdk","text":"\u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindosdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.net.URI; public class TestJindoSDK { public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. FileSystem fs = FileSystem.get(URI.create(\"oss://<Bucket>.<HDFS_Endpoint>/\"), conf); FSDataInputStream in = fs.open(new Path(\"/uttest/file1\")); in.read(); in.close(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency>","title":"Hadoop \u4f7f\u7528 JindoSDK \u96c6\u6210\u5f00\u53d1"},{"location":"backup/jindofs/hadoop/jindosdk_on_hadoop/","text":"Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 JindoSDK \u4e3a Apache Hadoop\u7684\u8ba1\u7b97\u5206\u6790\u5e94\u7528\uff08\u4f8b\u5982MapReduce\u3001Hive\u3001Spark\u3001Flink\u7b49\uff09\u63d0\u4f9b\u4e86\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u529f\u80fd\u3002 \u6b65\u9aa4 \u00a7 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u5c06 JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002 4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u00a7 \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 JindoFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 5. \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u00a7 \u7528 Hadoop Shell \u8bbf\u95ee JindoFS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<Bucket>.<Endpoint>/ ls \u64cd\u4f5c hadoop fs -ls oss://<Bucket>.<Endpoint>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<Bucket>.<Endpoint>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<Bucket>.<Endpoint>/<path> 6. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/hadoop/jindosdk_on_hadoop/#hadoop-jindosdk-oss-hdfs-jindofs","text":"JindoSDK \u4e3a Apache Hadoop\u7684\u8ba1\u7b97\u5206\u6790\u5e94\u7528\uff08\u4f8b\u5982MapReduce\u3001Hive\u3001Spark\u3001Flink\u7b49\uff09\u63d0\u4f9b\u4e86\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u529f\u80fd\u3002","title":"Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/hadoop/jindosdk_on_hadoop/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofs/hadoop/jindosdk_on_hadoop/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindofs/hadoop/jindosdk_on_hadoop/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofs/hadoop/jindosdk_on_hadoop/#3-oss-hdfs-access-key","text":"\u5c06 JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002","title":"3. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"backup/jindofs/hadoop/jindosdk_on_hadoop/#4-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 JindoFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002","title":"4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint"},{"location":"backup/jindofs/hadoop/jindosdk_on_hadoop/#5-jindosdk-oss-hdfs","text":"\u7528 Hadoop Shell \u8bbf\u95ee JindoFS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<Bucket>.<Endpoint>/ ls \u64cd\u4f5c hadoop fs -ls oss://<Bucket>.<Endpoint>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<Bucket>.<Endpoint>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<Bucket>.<Endpoint>/<path>","title":"5. \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS-HDFS \u670d\u52a1"},{"location":"backup/jindofs/hadoop/jindosdk_on_hadoop/#6","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"6. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofs/hbase/jindosdk_on_hbase/","text":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 \u00a7 HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002 OSS-HDFS \u670d\u52a1\u662f\u963f\u91cc\u4e91\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9 HDFS \u63a5\u53e3, JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\u53ca WAL \u6587\u4ef6\u7684\u5b58\u50a8\u5b9e\u73b0\u5b58\u7b97\u5206\u79bb\uff0c\u76f8\u5bf9\u4e8e\u672c\u5730HDFS\u5b58\u50a8\uff0c\u4f7f\u7528\u66f4\u52a0\u7075\u6d3b\uff0c\u51cf\u5c11\u8fd0\u7ef4\u6210\u672c\u3002 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06\u5df2\u5f00\u542f OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u00a7 \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 JindoFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 5. \u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84 \u00a7 \u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u548c hbase.wal.dir \u7684\u503c\u4e3a OSS-HDFS \u670d\u52a1\u5730\u5740\uff0c\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84\u548c WAL \u6587\u4ef6\u7684\u5b58\u50a8\u8def\u5f84\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 OSS-HDFS \u670d\u52a1\u3002\u53c2\u6570\u503c\u4e3a oss://bucket.endpoint/hbase-root-dir hbase.wal.dir \u6307\u5b9a HBase \u7684 WAL \u5b58\u50a8\u76ee\u5f55\u5230 OSS-HDFS \u670d\u52a1\u3002 \u53c2\u6570\u503c\u4e3a oss://bucket.endpoint/hbase","title":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8"},{"location":"backup/jindofs/hbase/jindosdk_on_hbase/#hbase-oss-hdfs-jindofs","text":"HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002 OSS-HDFS \u670d\u52a1\u662f\u963f\u91cc\u4e91\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9 HDFS \u63a5\u53e3, JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\u53ca WAL \u6587\u4ef6\u7684\u5b58\u50a8\u5b9e\u73b0\u5b58\u7b97\u5206\u79bb\uff0c\u76f8\u5bf9\u4e8e\u672c\u5730HDFS\u5b58\u50a8\uff0c\u4f7f\u7528\u66f4\u52a0\u7075\u6d3b\uff0c\u51cf\u5c11\u8fd0\u7ef4\u6210\u672c\u3002","title":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8"},{"location":"backup/jindofs/hbase/jindosdk_on_hbase/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindofs/hbase/jindosdk_on_hbase/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofs/hbase/jindosdk_on_hbase/#3-oss-hdfs-access-key","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06\u5df2\u5f00\u542f OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002","title":"3. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"backup/jindofs/hbase/jindosdk_on_hbase/#4-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 JindoFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002","title":"4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint"},{"location":"backup/jindofs/hbase/jindosdk_on_hbase/#5-hbase","text":"\u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u548c hbase.wal.dir \u7684\u503c\u4e3a OSS-HDFS \u670d\u52a1\u5730\u5740\uff0c\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84\u548c WAL \u6587\u4ef6\u7684\u5b58\u50a8\u8def\u5f84\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 OSS-HDFS \u670d\u52a1\u3002\u53c2\u6570\u503c\u4e3a oss://bucket.endpoint/hbase-root-dir hbase.wal.dir \u6307\u5b9a HBase \u7684 WAL \u5b58\u50a8\u76ee\u5f55\u5230 OSS-HDFS \u670d\u52a1\u3002 \u53c2\u6570\u503c\u4e3a oss://bucket.endpoint/hbase","title":"5. \u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84"},{"location":"backup/jindofs/hive/jindosdk_on_hive/","text":"Hive \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e \u00a7 Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002 \u5b89\u88c5\u914d\u7f6e \u00a7 1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/ 2. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u914d\u7f6e JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b \u5c06 JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key \u5c06\u5df2\u5f00\u542f OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 3. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u00a7 \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u6700\u540e\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 4. Hive on MR \u00a7 \u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002 5. Hive on Tez \u00a7 \u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002 6. Hive on Spark \u00a7 \u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002 OSS-HDFS \u670d\u52a1\u7528\u4e8e\u8868\u7684\u5b58\u50a8 \u00a7 \u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS-HDFS \u670d\u52a1\u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'oss://bucket_name.endpoint_name/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'oss://bucket_name.endpoint_name/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS-HDFS \u670d\u52a1\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>oss://bucket_name.endpoint_name/path/to/warehouse</value> </property> </configuration> \u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a \u00a7 ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'oss://bucket_name.endpoint_name/path/to/us/part210301cn'; \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Hive \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#hive-jindosdk-oss-hdfs-jindofs","text":"Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002","title":"Hive \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#_1","text":"","title":"\u5b89\u88c5\u914d\u7f6e"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#1-hive-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/","title":"1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#2-oss-hdfs-access-key","text":"\u914d\u7f6e JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b \u5c06 JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key \u5c06\u5df2\u5f00\u542f OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002","title":"2. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#3-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u6700\u540e\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002","title":"3. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#4-hive-on-mr","text":"\u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002","title":"4. Hive on MR"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#5-hive-on-tez","text":"\u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002","title":"5. Hive on Tez"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#6-hive-on-spark","text":"\u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002","title":"6. Hive on Spark"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#oss-hdfs","text":"\u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS-HDFS \u670d\u52a1\u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'oss://bucket_name.endpoint_name/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'oss://bucket_name.endpoint_name/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS-HDFS \u670d\u52a1\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>oss://bucket_name.endpoint_name/path/to/warehouse</value> </property> </configuration>","title":"OSS-HDFS \u670d\u52a1\u7528\u4e8e\u8868\u7684\u5b58\u50a8"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#oss","text":"ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'oss://bucket_name.endpoint_name/path/to/us/part210301cn';","title":"\u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#_2","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofs/impala/jindosdk_on_impala/","text":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e \u00a7 JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Impala \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Impala \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fddImpala\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728\u6240\u6709 Impala \u8282\u70b9\u5b89\u88c5 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06 sdk \u5305\u5b89\u88c5\u5230 Impala \u7684 classpath \u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $IMPALA_HOME/lib/ 3. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e IMPALA \u4f7f\u7528\u7684 core-site.xml \u914d\u7f6e JindoSDK \u8bbf\u95ee OSS \u00a7 \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 4. \u4f7f\u7528 Impala \u8bbf\u95ee OSS\u3002 \u00a7 \u4e0b\u8f7d\u6d4b\u8bd5\u6570\u636e\u96c6 \u94fe\u63a5 \u4e0a\u4f20\u6570\u636e bash hadoop fs -put test_data oss://bucket.endpoint/dir \u521b\u5efa\u8868 sql CREATE EXTERNAL TABLE customer_demographics ( `cd_demo_sk` INT, `cd_gender` STRING, `cd_marital_status` STRING, `cd_education_status` STRING, `cd_purchase_estimate` INT, `cd_credit_rating` STRING, `cd_dep_count` INT, `cd_dep_employed_count` INT, `cd_dep_college_count` INT) STORED AS PARQUET LOCATION 'oss://bucket.endpoint/dir'; \u67e5\u8be2 OSS \u8868 sql select * from customer_demographics; 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofs/impala/jindosdk_on_impala/#impala-jindosdk-oss-hdfs-jindofs","text":"JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Impala \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofs/impala/jindosdk_on_impala/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofs/impala/jindosdk_on_impala/#1-impala-hadoop","text":"\u786e\u4fddImpala\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Impala \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"backup/jindofs/impala/jindosdk_on_impala/#2-impala-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06 sdk \u5305\u5b89\u88c5\u5230 Impala \u7684 classpath \u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $IMPALA_HOME/lib/","title":"2. \u5728\u6240\u6709 Impala \u8282\u70b9\u5b89\u88c5 JindoSDK"},{"location":"backup/jindofs/impala/jindosdk_on_impala/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofs/impala/jindosdk_on_impala/#impala-core-sitexml-jindosdk-oss","text":"\u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002","title":"\u914d\u7f6e IMPALA \u4f7f\u7528\u7684 core-site.xml \u914d\u7f6e JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/jindofs/impala/jindosdk_on_impala/#4-impala-oss","text":"\u4e0b\u8f7d\u6d4b\u8bd5\u6570\u636e\u96c6 \u94fe\u63a5 \u4e0a\u4f20\u6570\u636e bash hadoop fs -put test_data oss://bucket.endpoint/dir \u521b\u5efa\u8868 sql CREATE EXTERNAL TABLE customer_demographics ( `cd_demo_sk` INT, `cd_gender` STRING, `cd_marital_status` STRING, `cd_education_status` STRING, `cd_purchase_estimate` INT, `cd_credit_rating` STRING, `cd_dep_count` INT, `cd_dep_employed_count` INT, `cd_dep_college_count` INT) STORED AS PARQUET LOCATION 'oss://bucket.endpoint/dir'; \u67e5\u8be2 OSS \u8868 sql select * from customer_demographics;","title":"4. \u4f7f\u7528 Impala \u8bbf\u95ee OSS\u3002"},{"location":"backup/jindofs/impala/jindosdk_on_impala/#5","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_QA/","text":"1\u3001list \u6027\u80fd\u4f4e \u00a7 \u73b0\u8c61 \u00a7 \u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u53d1\u73b0 list \u6027\u80fd\u8f83\u6162\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Successfully list objects with prefix xxx/yyy/ in bucket xxx recursive 0 result 315 dur 100036.615031MS \u5176\u4e2d dur 100036.615031MS \u4ee3\u8868\u6b64\u6b21 list \u7528\u65f6\uff0c\u5355\u4f4d\u6beb\u79d2\uff0c\u5982\u4e0a\u8ff0\u4e00\u6b21 list \u8017\u65f6 100 \u79d2\uff0cOSS \u4e0a\u6587\u4ef6\u6b63\u5e38\u7684 list \u901f\u5ea6\u662f 1000 \u6587\u4ef6\u5728 1s \u4ee5\u4e0b\uff0c\u60a8\u53ef\u6839\u636e\u5f53\u524d\u76ee\u5f55\u4e0b\u6587\u4ef6\u6570\u91cf\u6765\u5224\u65ad\u8be5 list \u8017\u65f6\u662f\u5426\u5f02\u5e38\uff0c\u5982\u4e0a\u8ff0\u4fe1\u606f\u663e\u793a list 315 \u4e2a\u6587\u4ef6\u7684\u76ee\u5f55\u9700\u8981 100s \u663e\u7136\u662f\u4e0d\u592a\u6b63\u5e38\u7684\u3002 \u89e3\u51b3\u529e\u6cd5 \u00a7 \u589e\u52a0\u5ba2\u6237\u7aef\u5185\u5b58\uff0c\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u589e\u5927\u5ba2\u6237\u7aef\u5185\u5b58 export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx4096m\" 2. checksum \u62a5\u9519 \u00a7 \u73b0\u8c61 \u00a7 \u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Failed to get checksum store. \u89e3\u51b3\u529e\u6cd5 \u00a7 OSS-HDFS \u9ed8\u8ba4\u7684 checksum \u7b97\u6cd5\u662f COMPOSITE_CRC\uff0c\u5982\u679c HDFS \u914d\u7f6e\u7684 dfs.checksum.combine.mode \u4e3a MD5MD5CRC\uff0c\u5219\u9700\u8981\u53d8\u66f4 OSS-HDFS \u7684 fs.oss.checksum.combine.mode \u914d\u7f6e\u4e3a MD5MD5CRC\u3002 hadoop jar jindo-distcp-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.checksum.combine.mode=MD5MD5CRC","title":"jindo distcp QA"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_QA/#1list","text":"","title":"1\u3001list \u6027\u80fd\u4f4e"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_QA/#_1","text":"\u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u53d1\u73b0 list \u6027\u80fd\u8f83\u6162\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Successfully list objects with prefix xxx/yyy/ in bucket xxx recursive 0 result 315 dur 100036.615031MS \u5176\u4e2d dur 100036.615031MS \u4ee3\u8868\u6b64\u6b21 list \u7528\u65f6\uff0c\u5355\u4f4d\u6beb\u79d2\uff0c\u5982\u4e0a\u8ff0\u4e00\u6b21 list \u8017\u65f6 100 \u79d2\uff0cOSS \u4e0a\u6587\u4ef6\u6b63\u5e38\u7684 list \u901f\u5ea6\u662f 1000 \u6587\u4ef6\u5728 1s \u4ee5\u4e0b\uff0c\u60a8\u53ef\u6839\u636e\u5f53\u524d\u76ee\u5f55\u4e0b\u6587\u4ef6\u6570\u91cf\u6765\u5224\u65ad\u8be5 list \u8017\u65f6\u662f\u5426\u5f02\u5e38\uff0c\u5982\u4e0a\u8ff0\u4fe1\u606f\u663e\u793a list 315 \u4e2a\u6587\u4ef6\u7684\u76ee\u5f55\u9700\u8981 100s \u663e\u7136\u662f\u4e0d\u592a\u6b63\u5e38\u7684\u3002","title":"\u73b0\u8c61"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_QA/#_2","text":"\u589e\u52a0\u5ba2\u6237\u7aef\u5185\u5b58\uff0c\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u589e\u5927\u5ba2\u6237\u7aef\u5185\u5b58 export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx4096m\"","title":"\u89e3\u51b3\u529e\u6cd5"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_QA/#2-checksum","text":"","title":"2. checksum \u62a5\u9519"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_QA/#_3","text":"\u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Failed to get checksum store.","title":"\u73b0\u8c61"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_QA/#_4","text":"OSS-HDFS \u9ed8\u8ba4\u7684 checksum \u7b97\u6cd5\u662f COMPOSITE_CRC\uff0c\u5982\u679c HDFS \u914d\u7f6e\u7684 dfs.checksum.combine.mode \u4e3a MD5MD5CRC\uff0c\u5219\u9700\u8981\u53d8\u66f4 OSS-HDFS \u7684 fs.oss.checksum.combine.mode \u914d\u7f6e\u4e3a MD5MD5CRC\u3002 hadoop jar jindo-distcp-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.checksum.combine.mode=MD5MD5CRC","title":"\u89e3\u51b3\u529e\u6cd5"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/","text":"\u4f7f\u7528 JindoDistCp \u00a7 \u4ecb\u7ecd \u00a7 JindoDistCp\uff08\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u5de5\u5177\uff09\u662f\u7528\u4e8e\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u62f7\u8d1d\u6587\u4ef6\u7684\u5de5\u5177\u3002 \u5b83\u4f7f\u7528 MapReduce \u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3a MapReduce \u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002\u5168\u91cf\u652f\u6301 HDFS->OSS-HDFS \u670d\u52a1\uff0cHDFS->HDFS\uff0cOSS-HDFS \u670d\u52a1->HDFS\uff0cOSS-HDFS \u670d\u52a1->OSS-HDFS \u670d\u52a1\uff0c\u63d0\u4f9b\u591a\u79cd\u4e2a\u6027\u5316\u62f7\u8d1d\u53c2\u6570\u548c\u591a\u79cd\u62f7\u8d1d\u7b56\u7565\u3002\u91cd\u70b9\u4f18\u5316 HDFS \u5230 OSS-HDFS \u670d\u52a1\u7684\u6570\u636e\u62f7\u8d1d\uff0c\u901a\u8fc7\u5b9a\u5236\u5316CopyCommitter\uff0c\u5b9e\u73b0No-Rename\u62f7\u8d1d\uff0c\u5e76\u4fdd\u8bc1\u6570\u636e\u62f7\u8d1d\u843d\u5730\u7684\u4e00\u81f4\u6027\u3002\u529f\u80fd\u5168\u91cf\u5bf9\u9f50S3 DistCp\u548cHDFS DistCp\uff0c\u6027\u80fd\u8f83HDFS DistCp\u6709\u8f83\u5927\u63d0\u5347\uff0c\u76ee\u6807\u63d0\u4f9b\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177\u3002 \u60a8\u53ef\u4ee5\u4f7f\u7528 JindoDistCp \u8fdb\u884c HDFS \u548c OSS-HDFS \u670d\u52a1\u4e4b\u95f4\u7684\u6570\u636e\u8fc1\u79fb\uff0c\u76f8\u5bf9\u4e8e Hadoop \u793e\u533a DistCp \u5b9e\u73b0\uff0c\u60a8\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u4f7f\u7528\u6307\u5357 \u00a7 JindoDistCp \u63d0\u4f9b jar \u5305\u5f62\u5f0f\u4f7f\u7528\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 hadoop jar \u547d\u4ee4\u914d\u5408\u4e00\u7cfb\u5217\u53c2\u6570\u6765\u5b8c\u6210 distcp \u64cd\u4f5c\u3002 1\u3001\u4f7f\u7528--src\u548c--dest (\u5fc5\u9009) \u00a7 --src \u8868\u793a\u6307\u5b9a\u6e90\u6587\u4ef6\u7684\u8def\u5f84 --dest \u8868\u793a\u76ee\u6807\u6587\u4ef6\u7684\u8def\u5f84 \u5982\u4eceHDFS\u4e0a/opt/tmp\u76ee\u5f55\u62f7\u8d1d\u5230OSS-HDFS \u670d\u52a1 bucket\uff0c\u53ef\u4ee5\u6267\u884c\u5982\u4e0b\u8bed\u53e5 hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/tmp \u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9adest\u8def\u5f84\u6765\u786e\u5b9a\u62f7\u8d1d\u540e\u7684\u6587\u4ef6\u5c42\u6b21\uff0c\u5982\u60a8\u9700\u8981\u5c06/opt/tmp\u4e0b\u7684\u6587\u4ef6\u62f7\u8d1d\u5230yang-hhht\u8fd9\u4e2abucket\u4e0b\u7684tmp\u76ee\u5f55\u4e0b\uff0c\u5219\u53ef\u4ee5\u4f7f\u7528\u4e0a\u8ff0\u8bed\u53e5\u6765\u5b8c\u6210\u3002\u6b64\u5904\u548cHadoop\u7684distcp\u884c\u4e3a\u6709\u6240\u4e0d\u540c\uff0cJindoDistCp\u4f1a\u9ed8\u8ba4\u5c06src\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6587\u4ef6\u62f7\u8d1d\u5230\u60a8\u6307\u5b9adest\u8def\u5f84\u4e0b\uff0c\u5e76\u4e0d\u5305\u62ec\u5f53\u524d\u7684\u6839\u76ee\u5f55\u540d\u79f0\uff0c\u60a8\u53ef\u4ee5\u5728dest\u4e2d\u6307\u5b9a\u62f7\u8d1d\u8def\u5f84\u7684\u6839\u76ee\u5f55\uff0c\u5982\u679c\u4e0d\u5b58\u5728\u4f1a\u81ea\u52a8\u521b\u5efa\u3002 \u6ce8\u610f\uff1a\u5982\u9700\u62f7\u8d1d\u5355\u4e2a\u6587\u4ef6, dest\u9700\u6307\u5b9a\u4e3a\u76ee\u5f55\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /test.txt --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/tmp 2\u3001\u4f7f\u7528--bandWidth \u00a7 \u5728\u60a8\u7684distcp\u8fc7\u7a0b\u4e2d\uff0c\u60a8\u53ef\u4ee5\u6307\u5b9a\u672c\u6b21distcp\u4efb\u52a1\u6240\u7528\u7684\u5355\u673a\u5e26\u5bbd(\u4ee5MB\u4e3a\u5355\u4f4d)\uff0c\u907f\u514d\u5355\u673a\u5360\u7528\u8fc7\u5927\u5e26\u5bbd \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --bandWidth 6 3\u3001\u4f7f\u7528--codec \u00a7 \u539f\u59cb\u6587\u4ef6\u901a\u5e38\u4ee5\u672a\u538b\u7f29\u7684\u6587\u672c\u683c\u5f0f\u8fdb\u5165OSS\u6216\u8005HDFS\u3002\u65e0\u8bba\u662f\u5b58\u50a8\u6210\u672c\u8fd8\u662f\u5bf9\u8be5\u6570\u636e\u8fdb\u884c\u5206\u6790\uff0c\u6b64\u683c\u5f0f\u90fd\u4e0d\u751a\u7406\u60f3\u3002JindoDistCp \u53ef\u4ee5\u4f7f\u7528 --codec \u9009\u9879\u5e2e\u52a9\u60a8\u5728\u7ebf\u9ad8\u6548\u5730\u5b58\u50a8\u6570\u636e\u548c\u538b\u7f29\u6587\u4ef6 \u547d\u4ee4\u793a\u4f8b\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --codec gz --parallelism 20 JindoDistCp \u7684\u5f53\u524d\u7248\u672c\u652f\u6301\u7f16\u89e3\u7801\u5668 gzip\u3001gz\u3001lzo\u3001lzop \u548c snappy \u4ee5\u53ca\u5173\u952e\u5b57 none \u548c keep \uff08\u9ed8\u8ba4\u503c\uff09\u3002\u8fd9\u4e9b\u5173\u952e\u5b57\u542b\u4e49\u5982\u4e0b\uff1a \"none\" \u2013 \u4fdd\u5b58\u4e3a\u672a\u538b\u7f29\u7684\u6587\u4ef6\u3002\u5982\u679c\u6587\u4ef6\u5df2\u538b\u7f29\uff0c\u5219 JindoDistCp \u4f1a\u5c06\u5176\u89e3\u538b\u7f29\u3002 \"keep\" \u2013 \u4e0d\u66f4\u6539\u6587\u4ef6\u538b\u7f29\u5f62\u6001\uff0c\u6309\u539f\u6837\u590d\u5236\u3002 \u6211\u4eec\u68c0\u67e5\u4e00\u4e0b\u76ee\u6807\u6587\u4ef6\u5939\u4e2d\u7684\u6587\u4ef6\uff0c\u8fd9\u4e9b\u6587\u4ef6\u73b0\u5728\u5df2\u7ecf\u7528 gz \u7f16\u89e3\u7801\u5668\u538b\u7f29\u4e86\uff1a [root@emr-header-1 opt]# hdfs dfs -ls oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03 Found 6 items -rw-rw-rw- 1 938 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/000151.sst.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/1.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/2.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/OPTIONS-000109.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/emp01.txt.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/emp06.txt.gz \u5982\u60a8\u5728\u5f00\u6e90Hadoop\u96c6\u7fa4\u73af\u5883\u4e2d\u4f7f\u7528lzo\u538b\u7f29\u529f\u80fd\uff0c\u5219\u60a8\u9700\u8981\u53bb\u5b89\u88c5gplcompression\u7684native\u5e93\u548chadoop-lzo\u5305\uff0c\u5982\u60a8\u7f3a\u5c11\u76f8\u5173\u73af\u5883\uff0c\u5efa\u8bae\u4f7f\u7528\u5176\u4ed6\u538b\u7f29\u65b9\u5f0f\u8fdb\u884c\u538b\u7f29\u3002 4\u3001\u4f7f\u7528--filters \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--filters\u6765\u6307\u5b9a\u6709\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --filters filter.txt \u5047\u8bbe filter.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u4e0d\u4f1a\u5c06\u8def\u5f84\u4e2d\u5e26\u6709 test \u7684\u6587\u4ef6\u62f7\u8d1d\u5230OSS\u3002 5\u3001\u4f7f\u7528--srcPrefixesFile \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--srcPrefixesFile\u6765\u6307\u5b9a\u6709\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --srcPrefixesFile prefixes.txt \u5047\u8bbe prefixes.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u53ea\u6709\u8def\u5f84\u4e2d\u5e26\u6709 test \u7684\u6587\u4ef6\u624d\u4f1a\u62f7\u8d1d\u5230OSS\u3002 6\u3001\u4f7f\u7528--parallelism \u00a7 parallelism \u53c2\u6570\u7528\u6765\u6307\u5b9aMR\u4efb\u52a1\u91cc\u7684 mapreduce.job.maps \u53c2\u6570\uff0c\u60a8\u53ef\u4ee5\u6839\u636e\u96c6\u7fa4\u7684\u8d44\u6e90\u60c5\u51b5\u81ea\u5b9a\u4e49 parallelism \u7684\u5927\u5c0f\u6765\u63a7\u5236 distcp \u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\u3002 \u5982\u4eceHDFS\u4e0a/opt/tmp\u76ee\u5f55\u62f7\u8d1d\u5230OSS bucket\uff0c\u53ef\u4ee5\u6267\u884c hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/tmp --parallelism 20 7\u3001\u4f7f\u7528--taskBatch \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--taskBatch\u6765\u6307\u5b9a\u6bcf\u4e2adistcp task\u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --taskBatch 10 8\u3001\u4f7f\u7528--tmp \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7tmp\u53c2\u6570\u6307\u5b9a\u4e00\u4e2a\u4e34\u65f6\u76ee\u5f55\u5728hdfs\u4e0a\u4f20\u5b58\u653e\u4e34\u65f6\u6570\u636e\uff0c\u9ed8\u8ba4\u503c\u4e3a/tmp\uff0c\u5373hdfs:///tmp/\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --parallelism 20 --tmp /tmp 9\u3001\u4f7f\u7528 OSS-HDFS \u670d\u52a1 AK \u00a7 \u5728 EMR \u5916\u6216\u8005\u514d\u5bc6\u670d\u52a1\u51fa\u73b0\u95ee\u9898\u7684\u60c5\u51b5\u4e0b\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9aAK\u6765\u83b7\u5f97\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u6743\u9650\u3002\u60a8\u53ef\u4ee5\u5728\u547d\u4ee4\u4e2d\u4f7f\u7528 --hadoopConf \u9009\u9879\u6765\u6307\u5b9aAK\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism 20 \u60a8\u4e5f\u53ef\u4ee5\u5c06 OSS-HDFS \u670d\u52a1\u7684 ak\u3001secret \u9884\u5148\u914d\u7f6e\u5728 hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u907f\u514d\u6bcf\u6b21\u4f7f\u7528\u65f6\u4e34\u65f6\u586b\u5199ak\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> 10\u3001\u4f7f\u7528--disableChecksum \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--disableChecksum\u5173\u95ed\u68c0\u67e5\u6587\u4ef6checksum\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --disableChecksum 11\u3001\u4f7f\u7528--deleteOnSuccess \u00a7 \u6709\u65f6\u5019\u6211\u4eec\u662f\u60f3\u8981\u79fb\u52a8\u6570\u636e\u800c\u4e0d\u662f\u590d\u5236\u6570\u636e\uff0c\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 --deleteOnSuccess \u9009\u9879\u3002\u6b64\u9009\u9879\u7c7b\u4f3c\u4e8emv\u64cd\u4f5c\uff0c\u9996\u5148\u590d\u5236\u6587\u4ef6\uff0c\u7136\u540e\u4ece\u6e90\u4f4d\u7f6e\u5220\u9664\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --deleteOnSuccess --parallelism 20 12\u3001\u4f7f\u7528--enableTransaction \u00a7 JindoDistCp\u9ed8\u8ba4\u4f7f\u7528task\u7ea7\u522b\u5b8c\u6574\u6027\uff0c\u5982\u60a8\u9700\u8981\u4fdd\u8bc1Job\u7ea7\u522b\u7684\u5b8c\u6574\u6027\u4ee5\u53ca\u4fdd\u8bc1Job\u4e4b\u95f4\u7684\u4e8b\u52a1\u652f\u6301\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528--enableTransaction\u53c2\u6570\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --enableTransaction --parallelism 20 13\u3001\u4f7f\u7528--ignore \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7ignore\u53c2\u6570\uff0c\u5ffd\u7565\u6570\u636e\u8fc1\u79fb\u671f\u95f4\u53d1\u751f\u7684\u5f02\u5e38\uff0c\u76f8\u5173\u62a5\u9519\u4e0d\u4f1a\u4e2d\u65ad\u4efb\u52a1\uff0c\u5e76\u6700\u7ec8\u4ee5DistCp Counter\u7684\u5f62\u5f0f\u900f\u51fa\u3002\uff08\u5982\u679c\u5f00\u542fCMS\u7684\u8bdd\uff0c\u4e5f\u4f1a\u4ee5\u6307\u5b9a\u65b9\u5f0f\u8fdb\u884c\u901a\u77e5\u3002\uff09 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --ignore --parallelism 20 14\u3001\u4f7f\u7528--diff \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5b8c\u6210\u540e\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9asrc\u548cdest\u6765\u67e5\u770b\u5f53\u524ddistcp\u7684\u6587\u4ef6\u5dee\u5f02, \u5982\u679csrc\u7684\u6587\u4ef6\u672a\u80fd\u540c\u6b65\u5230dest\u4e0a\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\u3002\u5982\u60a8\u7684distcp\u4efb\u52a1\u5305\u542b\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u5219--diff\u4e0d\u80fd\u663e\u793a\u6b63\u786e\u7684\u6587\u4ef6\u5dee\u5f02\uff0c\u56e0\u4e3a\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u4f1a\u6539\u53d8\u6587\u4ef6\u7684\u5927\u5c0f\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --diff \u5982\u679c\u6587\u4ef6\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter DIFF_FILES=1 \u5982\u679c\u60a8\u7684--dest\u4e3aHDFS\u8def\u5f84\uff0c\u73b0\u652f\u6301 /path \uff0c hdfs://hostname:ip/path \uff0c hdfs://headerIp:ip/path \u7684\u5199\u6cd5\uff0c\u6682\u4e0d\u652f\u6301 hdfs:///path \uff0c hdfs:/path \u548c\u5176\u4ed6\u81ea\u5b9a\u4e49\u5199\u6cd5\u3002 \u5982\u679c\u60f3\u67e5\u770b\u6587\u4ef6\u5143\u6570\u636e\u7684\u5dee\u5f02\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 --diff --preserveMeta \u547d\u4ee4\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --diff --preserveMeta \u5982\u679c\u6587\u4ef6\u5143\u6570\u636e\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5143\u6570\u636e\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter META_FILE_DIFF=1 15\u3001\u4f7f\u7528--update \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7update\u53c2\u6570\u4f7f\u7528\u589e\u91cf\u540c\u6b65\u529f\u80fd\uff0c\u8df3\u8fc7\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u548c\u76ee\u5f55\uff0c\u76f4\u63a5\u5c06src\u4e2d\u65b0\u589e\u6216\u53d1\u751f\u6539\u53d8\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u540c\u6b65\u5230dest\u4e0a\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --update --parallelism 20 16\u3001\u4f7f\u7528--preserveMeta \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/tmp --preserveMeta 17\u3001\u4f7f\u7528--jobBatch \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--jobBatch\u6765\u6307\u5b9a\u6bcf\u4e2adistcp job\u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10000\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --jobBatch 50000","title":"\u4f7f\u7528 JindoDistCp"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#jindodistcp","text":"","title":"\u4f7f\u7528 JindoDistCp"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#_1","text":"JindoDistCp\uff08\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u5de5\u5177\uff09\u662f\u7528\u4e8e\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u62f7\u8d1d\u6587\u4ef6\u7684\u5de5\u5177\u3002 \u5b83\u4f7f\u7528 MapReduce \u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3a MapReduce \u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002\u5168\u91cf\u652f\u6301 HDFS->OSS-HDFS \u670d\u52a1\uff0cHDFS->HDFS\uff0cOSS-HDFS \u670d\u52a1->HDFS\uff0cOSS-HDFS \u670d\u52a1->OSS-HDFS \u670d\u52a1\uff0c\u63d0\u4f9b\u591a\u79cd\u4e2a\u6027\u5316\u62f7\u8d1d\u53c2\u6570\u548c\u591a\u79cd\u62f7\u8d1d\u7b56\u7565\u3002\u91cd\u70b9\u4f18\u5316 HDFS \u5230 OSS-HDFS \u670d\u52a1\u7684\u6570\u636e\u62f7\u8d1d\uff0c\u901a\u8fc7\u5b9a\u5236\u5316CopyCommitter\uff0c\u5b9e\u73b0No-Rename\u62f7\u8d1d\uff0c\u5e76\u4fdd\u8bc1\u6570\u636e\u62f7\u8d1d\u843d\u5730\u7684\u4e00\u81f4\u6027\u3002\u529f\u80fd\u5168\u91cf\u5bf9\u9f50S3 DistCp\u548cHDFS DistCp\uff0c\u6027\u80fd\u8f83HDFS DistCp\u6709\u8f83\u5927\u63d0\u5347\uff0c\u76ee\u6807\u63d0\u4f9b\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177\u3002 \u60a8\u53ef\u4ee5\u4f7f\u7528 JindoDistCp \u8fdb\u884c HDFS \u548c OSS-HDFS \u670d\u52a1\u4e4b\u95f4\u7684\u6570\u636e\u8fc1\u79fb\uff0c\u76f8\u5bf9\u4e8e Hadoop \u793e\u533a DistCp \u5b9e\u73b0\uff0c\u60a8\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"\u4ecb\u7ecd"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#_2","text":"JindoDistCp \u63d0\u4f9b jar \u5305\u5f62\u5f0f\u4f7f\u7528\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 hadoop jar \u547d\u4ee4\u914d\u5408\u4e00\u7cfb\u5217\u53c2\u6570\u6765\u5b8c\u6210 distcp \u64cd\u4f5c\u3002","title":"\u4f7f\u7528\u6307\u5357"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#1-src-dest","text":"--src \u8868\u793a\u6307\u5b9a\u6e90\u6587\u4ef6\u7684\u8def\u5f84 --dest \u8868\u793a\u76ee\u6807\u6587\u4ef6\u7684\u8def\u5f84 \u5982\u4eceHDFS\u4e0a/opt/tmp\u76ee\u5f55\u62f7\u8d1d\u5230OSS-HDFS \u670d\u52a1 bucket\uff0c\u53ef\u4ee5\u6267\u884c\u5982\u4e0b\u8bed\u53e5 hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/tmp \u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9adest\u8def\u5f84\u6765\u786e\u5b9a\u62f7\u8d1d\u540e\u7684\u6587\u4ef6\u5c42\u6b21\uff0c\u5982\u60a8\u9700\u8981\u5c06/opt/tmp\u4e0b\u7684\u6587\u4ef6\u62f7\u8d1d\u5230yang-hhht\u8fd9\u4e2abucket\u4e0b\u7684tmp\u76ee\u5f55\u4e0b\uff0c\u5219\u53ef\u4ee5\u4f7f\u7528\u4e0a\u8ff0\u8bed\u53e5\u6765\u5b8c\u6210\u3002\u6b64\u5904\u548cHadoop\u7684distcp\u884c\u4e3a\u6709\u6240\u4e0d\u540c\uff0cJindoDistCp\u4f1a\u9ed8\u8ba4\u5c06src\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6587\u4ef6\u62f7\u8d1d\u5230\u60a8\u6307\u5b9adest\u8def\u5f84\u4e0b\uff0c\u5e76\u4e0d\u5305\u62ec\u5f53\u524d\u7684\u6839\u76ee\u5f55\u540d\u79f0\uff0c\u60a8\u53ef\u4ee5\u5728dest\u4e2d\u6307\u5b9a\u62f7\u8d1d\u8def\u5f84\u7684\u6839\u76ee\u5f55\uff0c\u5982\u679c\u4e0d\u5b58\u5728\u4f1a\u81ea\u52a8\u521b\u5efa\u3002 \u6ce8\u610f\uff1a\u5982\u9700\u62f7\u8d1d\u5355\u4e2a\u6587\u4ef6, dest\u9700\u6307\u5b9a\u4e3a\u76ee\u5f55\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /test.txt --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/tmp","title":"1\u3001\u4f7f\u7528--src\u548c--dest (\u5fc5\u9009)"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#2-bandwidth","text":"\u5728\u60a8\u7684distcp\u8fc7\u7a0b\u4e2d\uff0c\u60a8\u53ef\u4ee5\u6307\u5b9a\u672c\u6b21distcp\u4efb\u52a1\u6240\u7528\u7684\u5355\u673a\u5e26\u5bbd(\u4ee5MB\u4e3a\u5355\u4f4d)\uff0c\u907f\u514d\u5355\u673a\u5360\u7528\u8fc7\u5927\u5e26\u5bbd \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --bandWidth 6","title":"2\u3001\u4f7f\u7528--bandWidth"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#3-codec","text":"\u539f\u59cb\u6587\u4ef6\u901a\u5e38\u4ee5\u672a\u538b\u7f29\u7684\u6587\u672c\u683c\u5f0f\u8fdb\u5165OSS\u6216\u8005HDFS\u3002\u65e0\u8bba\u662f\u5b58\u50a8\u6210\u672c\u8fd8\u662f\u5bf9\u8be5\u6570\u636e\u8fdb\u884c\u5206\u6790\uff0c\u6b64\u683c\u5f0f\u90fd\u4e0d\u751a\u7406\u60f3\u3002JindoDistCp \u53ef\u4ee5\u4f7f\u7528 --codec \u9009\u9879\u5e2e\u52a9\u60a8\u5728\u7ebf\u9ad8\u6548\u5730\u5b58\u50a8\u6570\u636e\u548c\u538b\u7f29\u6587\u4ef6 \u547d\u4ee4\u793a\u4f8b\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --codec gz --parallelism 20 JindoDistCp \u7684\u5f53\u524d\u7248\u672c\u652f\u6301\u7f16\u89e3\u7801\u5668 gzip\u3001gz\u3001lzo\u3001lzop \u548c snappy \u4ee5\u53ca\u5173\u952e\u5b57 none \u548c keep \uff08\u9ed8\u8ba4\u503c\uff09\u3002\u8fd9\u4e9b\u5173\u952e\u5b57\u542b\u4e49\u5982\u4e0b\uff1a \"none\" \u2013 \u4fdd\u5b58\u4e3a\u672a\u538b\u7f29\u7684\u6587\u4ef6\u3002\u5982\u679c\u6587\u4ef6\u5df2\u538b\u7f29\uff0c\u5219 JindoDistCp \u4f1a\u5c06\u5176\u89e3\u538b\u7f29\u3002 \"keep\" \u2013 \u4e0d\u66f4\u6539\u6587\u4ef6\u538b\u7f29\u5f62\u6001\uff0c\u6309\u539f\u6837\u590d\u5236\u3002 \u6211\u4eec\u68c0\u67e5\u4e00\u4e0b\u76ee\u6807\u6587\u4ef6\u5939\u4e2d\u7684\u6587\u4ef6\uff0c\u8fd9\u4e9b\u6587\u4ef6\u73b0\u5728\u5df2\u7ecf\u7528 gz \u7f16\u89e3\u7801\u5668\u538b\u7f29\u4e86\uff1a [root@emr-header-1 opt]# hdfs dfs -ls oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03 Found 6 items -rw-rw-rw- 1 938 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/000151.sst.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/1.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/2.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/OPTIONS-000109.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/emp01.txt.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/emp06.txt.gz \u5982\u60a8\u5728\u5f00\u6e90Hadoop\u96c6\u7fa4\u73af\u5883\u4e2d\u4f7f\u7528lzo\u538b\u7f29\u529f\u80fd\uff0c\u5219\u60a8\u9700\u8981\u53bb\u5b89\u88c5gplcompression\u7684native\u5e93\u548chadoop-lzo\u5305\uff0c\u5982\u60a8\u7f3a\u5c11\u76f8\u5173\u73af\u5883\uff0c\u5efa\u8bae\u4f7f\u7528\u5176\u4ed6\u538b\u7f29\u65b9\u5f0f\u8fdb\u884c\u538b\u7f29\u3002","title":"3\u3001\u4f7f\u7528--codec"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#4-filters","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--filters\u6765\u6307\u5b9a\u6709\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --filters filter.txt \u5047\u8bbe filter.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u4e0d\u4f1a\u5c06\u8def\u5f84\u4e2d\u5e26\u6709 test \u7684\u6587\u4ef6\u62f7\u8d1d\u5230OSS\u3002","title":"4\u3001\u4f7f\u7528--filters"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#5-srcprefixesfile","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--srcPrefixesFile\u6765\u6307\u5b9a\u6709\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --srcPrefixesFile prefixes.txt \u5047\u8bbe prefixes.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u53ea\u6709\u8def\u5f84\u4e2d\u5e26\u6709 test \u7684\u6587\u4ef6\u624d\u4f1a\u62f7\u8d1d\u5230OSS\u3002","title":"5\u3001\u4f7f\u7528--srcPrefixesFile"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#6-parallelism","text":"parallelism \u53c2\u6570\u7528\u6765\u6307\u5b9aMR\u4efb\u52a1\u91cc\u7684 mapreduce.job.maps \u53c2\u6570\uff0c\u60a8\u53ef\u4ee5\u6839\u636e\u96c6\u7fa4\u7684\u8d44\u6e90\u60c5\u51b5\u81ea\u5b9a\u4e49 parallelism \u7684\u5927\u5c0f\u6765\u63a7\u5236 distcp \u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\u3002 \u5982\u4eceHDFS\u4e0a/opt/tmp\u76ee\u5f55\u62f7\u8d1d\u5230OSS bucket\uff0c\u53ef\u4ee5\u6267\u884c hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/tmp --parallelism 20","title":"6\u3001\u4f7f\u7528--parallelism"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#7-taskbatch","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--taskBatch\u6765\u6307\u5b9a\u6bcf\u4e2adistcp task\u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --taskBatch 10","title":"7\u3001\u4f7f\u7528--taskBatch"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#8-tmp","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7tmp\u53c2\u6570\u6307\u5b9a\u4e00\u4e2a\u4e34\u65f6\u76ee\u5f55\u5728hdfs\u4e0a\u4f20\u5b58\u653e\u4e34\u65f6\u6570\u636e\uff0c\u9ed8\u8ba4\u503c\u4e3a/tmp\uff0c\u5373hdfs:///tmp/\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --parallelism 20 --tmp /tmp","title":"8\u3001\u4f7f\u7528--tmp"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#9-oss-hdfs-ak","text":"\u5728 EMR \u5916\u6216\u8005\u514d\u5bc6\u670d\u52a1\u51fa\u73b0\u95ee\u9898\u7684\u60c5\u51b5\u4e0b\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9aAK\u6765\u83b7\u5f97\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u6743\u9650\u3002\u60a8\u53ef\u4ee5\u5728\u547d\u4ee4\u4e2d\u4f7f\u7528 --hadoopConf \u9009\u9879\u6765\u6307\u5b9aAK\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism 20 \u60a8\u4e5f\u53ef\u4ee5\u5c06 OSS-HDFS \u670d\u52a1\u7684 ak\u3001secret \u9884\u5148\u914d\u7f6e\u5728 hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u907f\u514d\u6bcf\u6b21\u4f7f\u7528\u65f6\u4e34\u65f6\u586b\u5199ak\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration>","title":"9\u3001\u4f7f\u7528 OSS-HDFS \u670d\u52a1 AK"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#10-disablechecksum","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--disableChecksum\u5173\u95ed\u68c0\u67e5\u6587\u4ef6checksum\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --disableChecksum","title":"10\u3001\u4f7f\u7528--disableChecksum"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#11-deleteonsuccess","text":"\u6709\u65f6\u5019\u6211\u4eec\u662f\u60f3\u8981\u79fb\u52a8\u6570\u636e\u800c\u4e0d\u662f\u590d\u5236\u6570\u636e\uff0c\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 --deleteOnSuccess \u9009\u9879\u3002\u6b64\u9009\u9879\u7c7b\u4f3c\u4e8emv\u64cd\u4f5c\uff0c\u9996\u5148\u590d\u5236\u6587\u4ef6\uff0c\u7136\u540e\u4ece\u6e90\u4f4d\u7f6e\u5220\u9664\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --deleteOnSuccess --parallelism 20","title":"11\u3001\u4f7f\u7528--deleteOnSuccess"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#12-enabletransaction","text":"JindoDistCp\u9ed8\u8ba4\u4f7f\u7528task\u7ea7\u522b\u5b8c\u6574\u6027\uff0c\u5982\u60a8\u9700\u8981\u4fdd\u8bc1Job\u7ea7\u522b\u7684\u5b8c\u6574\u6027\u4ee5\u53ca\u4fdd\u8bc1Job\u4e4b\u95f4\u7684\u4e8b\u52a1\u652f\u6301\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528--enableTransaction\u53c2\u6570\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --enableTransaction --parallelism 20","title":"12\u3001\u4f7f\u7528--enableTransaction"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#13-ignore","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7ignore\u53c2\u6570\uff0c\u5ffd\u7565\u6570\u636e\u8fc1\u79fb\u671f\u95f4\u53d1\u751f\u7684\u5f02\u5e38\uff0c\u76f8\u5173\u62a5\u9519\u4e0d\u4f1a\u4e2d\u65ad\u4efb\u52a1\uff0c\u5e76\u6700\u7ec8\u4ee5DistCp Counter\u7684\u5f62\u5f0f\u900f\u51fa\u3002\uff08\u5982\u679c\u5f00\u542fCMS\u7684\u8bdd\uff0c\u4e5f\u4f1a\u4ee5\u6307\u5b9a\u65b9\u5f0f\u8fdb\u884c\u901a\u77e5\u3002\uff09 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --ignore --parallelism 20","title":"13\u3001\u4f7f\u7528--ignore"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#14-diff","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5b8c\u6210\u540e\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9asrc\u548cdest\u6765\u67e5\u770b\u5f53\u524ddistcp\u7684\u6587\u4ef6\u5dee\u5f02, \u5982\u679csrc\u7684\u6587\u4ef6\u672a\u80fd\u540c\u6b65\u5230dest\u4e0a\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\u3002\u5982\u60a8\u7684distcp\u4efb\u52a1\u5305\u542b\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u5219--diff\u4e0d\u80fd\u663e\u793a\u6b63\u786e\u7684\u6587\u4ef6\u5dee\u5f02\uff0c\u56e0\u4e3a\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u4f1a\u6539\u53d8\u6587\u4ef6\u7684\u5927\u5c0f\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --diff \u5982\u679c\u6587\u4ef6\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter DIFF_FILES=1 \u5982\u679c\u60a8\u7684--dest\u4e3aHDFS\u8def\u5f84\uff0c\u73b0\u652f\u6301 /path \uff0c hdfs://hostname:ip/path \uff0c hdfs://headerIp:ip/path \u7684\u5199\u6cd5\uff0c\u6682\u4e0d\u652f\u6301 hdfs:///path \uff0c hdfs:/path \u548c\u5176\u4ed6\u81ea\u5b9a\u4e49\u5199\u6cd5\u3002 \u5982\u679c\u60f3\u67e5\u770b\u6587\u4ef6\u5143\u6570\u636e\u7684\u5dee\u5f02\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 --diff --preserveMeta \u547d\u4ee4\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --diff --preserveMeta \u5982\u679c\u6587\u4ef6\u5143\u6570\u636e\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5143\u6570\u636e\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter META_FILE_DIFF=1","title":"14\u3001\u4f7f\u7528--diff"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#15-update","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7update\u53c2\u6570\u4f7f\u7528\u589e\u91cf\u540c\u6b65\u529f\u80fd\uff0c\u8df3\u8fc7\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u548c\u76ee\u5f55\uff0c\u76f4\u63a5\u5c06src\u4e2d\u65b0\u589e\u6216\u53d1\u751f\u6539\u53d8\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u540c\u6b65\u5230dest\u4e0a\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --update --parallelism 20","title":"15\u3001\u4f7f\u7528--update"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#16-preservemeta","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/tmp --preserveMeta","title":"16\u3001\u4f7f\u7528--preserveMeta"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#17-jobbatch","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--jobBatch\u6765\u6307\u5b9a\u6bcf\u4e2adistcp job\u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10000\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --jobBatch 50000","title":"17\u3001\u4f7f\u7528--jobBatch"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_api/","text":"JindoDistCp API \u4f7f\u7528\u8bf4\u660e \u00a7 \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 \u5b89\u88c5\u4f9d\u8d56\u5305 \u00a7 \u76ee\u524d JindoSDK/JindoDistCP \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u4ee54.6.12\u7248\u672c\u4e3a\u4f8b\uff0c\u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoDistCP \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-distcp --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop with provided scope, your runtime should install hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> </dependencies> </project> \u793a\u4f8b\u4ee3\u7801 \u00a7 \u4ee5\u62f7\u8d1d HDFS \u6587\u4ef6\u5939\u5230\u963f\u91cc\u4e91 OSS \u4e3a\u4f8b\u3002 import com.aliyun.jindodata.distcp.util.DistCpCounter; import com.aliyun.jindodata.distjob.request.DistJobResponse; import org.apache.hadoop.conf.Configuration; import java.io.IOException; public class DistcpExample { static public void main(String[] args) throws IOException { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. conf.set(\"fs.oss.credentials.provider\", \"com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider\"); conf.set(\"fs.oss.accessKeyId\", \"XXX\"); conf.set(\"fs.oss.accessKeySecret\", \"XXX\"); DistCpRequest request = new DistCpRequest(); request.setSrcDir(\"hdfs:///user/root/random-data\"); request.setDestDir(\"oss://dls-bucket.cn-xxx.oss-dls.aliyuncs.com/dest\"); request.setWorkTempDir(\"hdfs:///user/root/tmp\"); request.setConf(conf); // Jindo DistCp Service DistCpService service = new DistCpService(); service.buildAllJobs(request); service.runAllJobs(); DistJobResponse response = service.collectAllJobResult(); printResponse(response); } static public void printResponse(DistJobResponse response) { System.out.println(\"ErrorCode: \" + response.getErrorCode()); System.out.println(\"ErrorMsg: \" + response.getErrorMsg()); System.out.println(\"FileCopied: \" + response.getCount(DistCpCounter.FILES_COPIED.getName())); System.out.println(\"BytesCopied: \" + response.getCount(DistCpCounter.BYTES_COPIED.getName())); } } \u8bf7\u6c42\u53c2\u6570 \u00a7 \u76f8\u5173\u65b9\u6cd5\u7684\u5177\u4f53\u63cf\u8ff0\u53ef\u53c2\u8003 \u300a\u4f7f\u7528Jindo DistCp\u300b \u65b9\u6cd5 \u53c2\u6570\u7c7b\u578b \u8bf4\u660e \u9ed8\u8ba4\u503c \u517c\u5bb9 void setSrc(String src) \u5fc5\u9009 \u8bbe\u7f6e\u6e90\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setDest(String dest) \u5fc5\u9009 \u8bbe\u7f6e\u76ee\u6807\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setConf(Configuration conf) \u5fc5\u9009 \u8bbe\u7f6e Configuration - 4.3.0+ void setMode(DistCpMode mode) \u53ef\u9009 \u8bbe\u7f6eDistCp\u6a21\u5f0f\uff0c\u652f\u6301\u7684\u6a21\u5f0f\u6709 COPY\uff08\u62f7\u8d1d\uff09 DIFF\uff08\u5dee\u5f02\u6bd4\u8f83\uff09 UPADTE\uff08\u589e\u91cf\u62f7\u8d1d\uff09 DistCpMode.COPY 4.3.0+ void setNumWorkers(int numWorkers) \u53ef\u9009 \u8bbe\u7f6eDistCp\u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\uff0c\u5bf9\u5e94MR\u4efb\u52a1\u4e2d\u7684 mapreduce.job.maps 10 4.3.0+ void setBandWidthLimit(int bandWidthLimit) \u53ef\u9009 \u8bbe\u7f6e\u5355\u4e2a\u8282\u70b9\u7684\u5e26\u5bbd\u9650\u5236\uff0c\u5355\u4f4dM -1 4.3.0+ void setWorkTempDir(String workTempDir) \u53ef\u9009 \u8bbe\u7f6e\u4e34\u65f6\u76ee\u5f55 /tmp 4.3.0+ void setOutputCodec(String outputCodec) \u53ef\u9009 \u8bbe\u7f6e\u538b\u7f29\u7c7b\u578b\uff0c\u652f\u6301\u7f16\u89e3\u7801\u5668\u6709 gzip\u3001gz\u3001lzo\u3001lzop\u3001lzop\u3001snappy keep\uff08\u4e0d\u66f4\u6539\u538b\u7f29\u7c7b\u578b\uff09 4.3.0+ void setEnableCMS(boolean enableCMS) \u53ef\u9009 \u662f\u5426\u5f00\u542f\u76d1\u63a7\u544a\u8b66 false 4.3.0+ void setDisableChecksum(boolean disableChecksum) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5173\u95edChecksum\u68c0\u67e5 false 4.3.0+ void setDeleteOnSuccess(boolean deleteOnSuccess) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5220\u9664\u6e90\u6587\u4ef6\uff0c\u7528\u4e8e\u79fb\u52a8\u6570\u636e false 4.3.0+ void setIncludeMatcherFile(String includeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setExcludeMatcherFile(String excludeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setIgnoreFailures(boolean ignoreFailures) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5ffd\u7565\u62f7\u8d1d\u4efb\u52a1\u4e2d\u629b\u51fa\u7684\u5f02\u5e38\uff0c\u907f\u514d\u4e2d\u65ad\u4efb\u52a1 False 4.3.0+ void setStorageClass(String storageClass) \u53ef\u9009 \u8bbe\u7f6e\u76ee\u6807\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301Standard\u3001IA\u3001Archive\u3001ColdArchive Standard 4.3.0+ void setEnableTransaction(boolean enableTransaction) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4e8b\u52a1\uff0c\u4ee5\u4fdd\u8bc1Job\u7ea7\u522b\u7684\u539f\u5b50\u6027 false 4.3.0+ void setEnablePreserveMetaStatus(boolean enablePreserveMetaStatus) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f false 4.4.0+ \u54cd\u5e94\u53c2\u6570 \u00a7 \u65b9\u6cd5 \u8bf4\u660e \u9ed8\u8ba4\u503c int getErrorCode() \u83b7\u53d6\u9519\u8bef\u7801 0 String getErrorMsg() \u83b7\u53d6\u9519\u8bef\u539f\u56e0 - long getStartTime() \u83b7\u53d6\u4efb\u52a1\u7684\u5f00\u59cb\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - long getEndTime() \u83b7\u53d6\u4efb\u52a1\u7684\u7ed3\u675f\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - getCount(String key) \u83b7\u53d6\u8ba1\u6570\u5668\u503c\uff0ckey\u4e3a\u8ba1\u6570\u5668\u540d\u79f0\uff0c - \u8ba1\u6570\u5668\u8bf4\u660e \u00a7 \u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf BYTES_EXPECTED \u9884\u671f\u7684copy\u5b57\u8282\u6570 FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 BYTES_COPIED copy\u6210\u529f\u7684\u5b57\u8282\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"JindoDistCp API \u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_api/#jindodistcp-api","text":"","title":"JindoDistCp API \u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_api/#jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"\u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_api/#_1","text":"\u76ee\u524d JindoSDK/JindoDistCP \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u4ee54.6.12\u7248\u672c\u4e3a\u4f8b\uff0c\u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoDistCP \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-distcp --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop with provided scope, your runtime should install hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> </dependencies> </project>","title":"\u5b89\u88c5\u4f9d\u8d56\u5305"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_api/#_2","text":"\u4ee5\u62f7\u8d1d HDFS \u6587\u4ef6\u5939\u5230\u963f\u91cc\u4e91 OSS \u4e3a\u4f8b\u3002 import com.aliyun.jindodata.distcp.util.DistCpCounter; import com.aliyun.jindodata.distjob.request.DistJobResponse; import org.apache.hadoop.conf.Configuration; import java.io.IOException; public class DistcpExample { static public void main(String[] args) throws IOException { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. conf.set(\"fs.oss.credentials.provider\", \"com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider\"); conf.set(\"fs.oss.accessKeyId\", \"XXX\"); conf.set(\"fs.oss.accessKeySecret\", \"XXX\"); DistCpRequest request = new DistCpRequest(); request.setSrcDir(\"hdfs:///user/root/random-data\"); request.setDestDir(\"oss://dls-bucket.cn-xxx.oss-dls.aliyuncs.com/dest\"); request.setWorkTempDir(\"hdfs:///user/root/tmp\"); request.setConf(conf); // Jindo DistCp Service DistCpService service = new DistCpService(); service.buildAllJobs(request); service.runAllJobs(); DistJobResponse response = service.collectAllJobResult(); printResponse(response); } static public void printResponse(DistJobResponse response) { System.out.println(\"ErrorCode: \" + response.getErrorCode()); System.out.println(\"ErrorMsg: \" + response.getErrorMsg()); System.out.println(\"FileCopied: \" + response.getCount(DistCpCounter.FILES_COPIED.getName())); System.out.println(\"BytesCopied: \" + response.getCount(DistCpCounter.BYTES_COPIED.getName())); } }","title":"\u793a\u4f8b\u4ee3\u7801"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_api/#_3","text":"\u76f8\u5173\u65b9\u6cd5\u7684\u5177\u4f53\u63cf\u8ff0\u53ef\u53c2\u8003 \u300a\u4f7f\u7528Jindo DistCp\u300b \u65b9\u6cd5 \u53c2\u6570\u7c7b\u578b \u8bf4\u660e \u9ed8\u8ba4\u503c \u517c\u5bb9 void setSrc(String src) \u5fc5\u9009 \u8bbe\u7f6e\u6e90\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setDest(String dest) \u5fc5\u9009 \u8bbe\u7f6e\u76ee\u6807\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setConf(Configuration conf) \u5fc5\u9009 \u8bbe\u7f6e Configuration - 4.3.0+ void setMode(DistCpMode mode) \u53ef\u9009 \u8bbe\u7f6eDistCp\u6a21\u5f0f\uff0c\u652f\u6301\u7684\u6a21\u5f0f\u6709 COPY\uff08\u62f7\u8d1d\uff09 DIFF\uff08\u5dee\u5f02\u6bd4\u8f83\uff09 UPADTE\uff08\u589e\u91cf\u62f7\u8d1d\uff09 DistCpMode.COPY 4.3.0+ void setNumWorkers(int numWorkers) \u53ef\u9009 \u8bbe\u7f6eDistCp\u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\uff0c\u5bf9\u5e94MR\u4efb\u52a1\u4e2d\u7684 mapreduce.job.maps 10 4.3.0+ void setBandWidthLimit(int bandWidthLimit) \u53ef\u9009 \u8bbe\u7f6e\u5355\u4e2a\u8282\u70b9\u7684\u5e26\u5bbd\u9650\u5236\uff0c\u5355\u4f4dM -1 4.3.0+ void setWorkTempDir(String workTempDir) \u53ef\u9009 \u8bbe\u7f6e\u4e34\u65f6\u76ee\u5f55 /tmp 4.3.0+ void setOutputCodec(String outputCodec) \u53ef\u9009 \u8bbe\u7f6e\u538b\u7f29\u7c7b\u578b\uff0c\u652f\u6301\u7f16\u89e3\u7801\u5668\u6709 gzip\u3001gz\u3001lzo\u3001lzop\u3001lzop\u3001snappy keep\uff08\u4e0d\u66f4\u6539\u538b\u7f29\u7c7b\u578b\uff09 4.3.0+ void setEnableCMS(boolean enableCMS) \u53ef\u9009 \u662f\u5426\u5f00\u542f\u76d1\u63a7\u544a\u8b66 false 4.3.0+ void setDisableChecksum(boolean disableChecksum) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5173\u95edChecksum\u68c0\u67e5 false 4.3.0+ void setDeleteOnSuccess(boolean deleteOnSuccess) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5220\u9664\u6e90\u6587\u4ef6\uff0c\u7528\u4e8e\u79fb\u52a8\u6570\u636e false 4.3.0+ void setIncludeMatcherFile(String includeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setExcludeMatcherFile(String excludeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setIgnoreFailures(boolean ignoreFailures) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5ffd\u7565\u62f7\u8d1d\u4efb\u52a1\u4e2d\u629b\u51fa\u7684\u5f02\u5e38\uff0c\u907f\u514d\u4e2d\u65ad\u4efb\u52a1 False 4.3.0+ void setStorageClass(String storageClass) \u53ef\u9009 \u8bbe\u7f6e\u76ee\u6807\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301Standard\u3001IA\u3001Archive\u3001ColdArchive Standard 4.3.0+ void setEnableTransaction(boolean enableTransaction) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4e8b\u52a1\uff0c\u4ee5\u4fdd\u8bc1Job\u7ea7\u522b\u7684\u539f\u5b50\u6027 false 4.3.0+ void setEnablePreserveMetaStatus(boolean enablePreserveMetaStatus) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f false 4.4.0+","title":"\u8bf7\u6c42\u53c2\u6570"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_api/#_4","text":"\u65b9\u6cd5 \u8bf4\u660e \u9ed8\u8ba4\u503c int getErrorCode() \u83b7\u53d6\u9519\u8bef\u7801 0 String getErrorMsg() \u83b7\u53d6\u9519\u8bef\u539f\u56e0 - long getStartTime() \u83b7\u53d6\u4efb\u52a1\u7684\u5f00\u59cb\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - long getEndTime() \u83b7\u53d6\u4efb\u52a1\u7684\u7ed3\u675f\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - getCount(String key) \u83b7\u53d6\u8ba1\u6570\u5668\u503c\uff0ckey\u4e3a\u8ba1\u6570\u5668\u540d\u79f0\uff0c -","title":"\u54cd\u5e94\u53c2\u6570"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_api/#_5","text":"\u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf BYTES_EXPECTED \u9884\u671f\u7684copy\u5b57\u8282\u6570 FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 BYTES_COPIED copy\u6210\u529f\u7684\u5b57\u8282\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"\u8ba1\u6570\u5668\u8bf4\u660e"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/","text":"JindoDistCp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66 \u00a7 JindoDistCp\uff08\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u5de5\u5177\uff09\u662f\u7528\u4e8e\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u62f7\u8d1d\u6587\u4ef6\u7684\u5de5\u5177\u3002\u4f7f\u7528MapReduce\u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3amap/reduce\u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002 CMS\uff08\u4e91\u76d1\u63a7\u670d\u52a1\uff09\uff0c\u53ef\u7528\u4e8e\u6536\u96c6\u963f\u91cc\u4e91\u8d44\u6e90\u7684\u76d1\u63a7\u6307\u6807\u6216\u7528\u6237\u81ea\u5b9a\u4e49\u7684\u76d1\u63a7\u6307\u6807\uff0c\u63a2\u6d4b\u670d\u52a1\u53ef\u7528\u6027\uff0c\u4ee5\u53ca\u9488\u5bf9\u6307\u6807\u8bbe\u7f6e\u8b66\u62a5\u3002\u4f7f\u60a8\u5168\u9762\u4e86\u89e3\u963f\u91cc\u4e91\u4e0a\u8d44\u6e90\u7684\u4f7f\u7528\u60c5\u51b5\u548c\u4e1a\u52a1\u8fd0\u884c\u72b6\u51b5\uff0c\u5e76\u53ca\u65f6\u5bf9\u6545\u969c\u8d44\u6e90\u8fdb\u884c\u5904\u7406\uff0c\u4fdd\u8bc1\u4e1a\u52a1\u6b63\u5e38\u8fd0\u884c\u3002 \u60a8\u53ef\u4ee5\u6307\u5b9a\u672c\u6b21JindoDistCp\u4efb\u52a1\u7ed3\u675f\u540e\u662f\u5426\u9700\u8981\u5f00\u542fCMS\uff0c\u4ee5\u4e0a\u62a5\u4efb\u52a1\u5931\u8d25\u4fe1\u606f\uff0c\u5e76\u4f7f\u7528CMS\u63a7\u5236\u53f0\u6765\u914d\u7f6e\u544a\u8b66\u529f\u80fd\u3002\u544a\u8b66\u6761\u4ef6\u53c2\u89c1 \u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668 \u3002 \u6b65\u9aa4 \u00a7 1. \u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u548c\u62a5\u8b66\u8054\u7cfb\u7ec4 \u00a7 \u5177\u4f53\u8bf7\u53c2\u8003\u6587\u6863 \u300a\u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u6216\u62a5\u8b66\u8054\u7cfb\u7ec4\u300b 2. \u83b7\u53d6\u62a5\u8b66Token \u00a7 \u70b9\u51fb\u5de6\u4fa7\u5bfc\u822a\u680f\u201c\u62a5\u8b66\u670d\u52a1\u201d->\"\u62a5\u8b66\u8054\u7cfb\u4eba\"->\u5207\u6362\u4e0a\u6d4b\u9875\u7b7e\u5230\"\u62a5\u8b66\u8054\u7cfb\u7ec4\"->\u9009\u62e9\u5bf9\u5e94\u7684\u62a5\u8b66\u8054\u7cfb\u7ec4->\u70b9\u51fb\u53f3\u4fa7\u63a5\u5165\u5916\u90e8\u62a5\u8b66 3. \u914d\u7f6e\u73af\u5883\u53d8\u91cf \u00a7 \u73af\u5883\u53d8\u91cf \u8bf4\u660e cmsAccessKeyId \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94AccessKey ID cmsAccessSecret \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94Access Key Secret cmsRegion \u8bbe\u7f6eCMS\u6240\u5728Region\uff0c\u9700\u8981\u4e0e\u5176\u4ed6\u76f8\u5173\u7684\u4e91\u4e0a\u8d44\u6e90\u4e00\u81f4\uff0c\u4f8b\u5982ECS\uff0cOSS\uff0cEMR\u6240\u5bf9\u5e94\u7684Region cmsLevel \u8bbe\u7f6e\u6807\u8bc6\u544a\u8b66\u901a\u77e5\u65b9\u5f0f\uff0c\u5982\u679c\u672a\u8bbe\u7f6e\uff0c\u9ed8\u8ba4\u4e3aWARN\u3002INFO\u4e3a \u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cWARN\u4e3a \u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cCRITICAL\u4e3a \u7535\u8bdd+\u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \u793a\u4f8b\u5982\u4e0b export cmsAccessKeyId=<your_key_id> export cmsAccessSecret=<your_key_secret> export cmsRegion=cn-hangzhou export cmsToken=<your_cms_token> export cmsLevel=WARN 4. \u5f00\u542fCMS\u5bf9\u5931\u8d25\u4efb\u52a1\u8fdb\u884c\u544a\u8b66 \u00a7 copy\u64cd\u4f5c \u00a7 hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table \\ --enableCMS diff\u64cd\u4f5c \u00a7 hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table \\ --diff \\ --enableCMS update\u64cd\u4f5c \u00a7 hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table \\ --update \\ --enableCMS \u544a\u8b66\u5185\u5bb9 \u00a7 \u544a\u8b66\u5185\u5bb9\u683c\u5f0f \u00a7 { \"JobTime\": \"2021-03-09 19:45:27~2021-03-09 19:45:28\", \"SrcPath\": \"hdfs://cluster-xxx:9000//data/incoming/hourly_table\", \"DestPath\": \"oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table\", \"Counter\": { \"COUNTER_1\": XXX, \"COUNTER_2\": XXX, ... } } ... \u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668 \u00a7 \u544a\u8b66\u5185\u5bb9\u4e2d\u7684counter\u8ba1\u6570\u5668\u6709\u4ee5\u4e0b\u51e0\u79cd\uff1a copy\u64cd\u4f5c \u00a7 \u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 diff\u64cd\u4f5c \u00a7 \u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"JindoDistCp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#jindodistcpcms","text":"JindoDistCp\uff08\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u5de5\u5177\uff09\u662f\u7528\u4e8e\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u62f7\u8d1d\u6587\u4ef6\u7684\u5de5\u5177\u3002\u4f7f\u7528MapReduce\u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3amap/reduce\u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002 CMS\uff08\u4e91\u76d1\u63a7\u670d\u52a1\uff09\uff0c\u53ef\u7528\u4e8e\u6536\u96c6\u963f\u91cc\u4e91\u8d44\u6e90\u7684\u76d1\u63a7\u6307\u6807\u6216\u7528\u6237\u81ea\u5b9a\u4e49\u7684\u76d1\u63a7\u6307\u6807\uff0c\u63a2\u6d4b\u670d\u52a1\u53ef\u7528\u6027\uff0c\u4ee5\u53ca\u9488\u5bf9\u6307\u6807\u8bbe\u7f6e\u8b66\u62a5\u3002\u4f7f\u60a8\u5168\u9762\u4e86\u89e3\u963f\u91cc\u4e91\u4e0a\u8d44\u6e90\u7684\u4f7f\u7528\u60c5\u51b5\u548c\u4e1a\u52a1\u8fd0\u884c\u72b6\u51b5\uff0c\u5e76\u53ca\u65f6\u5bf9\u6545\u969c\u8d44\u6e90\u8fdb\u884c\u5904\u7406\uff0c\u4fdd\u8bc1\u4e1a\u52a1\u6b63\u5e38\u8fd0\u884c\u3002 \u60a8\u53ef\u4ee5\u6307\u5b9a\u672c\u6b21JindoDistCp\u4efb\u52a1\u7ed3\u675f\u540e\u662f\u5426\u9700\u8981\u5f00\u542fCMS\uff0c\u4ee5\u4e0a\u62a5\u4efb\u52a1\u5931\u8d25\u4fe1\u606f\uff0c\u5e76\u4f7f\u7528CMS\u63a7\u5236\u53f0\u6765\u914d\u7f6e\u544a\u8b66\u529f\u80fd\u3002\u544a\u8b66\u6761\u4ef6\u53c2\u89c1 \u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668 \u3002","title":"JindoDistCp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#1","text":"\u5177\u4f53\u8bf7\u53c2\u8003\u6587\u6863 \u300a\u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u6216\u62a5\u8b66\u8054\u7cfb\u7ec4\u300b","title":"1. \u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u548c\u62a5\u8b66\u8054\u7cfb\u7ec4"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#2-token","text":"\u70b9\u51fb\u5de6\u4fa7\u5bfc\u822a\u680f\u201c\u62a5\u8b66\u670d\u52a1\u201d->\"\u62a5\u8b66\u8054\u7cfb\u4eba\"->\u5207\u6362\u4e0a\u6d4b\u9875\u7b7e\u5230\"\u62a5\u8b66\u8054\u7cfb\u7ec4\"->\u9009\u62e9\u5bf9\u5e94\u7684\u62a5\u8b66\u8054\u7cfb\u7ec4->\u70b9\u51fb\u53f3\u4fa7\u63a5\u5165\u5916\u90e8\u62a5\u8b66","title":"2. \u83b7\u53d6\u62a5\u8b66Token"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#3","text":"\u73af\u5883\u53d8\u91cf \u8bf4\u660e cmsAccessKeyId \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94AccessKey ID cmsAccessSecret \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94Access Key Secret cmsRegion \u8bbe\u7f6eCMS\u6240\u5728Region\uff0c\u9700\u8981\u4e0e\u5176\u4ed6\u76f8\u5173\u7684\u4e91\u4e0a\u8d44\u6e90\u4e00\u81f4\uff0c\u4f8b\u5982ECS\uff0cOSS\uff0cEMR\u6240\u5bf9\u5e94\u7684Region cmsLevel \u8bbe\u7f6e\u6807\u8bc6\u544a\u8b66\u901a\u77e5\u65b9\u5f0f\uff0c\u5982\u679c\u672a\u8bbe\u7f6e\uff0c\u9ed8\u8ba4\u4e3aWARN\u3002INFO\u4e3a \u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cWARN\u4e3a \u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cCRITICAL\u4e3a \u7535\u8bdd+\u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \u793a\u4f8b\u5982\u4e0b export cmsAccessKeyId=<your_key_id> export cmsAccessSecret=<your_key_secret> export cmsRegion=cn-hangzhou export cmsToken=<your_cms_token> export cmsLevel=WARN","title":"3. \u914d\u7f6e\u73af\u5883\u53d8\u91cf"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#4-cms","text":"","title":"4. \u5f00\u542fCMS\u5bf9\u5931\u8d25\u4efb\u52a1\u8fdb\u884c\u544a\u8b66"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#copy","text":"hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table \\ --enableCMS","title":"copy\u64cd\u4f5c"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#diff","text":"hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table \\ --diff \\ --enableCMS","title":"diff\u64cd\u4f5c"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#update","text":"hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table \\ --update \\ --enableCMS","title":"update\u64cd\u4f5c"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#_2","text":"","title":"\u544a\u8b66\u5185\u5bb9"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#_3","text":"{ \"JobTime\": \"2021-03-09 19:45:27~2021-03-09 19:45:28\", \"SrcPath\": \"hdfs://cluster-xxx:9000//data/incoming/hourly_table\", \"DestPath\": \"oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table\", \"Counter\": { \"COUNTER_1\": XXX, \"COUNTER_2\": XXX, ... } } ...","title":"\u544a\u8b66\u5185\u5bb9\u683c\u5f0f"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#_4","text":"\u544a\u8b66\u5185\u5bb9\u4e2d\u7684counter\u8ba1\u6570\u5668\u6709\u4ee5\u4e0b\u51e0\u79cd\uff1a","title":"\u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#copy_1","text":"\u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570","title":"copy\u64cd\u4f5c"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#diff_1","text":"\u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"diff\u64cd\u4f5c"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/","text":"\u6570\u636e\u5728 OSS-HDFS \u670d\u52a1\u4e0d\u540c bucket \u4e4b\u95f4\u8fc1\u79fb \u00a7 \u4f7f\u7528\u524d\u987b\u77e5 \u00a7 \u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\uff0c\u4e5f\u53ef \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988 1\u3001\u62f7\u8d1d\u6570\u636e\u5230 OSS-HDFS \u670d\u52a1\u4e0a \u00a7 \u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS-HDFS \u670d\u52a1\u4e0a\u3002 \u5982\u679c\u60a8\u7684\u4e24\u4e2a\u8981\u62f7\u8d1d\u7684 bucket \u90fd\u5728\u540c\u4e00\u4e2a region \u4e0b\uff0c\u4e14\u53ef\u7528\u540c\u4e00\u4e2a ossKey \u548c ossSecret \u8fdb\u884c\u8bfb\u5199\uff0c\u60a8\u53ef\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket.cn-xxx.oss-dls.aliyuncs.com/ --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src OSS-HDFS \u670d\u52a1\u7684\u6e90\u8def\u5f84\u3002 oss://srcBucket.cn-xxx.oss-dls.aliyuncs.com/ --dest OSS-HDFS \u670d\u52a1\u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/ --hadoopConf \u6307\u5b9a OSS-HDFS \u670d\u52a1\u7684 Access Key ID , Access Key Secret * \u914d\u7f6e OSS-HDFS \u670d\u52a1\u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS-HDFS \u670d\u52a1\u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10 \u5982\u679c\u60a8\u7684 src \u548c dest \u7684 bucket \u4e0d\u5728\u540c\u4e00\u4e2a region \u6216\u8005\u4e0d\u80fd\u4f7f\u7528\u540c\u4e00\u4e2a ossKey \u548c ossSecret \u8fdb\u884c\u8bfb\u5199\uff0c\u90a3\u4e48\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcbucket.oss-cn-xxx.aliyuncs.com/ --dest oss://destBucket.oss-cn-xxx.aliyuncs.com/ --hadoopConf fs.oss.bucket.srcbucket.accessKeyId=yourkey --hadoopConf fs.oss.bucket.srcbucket.accessKeySecret=yoursecret --hadoopConf --hadoopConf fs.oss.bucket.destBucket.accessKeyId=yourkey --hadoopConf fs.oss.bucket.destBucket.accessKeySecret=yoursecret --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --hadoopConf \u7ed9 src \u548c dest \u7684 bucket \u6307\u5b9a\u4e0d\u540c Access Key ID , Access Key Secret * \u914d\u7f6e XXX bucket \u7684 AccessKeyId: --hadoopConf fs.oss.bucket.XXX.accessKeyId=yourkey * \u914d\u7f6e OSS-HDFS \u670d\u52a1\u7684 AccessKeySecret: --hadoopConf fs.oss.bucket.XXX.accessKeySecret=yoursecret 2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6 \u00a7 \u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53ea Copy \u5269\u4e0b\u672a Copy \u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a \u4f7f\u7528 --update \u547d\u4ee4\uff0c\u83b7\u5f97\u589e\u91cf\u7684\u6587\u4ef6\u5217\u8868 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket.oss-cn-xxx.aliyuncs.com/ --dest oss://destBucket.oss-cn-xxx.aliyuncs.com/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --update --parallelism 20 3\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9 \u00a7 \u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket.oss-cn-xxx.aliyuncs.com/ --dest oss://destBucket.oss-cn-xxx.aliyuncs.com/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB 4\u3001\u514d\u5bc6\u53ca\u5bc6\u94a5\u56fa\u5b9a\u5b58\u50a8 \u00a7 \u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS-HDFS \u670d\u52a1\u7684 AccessKey/AccessSecret \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662fJindoDistCp\u53ef\u4ee5\u5c06\u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> 5\u3001\u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u00a7 \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 JindoFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 6\u3001\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f \u00a7 \u4f7f\u7528preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket.oss-cn-xxx.aliyuncs.com/ --dest oss://destBucket.oss-cn-xxx.aliyuncs.com/ --preserveMeta 7\u3001\u5176\u4ed6\u529f\u80fd \u00a7 \u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 * JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e","title":"\u6570\u636e\u5728 OSS-HDFS \u670d\u52a1\u4e0d\u540c bucket \u4e4b\u95f4\u8fc1\u79fb"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/#oss-hdfs-bucket","text":"","title":"\u6570\u636e\u5728 OSS-HDFS \u670d\u52a1\u4e0d\u540c bucket \u4e4b\u95f4\u8fc1\u79fb"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/#_1","text":"\u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\uff0c\u4e5f\u53ef \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988","title":"\u4f7f\u7528\u524d\u987b\u77e5"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/#1-oss-hdfs","text":"\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS-HDFS \u670d\u52a1\u4e0a\u3002 \u5982\u679c\u60a8\u7684\u4e24\u4e2a\u8981\u62f7\u8d1d\u7684 bucket \u90fd\u5728\u540c\u4e00\u4e2a region \u4e0b\uff0c\u4e14\u53ef\u7528\u540c\u4e00\u4e2a ossKey \u548c ossSecret \u8fdb\u884c\u8bfb\u5199\uff0c\u60a8\u53ef\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket.cn-xxx.oss-dls.aliyuncs.com/ --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src OSS-HDFS \u670d\u52a1\u7684\u6e90\u8def\u5f84\u3002 oss://srcBucket.cn-xxx.oss-dls.aliyuncs.com/ --dest OSS-HDFS \u670d\u52a1\u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/ --hadoopConf \u6307\u5b9a OSS-HDFS \u670d\u52a1\u7684 Access Key ID , Access Key Secret * \u914d\u7f6e OSS-HDFS \u670d\u52a1\u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS-HDFS \u670d\u52a1\u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10 \u5982\u679c\u60a8\u7684 src \u548c dest \u7684 bucket \u4e0d\u5728\u540c\u4e00\u4e2a region \u6216\u8005\u4e0d\u80fd\u4f7f\u7528\u540c\u4e00\u4e2a ossKey \u548c ossSecret \u8fdb\u884c\u8bfb\u5199\uff0c\u90a3\u4e48\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcbucket.oss-cn-xxx.aliyuncs.com/ --dest oss://destBucket.oss-cn-xxx.aliyuncs.com/ --hadoopConf fs.oss.bucket.srcbucket.accessKeyId=yourkey --hadoopConf fs.oss.bucket.srcbucket.accessKeySecret=yoursecret --hadoopConf --hadoopConf fs.oss.bucket.destBucket.accessKeyId=yourkey --hadoopConf fs.oss.bucket.destBucket.accessKeySecret=yoursecret --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --hadoopConf \u7ed9 src \u548c dest \u7684 bucket \u6307\u5b9a\u4e0d\u540c Access Key ID , Access Key Secret * \u914d\u7f6e XXX bucket \u7684 AccessKeyId: --hadoopConf fs.oss.bucket.XXX.accessKeyId=yourkey * \u914d\u7f6e OSS-HDFS \u670d\u52a1\u7684 AccessKeySecret: --hadoopConf fs.oss.bucket.XXX.accessKeySecret=yoursecret","title":"1\u3001\u62f7\u8d1d\u6570\u636e\u5230 OSS-HDFS \u670d\u52a1\u4e0a"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/#2","text":"\u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53ea Copy \u5269\u4e0b\u672a Copy \u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a","title":"2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/#-update","text":"hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket.oss-cn-xxx.aliyuncs.com/ --dest oss://destBucket.oss-cn-xxx.aliyuncs.com/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --update --parallelism 20","title":"\u4f7f\u7528 --update \u547d\u4ee4\uff0c\u83b7\u5f97\u589e\u91cf\u7684\u6587\u4ef6\u5217\u8868"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/#3yarn","text":"\u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket.oss-cn-xxx.aliyuncs.com/ --dest oss://destBucket.oss-cn-xxx.aliyuncs.com/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB","title":"3\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/#4","text":"\u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS-HDFS \u670d\u52a1\u7684 AccessKey/AccessSecret \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662fJindoDistCp\u53ef\u4ee5\u5c06\u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration>","title":"4\u3001\u514d\u5bc6\u53ca\u5bc6\u94a5\u56fa\u5b9a\u5b58\u50a8"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/#5-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 JindoFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002","title":"5\u3001\u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/#6","text":"\u4f7f\u7528preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket.oss-cn-xxx.aliyuncs.com/ --dest oss://destBucket.oss-cn-xxx.aliyuncs.com/ --preserveMeta","title":"6\u3001\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/#7","text":"\u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 * JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e","title":"7\u3001\u5176\u4ed6\u529f\u80fd"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/","text":"\u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\u4e0a \u00a7 \u4f7f\u7528\u524d\u987b\u77e5 \u00a7 \u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\uff0c\u4e5f\u53ef \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988 1\u3001\u62f7\u8d1d\u6570\u636e\u5230\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a \u00a7 \u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06 HDFS \u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS-HDFS \u670d\u52a1\u4e0a hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src HDFS \u7684\u6e90\u8def\u5f84\u3002 /data --dest OSS-HDFS \u670d\u52a1\u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/ --hadoopConf \u6307\u5b9a OSS-HDFS \u670d\u52a1\u7684 Access Key ID , Access Key Secret * \u914d\u7f6e OSS-HDFS \u670d\u52a1\u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS-HDFS \u670d\u52a1 \u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10 2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6 \u00a7 \u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53eaCopy\u5269\u4e0b\u672aCopy\u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a \u4f7f\u7528 --update \u547d\u4ee4\uff0c\u83b7\u5f97\u589e\u91cf\u7684\u6587\u4ef6\u5217\u8868 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --update --parallelism 20 3\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9 \u00a7 \u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB 4\u3001\u514d\u5bc6\u53ca\u5bc6\u94a5\u56fa\u5b9a\u5b58\u50a8 \u00a7 \u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS-HDFS \u670d\u52a1 AccessKey/AccessSecret \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662fJindoDistCp\u53ef\u4ee5\u5c06 \u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> 5\u3001\u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u00a7 \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 JindoFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u6ce8\u610f \uff1a JindoSDK 4.4.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u9ed8\u8ba4\u4f7f\u7528\u57df\u540d\u5206\u79bb\u7684\u65b9\u5f0f\uff0c\u6570\u636e\u8bfb\u5199\u57df\u540d\u9ed8\u8ba4\u4f7f\u7528\u5185\u7f51\u6807\u51c6 OSS \u57df\u540d\uff0c\u5982\u679c\u5728\u975e\u963f\u91cc\u4e91\u5185\u7f51\u73af\u5883\u6267\u884c distcp \u547d\u4ee4\uff0c\u5219\u9700\u8981\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u914d\u7f6e\u6570\u636e\u6d41\u8bbf\u95ee\u7684\u516c\u7f51 OSS endpoint\u3002 <configuration> <property> <name>fs.oss.data.endpoint</name> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> 6\u3001\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f \u00a7 \u4f7f\u7528preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --preserveMeta 7\u3001\u5176\u4ed6\u529f\u80fd \u00a7 \u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 * JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e","title":"\u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\u4e0a"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/#hdfs-oss-hdfs","text":"","title":"\u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\u4e0a"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/#_1","text":"\u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\uff0c\u4e5f\u53ef \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988","title":"\u4f7f\u7528\u524d\u987b\u77e5"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/#1-oss-hdfs-jindofs","text":"\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06 HDFS \u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS-HDFS \u670d\u52a1\u4e0a hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src HDFS \u7684\u6e90\u8def\u5f84\u3002 /data --dest OSS-HDFS \u670d\u52a1\u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/ --hadoopConf \u6307\u5b9a OSS-HDFS \u670d\u52a1\u7684 Access Key ID , Access Key Secret * \u914d\u7f6e OSS-HDFS \u670d\u52a1\u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS-HDFS \u670d\u52a1 \u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10","title":"1\u3001\u62f7\u8d1d\u6570\u636e\u5230\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/#2","text":"\u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53eaCopy\u5269\u4e0b\u672aCopy\u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a","title":"2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/#-update","text":"hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --update --parallelism 20","title":"\u4f7f\u7528 --update \u547d\u4ee4\uff0c\u83b7\u5f97\u589e\u91cf\u7684\u6587\u4ef6\u5217\u8868"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/#3yarn","text":"\u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB","title":"3\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/#4","text":"\u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS-HDFS \u670d\u52a1 AccessKey/AccessSecret \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662fJindoDistCp\u53ef\u4ee5\u5c06 \u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration>","title":"4\u3001\u514d\u5bc6\u53ca\u5bc6\u94a5\u56fa\u5b9a\u5b58\u50a8"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/#5-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 JindoFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u6ce8\u610f \uff1a JindoSDK 4.4.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u9ed8\u8ba4\u4f7f\u7528\u57df\u540d\u5206\u79bb\u7684\u65b9\u5f0f\uff0c\u6570\u636e\u8bfb\u5199\u57df\u540d\u9ed8\u8ba4\u4f7f\u7528\u5185\u7f51\u6807\u51c6 OSS \u57df\u540d\uff0c\u5982\u679c\u5728\u975e\u963f\u91cc\u4e91\u5185\u7f51\u73af\u5883\u6267\u884c distcp \u547d\u4ee4\uff0c\u5219\u9700\u8981\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u914d\u7f6e\u6570\u636e\u6d41\u8bbf\u95ee\u7684\u516c\u7f51 OSS endpoint\u3002 <configuration> <property> <name>fs.oss.data.endpoint</name> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration>","title":"5\u3001\u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/#6","text":"\u4f7f\u7528preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --preserveMeta","title":"6\u3001\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/#7","text":"\u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 * JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e","title":"7\u3001\u5176\u4ed6\u529f\u80fd"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_overview/","text":"JindoDistCp \u6570\u636e\u8fc1\u79fb \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 JDK 1.8\u53ca\u4ee5\u4e0a Hadoop 2.3+\u7248\u672c\uff0c\u8bf7\u4e0b\u8f7d\u6700\u65b0\u7248\u7684 jindo-distcp-tool-x.x.x.jar \uff0c\u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz \u5185\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 tools/ \u76ee\u5f55\u4e0b\u627e\u5230\uff0c \u4e0b\u8f7d\u9875\u9762 \u3002 \u8be5\u7248\u672c\u57fa\u4e8e native \u4ee3\u7801\u5b9e\u73b0\uff0c\u652f\u6301\u529f\u80fd\u8f83\u4e30\u5bcc (2.3 \u4ee5\u524d\u7248\u672c\u6682\u672a\u6d4b\u8bd5\uff0c\u5982\u6709\u95ee\u9898\u8bf7 \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988) JindoDistCp \u4f7f\u7528\u573a\u666f \u00a7 \u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\u4e0a \u6570\u636e\u5728 OSS-HDFS \u670d\u52a1\u4e0d\u540c bucket \u4e4b\u95f4\u8fc1\u79fb JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u00a7 \u6211\u4eec\u603b\u7ed3\u4e86\u4e00\u4e9b\u5728\u4e0d\u540c\u73af\u5883\u4f7f\u7528 JindoDistCp \u7528\u6237\u53ef\u80fd\u4f1a\u9047\u5230\u7684\u4e00\u4e9b\u95ee\u9898\uff0c\u60a8\u53ef\u4ee5\u53c2\u7167\u4e0b\u6587\u8fdb\u884c\u5e38\u89c1\u95ee\u9898\u6392\u67e5\u548c\u89e3\u51b3 * JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 JindoDistCp \u7cfb\u5217\u6587\u7ae0 \u00a7 \u300a\u518d\u51fa\u738b\u724c\uff1a\u963f\u91cc\u4e91 JindoDistCp \u5168\u9762\u5f00\u653e\u4f7f\u7528\uff0c\u6210\u4e3a\u963f\u91cc\u4e91\u6570\u636e\u8fc1\u79fb\u5229\u5668\u300b JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e (\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177)","title":"JindoDistCp \u6570\u636e\u8fc1\u79fb"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_overview/#jindodistcp","text":"","title":"JindoDistCp \u6570\u636e\u8fc1\u79fb"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_overview/#_1","text":"JDK 1.8\u53ca\u4ee5\u4e0a Hadoop 2.3+\u7248\u672c\uff0c\u8bf7\u4e0b\u8f7d\u6700\u65b0\u7248\u7684 jindo-distcp-tool-x.x.x.jar \uff0c\u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz \u5185\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 tools/ \u76ee\u5f55\u4e0b\u627e\u5230\uff0c \u4e0b\u8f7d\u9875\u9762 \u3002 \u8be5\u7248\u672c\u57fa\u4e8e native \u4ee3\u7801\u5b9e\u73b0\uff0c\u652f\u6301\u529f\u80fd\u8f83\u4e30\u5bcc (2.3 \u4ee5\u524d\u7248\u672c\u6682\u672a\u6d4b\u8bd5\uff0c\u5982\u6709\u95ee\u9898\u8bf7 \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988)","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_overview/#jindodistcp_1","text":"\u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\u4e0a \u6570\u636e\u5728 OSS-HDFS \u670d\u52a1\u4e0d\u540c bucket \u4e4b\u95f4\u8fc1\u79fb","title":"JindoDistCp \u4f7f\u7528\u573a\u666f"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_overview/#jindodistcp_2","text":"\u6211\u4eec\u603b\u7ed3\u4e86\u4e00\u4e9b\u5728\u4e0d\u540c\u73af\u5883\u4f7f\u7528 JindoDistCp \u7528\u6237\u53ef\u80fd\u4f1a\u9047\u5230\u7684\u4e00\u4e9b\u95ee\u9898\uff0c\u60a8\u53ef\u4ee5\u53c2\u7167\u4e0b\u6587\u8fdb\u884c\u5e38\u89c1\u95ee\u9898\u6392\u67e5\u548c\u89e3\u51b3 * JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357","title":"JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_overview/#jindodistcp_3","text":"\u300a\u518d\u51fa\u738b\u724c\uff1a\u963f\u91cc\u4e91 JindoDistCp \u5168\u9762\u5f00\u653e\u4f7f\u7528\uff0c\u6210\u4e3a\u963f\u91cc\u4e91\u6570\u636e\u8fc1\u79fb\u5229\u5668\u300b JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e (\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177)","title":"JindoDistCp \u7cfb\u5217\u6587\u7ae0"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/","text":"\u4f7f\u7528 JindoFuse \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u901a\u8fc7 JindoFuse \u63d0\u4f9b POSIX \u652f\u6301\u3002JindoFuse \u53ef\u4ee5\u628a OSS-HDFS \u670d\u52a1\u4e0a\u7684\u6587\u4ef6\u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c\u8be5\u670d\u52a1\u4e0a\u7684\u6587\u4ef6\u3002 \u57fa\u672c\u4f7f\u7528 \u00a7 \u73af\u5883\u51c6\u5907 \u00a7 # CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev \u914d\u7f6e\u5ba2\u6237\u7aef \u00a7 \u914d\u7f6e\u76ee\u5f55 \u00a7 \u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf \u914d\u7f6e\u6587\u4ef6 \u00a7 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u5f00\u542fHDFS\u670d\u52a1\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> \u514d\u5bc6\u8bbf\u95ee \u00a7 \u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7\u89d2\u8272\u6388\u6743\u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u5f00\u542fHDFS\u670d\u52a1\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE fs.oss.provider.format = JSON \u6302\u8f7d JindoFuse \u00a7 \u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002 \u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 mkdir -p <mount_point> \u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 jindo-fuse <mount_point> -ouri=[<oss_path>] -ouri \u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684 oss \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a Bucket \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002 \u8bbf\u95ee JindoFuse \u00a7 \u5982\u679c\u5c06 OSS-HDFS \u670d\u52a1\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1 ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/ \u5378\u8f7d JindoFuse \u00a7 \u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point> \u81ea\u52a8\u5378\u8f7d JindoFuse \u00a7 \u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u7279\u6027\u652f\u6301 \u00a7 \u76ee\u524d JindoFuse \u5df2\u7ecf\u652f\u6301\u4ee5\u4e0b POSIX API\uff1a \u7279\u6027 \u8bf4\u660e getattr() \u67e5\u8be2\u6587\u4ef6\u5c5e\u6027\uff0c\u7c7b\u4f3c ls mkdir() \u521b\u5efa\u76ee\u5f55\uff0c\u7c7b\u4f3c mkdir rmdir() \u5220\u9664\u76ee\u5f55\uff0c\u7c7b\u4f3c rm -rf unlink() \u5220\u9664\u6587\u4ef6\uff0c\u7c7b\u4f3c unlink rename() \u91cd\u547d\u540d\uff0c\u7c7b\u4f3c mv read() \u987a\u5e8f\u8bfb\u53d6 pread() \u968f\u673a\u8bfb write() \u987a\u5e8f\u5199 pwrite() \u968f\u673a\u5199 flush() \u5237\u65b0\u5185\u5b58\u5230\u5185\u6838\u7f13\u51b2\u533a fsync() \u5237\u65b0\u5185\u5b58\u5230\u78c1\u76d8 release() \u5173\u95ed\u6587\u4ef6 readdir() \u8bfb\u53d6\u76ee\u5f55 create() \u521b\u5efa\u6587\u4ef6 open() O_APPEND \u901a\u8fc7\u8ffd\u52a0\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 open() O_TRUNC \u901a\u8fc7\u8986\u76d6\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 ftruncate() \u5bf9\u6253\u5f00\u7684\u6587\u4ef6\u8fdb\u884c\u622a\u65ad truncate() \u5bf9\u672a\u6253\u5f00\u7684\u6587\u4ef6\u8fdb\u884c\u622a\u65ad\uff0c\u7c7b\u4f3c truncate -s lseek() \u6307\u5b9a\u6253\u5f00\u6587\u4ef6\u4e2d\u7684\u8bfb\u5199\u4f4d\u7f6e\u3002 chmod() \u4fee\u6539\u6587\u4ef6\u6743\u9650\uff0c\u7c7b\u4f3c chmod access() \u67e5\u8be2\u6587\u4ef6\u6743\u9650 utimes() \u4fee\u6539\u6587\u4ef6\u7684\u5b58\u53d6\u65f6\u95f4\u548c\u66f4\u6539\u65f6\u95f4(\u4ec5\u652f\u6301\u4fee\u6539\u4e3a\u66f4\u65b0\u7684\u65f6\u95f4) setxattr() \u4fee\u6539\u6587\u4ef6 xattr \u5c5e\u6027 getxattr() \u83b7\u53d6\u6587\u4ef6 xattr \u5c5e\u6027 listxattr() \u5217\u4e3e\u6587\u4ef6 xattr \u5c5e\u6027 removexattr() \u5220\u9664\u6587\u4ef6 xattr \u5c5e\u6027 lock() \u652f\u6301 posix \u9501\uff0c\u7c7b\u4f3c fcntl fallocate() \u4e3a\u6587\u4ef6\u9884\u5206\u914d\u7269\u7406\u7a7a\u95f4 symlink() \u521b\u5efa\u8f6f\u8fde\u63a5\uff0c\u76ee\u524d\u4ec5\u652f\u6301 OSS-HDFS \u5185\u90e8\u4f7f\u7528\uff0c\u4e14\u4e0d\u652f\u6301\u7f13\u5b58\u52a0\u901f readlink() \u8bfb\u53d6\u8f6f\u8fde\u63a5 \u9ad8\u9636\u4f7f\u7528 \u00a7 \u6302\u8f7d\u9009\u9879 \u00a7 \u53c2\u6570\u540d\u79f0 \u5fc5\u9009 \u53c2\u6570\u8bf4\u660e \u4f7f\u7528\u8303\u4f8b uri \u2713 \u914d\u7f6e\u9700\u8981\u6620\u5c04\u7684 oss \u8def\u5f84\u3002\u8def\u5f84\u53ef\u4ee5\u662f\u6839\u76ee\u5f55\uff0c\u4e5f\u53ef\u4ee5\u662f\u5b50\u76ee\u5f55\u3002\u4f8b\u5982\uff1aoss://bucket.endpoint/ \u6216 oss://bucket.endpoint/subdir\u3002 -ouri=oss://examplebucket.cn-beijing.oss-dls.aliyuncs.com/ f \u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u9ed8\u8ba4\u4f7f\u7528\u5b88\u62a4\u8fdb\u7a0b\u65b9\u5f0f\u540e\u53f0\u542f\u52a8\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -f d \u4f7f\u7528 Debug \u6a21\u5f0f\uff0c\u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -d auto_unmount fuse\u8fdb\u7a0b\u9000\u51fa\u540e\u81ea\u52a8umount\u6302\u8f7d\u8282\u70b9\u3002 -oauto_unmount ro \u53ea\u8bfb\u6302\u8f7d\uff0c\u542f\u7528\u53c2\u6570\u540e\u4e0d\u5141\u8bb8\u5199\u64cd\u4f5c\u3002 -oro direct_io \u5f00\u542f\u540e\uff0c\u8bfb\u5199\u6587\u4ef6\u53ef\u4ee5\u7ed5\u8fc7page cache\u3002 -odirect_io kernel_cache \u5f00\u542f\u540e\uff0c\u5229\u7528\u5185\u6838\u7f13\u5b58\u4f18\u5316\u8bfb\u6027\u80fd\u3002 -okernel_cache auto_cache \u9ed8\u8ba4\u5f00\u542f\uff0c\u4e0ekernel_cache \u4e8c\u9009\u4e00\uff0c\u4e0ekernel_cache\u4e0d\u540c\u7684\u662f\uff0c\u5982\u679c\u6587\u4ef6\u5927\u5c0f\u6216\u4fee\u6539\u65f6\u95f4\u53d1\u751f\u53d8\u5316\uff0c\u7f13\u5b58\u5c31\u4f1a\u5931\u6548\u3002 entry_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u76ee\u5f55\u6761\u76ee\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oentry_timeout=60 attr_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oattr_timeout=60 negative_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u5931\u8d25\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -onegative_timeout=0 jindo_entry_size \u9ed8\u8ba4\u503c\uff0c5000\u3002\u76ee\u5f55\u6761\u76ee\u7f13\u5b58\u6570\u91cf\uff0c\u7528\u4e8e\u4f18\u5316readdir\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -ojindo_entry_size=5000 jindo_attr_size \u9ed8\u8ba4\u503c\uff0c50000\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u6570\u91cf\uff0c\u7528\u4e8e\u4f18\u5316getattr\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -ojindo_attr_sizet=50000 max_idle_threads \u9ed8\u8ba4\u503c\uff0c10\u3002\u6700\u5927\u7a7a\u95f2\u7ebf\u7a0b\u6570\u3002 -omax_idle_threads=10 metrics_port \u9ed8\u8ba4\u503c\uff0c9090\u3002\u5f00\u542fhttp\u7aef\u53e3\uff0c\u8f93\u51fametrics\uff0c\u5982http://localhost:9090/brpc_metrics -ometrics_port=9090 enable_pread \u5f00\u542f\u540e\uff0c\u4f7f\u7528pread\u63a5\u53e3\u8bfb\u53d6\u6587\u4ef6\u3002 -oenable_pread \u914d\u7f6e\u9009\u9879 \u00a7 \u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u914d\u7f6e\u8282 \u8bf4\u660e logger.dir /tmp/fuse-log common \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false common \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false common \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\u3002 \u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002 logger.verbose 0 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false common \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u5730\u5740\uff0c\u5982cn-xxx.oss-dls.aliyuncs.com fs.oss.accessKeyId jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeySecret \u66f4\u591a\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<oss_path>] -ofs.oss.endpoint=[<your_endpoint>] -ofs.oss.accessKeyId=[<your_key_id>] -ofs.oss.accessKeySecret=[<your_key_secret>] \u5e38\u89c1\u95ee\u9898 \u00a7 Input/Output error \u00a7 \u4e0d\u50cf\u4f7f\u7528 JindoSDK \u8c03\u7528 API \u53ef\u4ee5\u83b7\u53d6\u66f4\u4e3a\u5177\u4f53\u7684 ErrorMsg\uff0cJindoFuse \u53ea\u80fd\u663e\u793a\u64cd\u4f5c\u7cfb\u7edf\u9884\u8bbe\u7684\u9519\u8bef\u4fe1\u606f\uff0c\u6bd4\u5982\u4ee5\u4e0b\u9519\u8bef\u5c31\u975e\u5e38\u5e38\u89c1\uff1a $ ls /mnt/oss/ ls: /mnt/oss/: Input/output error \u5982\u679c\u9700\u8981\u5b9a\u4f4d\u5177\u4f53\u7684\u9519\u8bef\u539f\u56e0\uff0c\u53ef\u4ee5\u6839\u636e JindoSDK \u914d\u7f6e\u4e2d\u7684 logger.dir \uff0c\u5728\u6307\u5b9a\u8def\u5f84\u4e0b\u7684 jindosdk.log \u6587\u4ef6\u4e2d\uff0c\u5bfb\u627e\u5177\u4f53\u7684\u9519\u8bef\u3002 \u4e0b\u9762\u5c55\u793a\u7684\u8fd9\u4e2a\u9519\u8bef\u5c31\u6765\u81ea jindosdk.log \uff0c\u6839\u636e\u62a5\u9519\u4fe1\u606f\u53ef\u89c1\uff0c\u8fd9\u4e2a\u9519\u8bef\u662f\u5e38\u89c1\u7684\u9274\u6743\u95ee\u9898\u3002 EMMDD HH:mm:ss jindofs_connectivity.cpp:13] Please check your Endpoint/Bucket/RoleArn. Failed test connectivity, operation: mkdir, errMsg: [RequestId]: 618B8183343EA53531C62B74 [HostId]: oss-cn-shanghai-internal.aliyuncs.com [ErrorMessage]: [E1010]HTTP/1.1 403 Forbidden ...","title":"\u4f7f\u7528 JindoFuse \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#jindofuse-oss-hdfs-jindofs","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u901a\u8fc7 JindoFuse \u63d0\u4f9b POSIX \u652f\u6301\u3002JindoFuse \u53ef\u4ee5\u628a OSS-HDFS \u670d\u52a1\u4e0a\u7684\u6587\u4ef6\u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c\u8be5\u670d\u52a1\u4e0a\u7684\u6587\u4ef6\u3002","title":"\u4f7f\u7528 JindoFuse \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_1","text":"","title":"\u57fa\u672c\u4f7f\u7528"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_2","text":"# CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev","title":"\u73af\u5883\u51c6\u5907"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_3","text":"","title":"\u914d\u7f6e\u5ba2\u6237\u7aef"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_4","text":"\u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf","title":"\u914d\u7f6e\u76ee\u5f55"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_5","text":"\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u5f00\u542fHDFS\u670d\u52a1\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret>","title":"\u914d\u7f6e\u6587\u4ef6"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_6","text":"\u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7\u89d2\u8272\u6388\u6743\u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u5f00\u542fHDFS\u670d\u52a1\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE fs.oss.provider.format = JSON","title":"\u514d\u5bc6\u8bbf\u95ee"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#jindofuse","text":"\u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002","title":"\u6302\u8f7d JindoFuse"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_7","text":"mkdir -p <mount_point>","title":"\u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#fuse","text":"jindo-fuse <mount_point> -ouri=[<oss_path>] -ouri \u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684 oss \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a Bucket \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002","title":"\u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#jindofuse_1","text":"\u5982\u679c\u5c06 OSS-HDFS \u670d\u52a1\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1 ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/","title":"\u8bbf\u95ee JindoFuse"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#jindofuse_2","text":"\u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point>","title":"\u5378\u8f7d JindoFuse"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#jindofuse_3","text":"\u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002","title":"\u81ea\u52a8\u5378\u8f7d JindoFuse"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_8","text":"\u76ee\u524d JindoFuse \u5df2\u7ecf\u652f\u6301\u4ee5\u4e0b POSIX API\uff1a \u7279\u6027 \u8bf4\u660e getattr() \u67e5\u8be2\u6587\u4ef6\u5c5e\u6027\uff0c\u7c7b\u4f3c ls mkdir() \u521b\u5efa\u76ee\u5f55\uff0c\u7c7b\u4f3c mkdir rmdir() \u5220\u9664\u76ee\u5f55\uff0c\u7c7b\u4f3c rm -rf unlink() \u5220\u9664\u6587\u4ef6\uff0c\u7c7b\u4f3c unlink rename() \u91cd\u547d\u540d\uff0c\u7c7b\u4f3c mv read() \u987a\u5e8f\u8bfb\u53d6 pread() \u968f\u673a\u8bfb write() \u987a\u5e8f\u5199 pwrite() \u968f\u673a\u5199 flush() \u5237\u65b0\u5185\u5b58\u5230\u5185\u6838\u7f13\u51b2\u533a fsync() \u5237\u65b0\u5185\u5b58\u5230\u78c1\u76d8 release() \u5173\u95ed\u6587\u4ef6 readdir() \u8bfb\u53d6\u76ee\u5f55 create() \u521b\u5efa\u6587\u4ef6 open() O_APPEND \u901a\u8fc7\u8ffd\u52a0\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 open() O_TRUNC \u901a\u8fc7\u8986\u76d6\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 ftruncate() \u5bf9\u6253\u5f00\u7684\u6587\u4ef6\u8fdb\u884c\u622a\u65ad truncate() \u5bf9\u672a\u6253\u5f00\u7684\u6587\u4ef6\u8fdb\u884c\u622a\u65ad\uff0c\u7c7b\u4f3c truncate -s lseek() \u6307\u5b9a\u6253\u5f00\u6587\u4ef6\u4e2d\u7684\u8bfb\u5199\u4f4d\u7f6e\u3002 chmod() \u4fee\u6539\u6587\u4ef6\u6743\u9650\uff0c\u7c7b\u4f3c chmod access() \u67e5\u8be2\u6587\u4ef6\u6743\u9650 utimes() \u4fee\u6539\u6587\u4ef6\u7684\u5b58\u53d6\u65f6\u95f4\u548c\u66f4\u6539\u65f6\u95f4(\u4ec5\u652f\u6301\u4fee\u6539\u4e3a\u66f4\u65b0\u7684\u65f6\u95f4) setxattr() \u4fee\u6539\u6587\u4ef6 xattr \u5c5e\u6027 getxattr() \u83b7\u53d6\u6587\u4ef6 xattr \u5c5e\u6027 listxattr() \u5217\u4e3e\u6587\u4ef6 xattr \u5c5e\u6027 removexattr() \u5220\u9664\u6587\u4ef6 xattr \u5c5e\u6027 lock() \u652f\u6301 posix \u9501\uff0c\u7c7b\u4f3c fcntl fallocate() \u4e3a\u6587\u4ef6\u9884\u5206\u914d\u7269\u7406\u7a7a\u95f4 symlink() \u521b\u5efa\u8f6f\u8fde\u63a5\uff0c\u76ee\u524d\u4ec5\u652f\u6301 OSS-HDFS \u5185\u90e8\u4f7f\u7528\uff0c\u4e14\u4e0d\u652f\u6301\u7f13\u5b58\u52a0\u901f readlink() \u8bfb\u53d6\u8f6f\u8fde\u63a5","title":"\u7279\u6027\u652f\u6301"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_9","text":"","title":"\u9ad8\u9636\u4f7f\u7528"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_10","text":"\u53c2\u6570\u540d\u79f0 \u5fc5\u9009 \u53c2\u6570\u8bf4\u660e \u4f7f\u7528\u8303\u4f8b uri \u2713 \u914d\u7f6e\u9700\u8981\u6620\u5c04\u7684 oss \u8def\u5f84\u3002\u8def\u5f84\u53ef\u4ee5\u662f\u6839\u76ee\u5f55\uff0c\u4e5f\u53ef\u4ee5\u662f\u5b50\u76ee\u5f55\u3002\u4f8b\u5982\uff1aoss://bucket.endpoint/ \u6216 oss://bucket.endpoint/subdir\u3002 -ouri=oss://examplebucket.cn-beijing.oss-dls.aliyuncs.com/ f \u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u9ed8\u8ba4\u4f7f\u7528\u5b88\u62a4\u8fdb\u7a0b\u65b9\u5f0f\u540e\u53f0\u542f\u52a8\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -f d \u4f7f\u7528 Debug \u6a21\u5f0f\uff0c\u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -d auto_unmount fuse\u8fdb\u7a0b\u9000\u51fa\u540e\u81ea\u52a8umount\u6302\u8f7d\u8282\u70b9\u3002 -oauto_unmount ro \u53ea\u8bfb\u6302\u8f7d\uff0c\u542f\u7528\u53c2\u6570\u540e\u4e0d\u5141\u8bb8\u5199\u64cd\u4f5c\u3002 -oro direct_io \u5f00\u542f\u540e\uff0c\u8bfb\u5199\u6587\u4ef6\u53ef\u4ee5\u7ed5\u8fc7page cache\u3002 -odirect_io kernel_cache \u5f00\u542f\u540e\uff0c\u5229\u7528\u5185\u6838\u7f13\u5b58\u4f18\u5316\u8bfb\u6027\u80fd\u3002 -okernel_cache auto_cache \u9ed8\u8ba4\u5f00\u542f\uff0c\u4e0ekernel_cache \u4e8c\u9009\u4e00\uff0c\u4e0ekernel_cache\u4e0d\u540c\u7684\u662f\uff0c\u5982\u679c\u6587\u4ef6\u5927\u5c0f\u6216\u4fee\u6539\u65f6\u95f4\u53d1\u751f\u53d8\u5316\uff0c\u7f13\u5b58\u5c31\u4f1a\u5931\u6548\u3002 entry_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u76ee\u5f55\u6761\u76ee\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oentry_timeout=60 attr_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oattr_timeout=60 negative_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u5931\u8d25\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -onegative_timeout=0 jindo_entry_size \u9ed8\u8ba4\u503c\uff0c5000\u3002\u76ee\u5f55\u6761\u76ee\u7f13\u5b58\u6570\u91cf\uff0c\u7528\u4e8e\u4f18\u5316readdir\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -ojindo_entry_size=5000 jindo_attr_size \u9ed8\u8ba4\u503c\uff0c50000\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u6570\u91cf\uff0c\u7528\u4e8e\u4f18\u5316getattr\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -ojindo_attr_sizet=50000 max_idle_threads \u9ed8\u8ba4\u503c\uff0c10\u3002\u6700\u5927\u7a7a\u95f2\u7ebf\u7a0b\u6570\u3002 -omax_idle_threads=10 metrics_port \u9ed8\u8ba4\u503c\uff0c9090\u3002\u5f00\u542fhttp\u7aef\u53e3\uff0c\u8f93\u51fametrics\uff0c\u5982http://localhost:9090/brpc_metrics -ometrics_port=9090 enable_pread \u5f00\u542f\u540e\uff0c\u4f7f\u7528pread\u63a5\u53e3\u8bfb\u53d6\u6587\u4ef6\u3002 -oenable_pread","title":"\u6302\u8f7d\u9009\u9879"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_11","text":"\u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u914d\u7f6e\u8282 \u8bf4\u660e logger.dir /tmp/fuse-log common \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false common \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false common \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\u3002 \u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002 logger.verbose 0 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false common \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u5730\u5740\uff0c\u5982cn-xxx.oss-dls.aliyuncs.com fs.oss.accessKeyId jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeySecret \u66f4\u591a\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<oss_path>] -ofs.oss.endpoint=[<your_endpoint>] -ofs.oss.accessKeyId=[<your_key_id>] -ofs.oss.accessKeySecret=[<your_key_secret>]","title":"\u914d\u7f6e\u9009\u9879"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_12","text":"","title":"\u5e38\u89c1\u95ee\u9898"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#inputoutput-error","text":"\u4e0d\u50cf\u4f7f\u7528 JindoSDK \u8c03\u7528 API \u53ef\u4ee5\u83b7\u53d6\u66f4\u4e3a\u5177\u4f53\u7684 ErrorMsg\uff0cJindoFuse \u53ea\u80fd\u663e\u793a\u64cd\u4f5c\u7cfb\u7edf\u9884\u8bbe\u7684\u9519\u8bef\u4fe1\u606f\uff0c\u6bd4\u5982\u4ee5\u4e0b\u9519\u8bef\u5c31\u975e\u5e38\u5e38\u89c1\uff1a $ ls /mnt/oss/ ls: /mnt/oss/: Input/output error \u5982\u679c\u9700\u8981\u5b9a\u4f4d\u5177\u4f53\u7684\u9519\u8bef\u539f\u56e0\uff0c\u53ef\u4ee5\u6839\u636e JindoSDK \u914d\u7f6e\u4e2d\u7684 logger.dir \uff0c\u5728\u6307\u5b9a\u8def\u5f84\u4e0b\u7684 jindosdk.log \u6587\u4ef6\u4e2d\uff0c\u5bfb\u627e\u5177\u4f53\u7684\u9519\u8bef\u3002 \u4e0b\u9762\u5c55\u793a\u7684\u8fd9\u4e2a\u9519\u8bef\u5c31\u6765\u81ea jindosdk.log \uff0c\u6839\u636e\u62a5\u9519\u4fe1\u606f\u53ef\u89c1\uff0c\u8fd9\u4e2a\u9519\u8bef\u662f\u5e38\u89c1\u7684\u9274\u6743\u95ee\u9898\u3002 EMMDD HH:mm:ss jindofs_connectivity.cpp:13] Please check your Endpoint/Bucket/RoleArn. Failed test connectivity, operation: mkdir, errMsg: [RequestId]: 618B8183343EA53531C62B74 [HostId]: oss-cn-shanghai-internal.aliyuncs.com [ErrorMessage]: [E1010]HTTP/1.1 403 Forbidden ...","title":"Input/Output error"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/","text":"\u4f7f\u7528 Jindo Tensorflow Collector \u8bbf\u95ee OSS-HDFS \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u901a\u8fc7 Jindo Tensorflow Collector \u63d0\u4f9b\u4f7f\u7528 tensorflow gfile \u8bbf\u95ee OSS-HDFS \u7684\u652f\u6301\u3002 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u4fee\u6539\u914d\u7f6e \u00a7 2.1 \u5b89\u88c5 tensorflow \u00a7 \u76ee\u524d\u4ec5\u652f\u6301 tensorflow 1.15 \u6216 tensorflow 2.8 \u7248\u672c\uff0c\u4ee5\u4e0b\u4ee5 tensorflow 2.8 \u4e3a\u4f8b\u3002 pip3 install tensorflow==2.8 2.2 \u6dfb\u52a0\u73af\u5883\u53d8\u91cf \u00a7 2.2.1 JindoSDK \u73af\u5883\u53d8\u91cf \u00a7 export LD_LIBRARY_PATH=\"/path/to/jindosdk-x.x.x/lib/native\":$LD_LIBRARY_PATH export JINDOSDK_CONF_DIR=/path/to/jindosdk-x.x.x/conf 2.2.2 \u4f9d\u8d56\u7684 Tensorflow \u73af\u5883\u53d8\u91cf \u00a7 \u540c\u65f6\uff0cLD_LIBRARY_PATH \u9700\u8981\u6dfb\u52a0 tensorflow \u6240\u5728\u7684\u4f9d\u8d56\u8def\u5f84\uff0c\u4e0d\u540c\u7684 python \u73af\u5883\u5b89\u88c5\u8def\u5f84\u53ef\u80fd\u4e0d\u540c\uff0c\u5982\uff1a # tensorflow1.15 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow_core/\":$LD_LIBRARY_PATH # tensorflow2.8 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow/\":$LD_LIBRARY_PATH 2.3 \u914d\u7f6e\u6587\u4ef6 \u00a7 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \u3002\u53ef\u4ee5\u9009\u62e9\u56fa\u5b9a\u6216\u8005\u514d\u5bc6\u4e24\u79cd\u914d\u7f6e\u65b9\u5f0f\u3002 2.3.1 \u56fa\u5b9a AK \u8bbf\u95ee \u00a7 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/tfio-log [jindosdk] # \u5df2\u5f00\u542fHDFS\u670d\u52a1\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> \u914d\u7f6e\u9009\u9879 \u00a7 \u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u914d\u7f6e\u8282 \u8bf4\u660e logger.dir /tmp/tfio-log common \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false common \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false common \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\u3002 \u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002 logger.verbose 0 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false common \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u5730\u5740\uff0c\u5982cn-xxx.oss-dls.aliyuncs.com fs.oss.accessKeyId jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeySecret \u66f4\u591a\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 2.3.2 \u514d\u5bc6\u8bbf\u95ee \u00a7 \u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7\u89d2\u8272\u6388\u6743\u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u5f00\u542fHDFS\u670d\u52a1\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE fs.oss.provider.format = JSON 3. \u4f7f\u7528 Jindo Tensorflow \u00a7 \u4ee5\u52a0\u8f7d jindo-tensorflow2.8 \u4e3a\u4f8b 3.1 \u901a\u8fc7 load_library \u6dfb\u52a0 collector \u00a7 import tensorflow as tf tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\") 3.2 \u901a\u8fc7 gfile \u8bbf\u95ee oss-hdfs \u00a7 \u4e3a\u907f\u514d\u4e0e tfio \u4e2d\u7684 oss \u524d\u7f00\u51b2\u7a81\uff0cjindo-tensorflow \u4f7f\u7528 joss \u524d\u7f00\u3002 # \u76ee\u5f55\u64cd\u4f5c tf.io.gfile.mkdir(\"joss://<bucket>.<endpoint>/test_mkdir\") content = tf.io.gfile.listdir(\"joss://<bucket>.<endpoint>/test_mkdir\") # \u8bfb gmnist = tf.io.gfile.GFile(\"joss://<bucket>.<endpoint>/mnist.npz\", \"rb\") gmnist_file = gmnist.read() # \u5199 fh = gfile.Open(f, mode=\"w\") content = \"file content\" fh.write(content) fh.close() 3.3 \u8bfb\u53d6 TFRecordDataset \u00a7 tfRecordJfsPath = \"joss://<bucket>.<endpoint>/test.tfrecord\" filenames = [tfRecordJfsPath] raw_dataset = tf.data.TFRecordDataset(filenames) 3.4 \u5b8c\u6574\u793a\u4f8b \u00a7 import tensorflow as tf import numpy as np tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\") gmnist = tf.io.gfile.GFile(\"joss://<bucket>.<endpoint>/mnist.npz\", 'rb') print(gmnist.size()) with np.load(gmnist) as f: x_train, y_train = f['x_train'], f['y_train'] x_test, y_test = f['x_test'], f['y_test'] model = tf.keras.models.Sequential([ tf.keras.layers.Flatten(input_shape=(28, 28)), tf.keras.layers.Dense(128, activation='relu'), tf.keras.layers.Dropout(0.2), tf.keras.layers.Dense(10, activation='softmax') ]) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) model.fit(x_train, y_train, epochs=5)","title":"\u4f7f\u7528 Jindo Tensorflow Collector \u8bbf\u95ee OSS-HDFS"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#jindo-tensorflow-collector-oss-hdfs","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u901a\u8fc7 Jindo Tensorflow Collector \u63d0\u4f9b\u4f7f\u7528 tensorflow gfile \u8bbf\u95ee OSS-HDFS \u7684\u652f\u6301\u3002","title":"\u4f7f\u7528 Jindo Tensorflow Collector \u8bbf\u95ee OSS-HDFS"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#2","text":"","title":"2. \u4fee\u6539\u914d\u7f6e"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#21-tensorflow","text":"\u76ee\u524d\u4ec5\u652f\u6301 tensorflow 1.15 \u6216 tensorflow 2.8 \u7248\u672c\uff0c\u4ee5\u4e0b\u4ee5 tensorflow 2.8 \u4e3a\u4f8b\u3002 pip3 install tensorflow==2.8","title":"2.1 \u5b89\u88c5 tensorflow"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#22","text":"","title":"2.2 \u6dfb\u52a0\u73af\u5883\u53d8\u91cf"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#221-jindosdk","text":"export LD_LIBRARY_PATH=\"/path/to/jindosdk-x.x.x/lib/native\":$LD_LIBRARY_PATH export JINDOSDK_CONF_DIR=/path/to/jindosdk-x.x.x/conf","title":"2.2.1 JindoSDK \u73af\u5883\u53d8\u91cf"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#222-tensorflow","text":"\u540c\u65f6\uff0cLD_LIBRARY_PATH \u9700\u8981\u6dfb\u52a0 tensorflow \u6240\u5728\u7684\u4f9d\u8d56\u8def\u5f84\uff0c\u4e0d\u540c\u7684 python \u73af\u5883\u5b89\u88c5\u8def\u5f84\u53ef\u80fd\u4e0d\u540c\uff0c\u5982\uff1a # tensorflow1.15 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow_core/\":$LD_LIBRARY_PATH # tensorflow2.8 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow/\":$LD_LIBRARY_PATH","title":"2.2.2 \u4f9d\u8d56\u7684 Tensorflow \u73af\u5883\u53d8\u91cf"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#23","text":"\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \u3002\u53ef\u4ee5\u9009\u62e9\u56fa\u5b9a\u6216\u8005\u514d\u5bc6\u4e24\u79cd\u914d\u7f6e\u65b9\u5f0f\u3002","title":"2.3 \u914d\u7f6e\u6587\u4ef6"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#231-ak","text":"\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/tfio-log [jindosdk] # \u5df2\u5f00\u542fHDFS\u670d\u52a1\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret>","title":"2.3.1 \u56fa\u5b9a AK \u8bbf\u95ee"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#_1","text":"\u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u914d\u7f6e\u8282 \u8bf4\u660e logger.dir /tmp/tfio-log common \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false common \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false common \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\u3002 \u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002 logger.verbose 0 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false common \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u5730\u5740\uff0c\u5982cn-xxx.oss-dls.aliyuncs.com fs.oss.accessKeyId jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeySecret \u66f4\u591a\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002","title":"\u914d\u7f6e\u9009\u9879"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#232","text":"\u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7\u89d2\u8272\u6388\u6743\u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u5f00\u542fHDFS\u670d\u52a1\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE fs.oss.provider.format = JSON","title":"2.3.2 \u514d\u5bc6\u8bbf\u95ee"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#3-jindo-tensorflow","text":"\u4ee5\u52a0\u8f7d jindo-tensorflow2.8 \u4e3a\u4f8b","title":"3. \u4f7f\u7528 Jindo Tensorflow"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#31-load_library-collector","text":"import tensorflow as tf tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\")","title":"3.1 \u901a\u8fc7 load_library \u6dfb\u52a0 collector"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#32-gfile-oss-hdfs","text":"\u4e3a\u907f\u514d\u4e0e tfio \u4e2d\u7684 oss \u524d\u7f00\u51b2\u7a81\uff0cjindo-tensorflow \u4f7f\u7528 joss \u524d\u7f00\u3002 # \u76ee\u5f55\u64cd\u4f5c tf.io.gfile.mkdir(\"joss://<bucket>.<endpoint>/test_mkdir\") content = tf.io.gfile.listdir(\"joss://<bucket>.<endpoint>/test_mkdir\") # \u8bfb gmnist = tf.io.gfile.GFile(\"joss://<bucket>.<endpoint>/mnist.npz\", \"rb\") gmnist_file = gmnist.read() # \u5199 fh = gfile.Open(f, mode=\"w\") content = \"file content\" fh.write(content) fh.close()","title":"3.2 \u901a\u8fc7 gfile \u8bbf\u95ee oss-hdfs"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#33-tfrecorddataset","text":"tfRecordJfsPath = \"joss://<bucket>.<endpoint>/test.tfrecord\" filenames = [tfRecordJfsPath] raw_dataset = tf.data.TFRecordDataset(filenames)","title":"3.3 \u8bfb\u53d6 TFRecordDataset"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#34","text":"import tensorflow as tf import numpy as np tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\") gmnist = tf.io.gfile.GFile(\"joss://<bucket>.<endpoint>/mnist.npz\", 'rb') print(gmnist.size()) with np.load(gmnist) as f: x_train, y_train = f['x_train'], f['y_train'] x_test, y_test = f['x_test'], f['y_test'] model = tf.keras.models.Sequential([ tf.keras.layers.Flatten(input_shape=(28, 28)), tf.keras.layers.Dense(128, activation='relu'), tf.keras.layers.Dropout(0.2), tf.keras.layers.Dense(10, activation='softmax') ]) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) model.fit(x_train, y_train, epochs=5)","title":"3.4 \u5b8c\u6574\u793a\u4f8b"},{"location":"backup/jindofs/jindotable/table_moveto/","text":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 HDFS \u53d7\u9650\u4e8e\u96c6\u7fa4\u89c4\u6a21\u548c\u6210\u672c\u56e0\u7d20\uff0c\u65e0\u6cd5\u65e0\u9650\u6269\u5c55\uff0c\u5bb9\u91cf\u5b58\u5728\u74f6\u9888\u3002\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\u53ef\u4ee5\u4f5c\u4e3a HDFS \u7684\u66ff\u4ee3\u6216\u8865\u5145\uff0c\u6269\u5c55\u4e91\u4e0a Hadoop \u5e73\u53f0\u7684\u5b58\u50a8\u80fd\u529b\u3002 \u672c\u5de5\u5177\u7528\u4e8e\u628a Hive \u6570\u636e\u6839\u636e\u5206\u533a\u952e\u89c4\u5219\u7b5b\u9009\uff0c\u5728 HDFS \u548c \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\u4e4b\u95f4\u8f6c\u79fb\u5206\u533a\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u90e8\u7f72 JindoSDK \u00a7 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8 \u3002 \u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883 \u00a7 \u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e \u914d\u7f6e MoveTo \u5de5\u5177\u5728 HDFS \u4e0b\u7684\u9501\u76ee\u5f55\uff1a \u00a7 \u5728 Hadoop \u914d\u7f6e\u6587\u4ef6 core-site.xml \u6216 hdfs-site.xml \uff08\u4efb\u4e00\u5373\u53ef\uff0c\u5728 $HADOOP_CONF_DIR \u76ee\u5f55\u4e0b\uff09\u65b0\u589e\u914d\u7f6e\u9879\uff1a jindotable.moveto.tablelock.base.dir \u8be5\u914d\u7f6e\u7684\u503c\u5e94\u6307\u5411\u4e00\u4e2a HDFS \u76ee\u5f55\uff0c\u76ee\u7684\u662f\u5b58\u653e MoveTo \u5de5\u5177\u5728\u8fd0\u884c\u65f6\u81ea\u52a8\u521b\u5efa\u7684\u9501\u6587\u4ef6\u3002\u9700\u786e\u4fdd\u8be5\u76ee\u5f55\u53ea\u4f1a\u88ab MoveTo \u5de5\u5177\u8bbf\u95ee\uff0c\u5e76\u4e14\u6709\u8bbf\u95ee\u6743\u9650\u3002\u5982\u679c\u4e0d\u914d\u7f6e\uff0c\u5219\u4f7f\u7528\u7f3a\u7701\u503c hdfs:///tmp/jindotable-lock/ \uff0c\u65e0\u6743\u9650\u5219\u62a5\u9519\u3002 \u4f7f\u7528\u8bf4\u660e \u00a7 \u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f \u00a7 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help moveTo \u53c2\u6570\u8bf4\u660e \u00a7 jindotable -moveTo -t <dbName.tableName> -d <destination path> [-c \"<condition>\" | -fullTable] [-b/-before <before days>] [-p/-parallel <parallelism>] [-s/-storagePolicy <OSS storage policy>] [-o/-overWrite] [-r/-removeSource] [-skipTrash] [-e/-explain] [-q/-queue <yarn queue>] [-w/-workingDir <working directory>][-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u8981\u79fb\u52a8\u7684\u8868\u3002 \u662f -d <destination path> \u76ee\u6807\u8def\u5f84\uff0c\u4e3a\u8868\u7ea7\u522b\u7684\u8def\u5f84\uff0c\u5206\u533a\u8def\u5f84\u4f1a\u5728\u8fd9\u4e2a\u8def\u5f84\u4e0b\u81ea\u52a8\u521b\u5efa\u3002 \u662f -c \"<condition>\" / -fullTable \u5206\u533a\u8fc7\u6ee4\u6761\u4ef6\u8868\u8fbe\u5f0f\uff0c\u652f\u6301\u57fa\u672c\u8fd0\u7b97\u7b26\uff0c\u4e0d\u652f\u6301udf\u3002 \u5426 -b/before <before days> \u6839\u636e\u5206\u533a\u521b\u5efa\u65f6\u95f4\uff0c\u521b\u5efa\u65f6\u95f4\u8d85\u8fc7\u7ed9\u5b9a\u5929\u6570\u7684\u5206\u533a\u624d\u8fdb\u884c\u79fb\u52a8\u3002 \u5426 -p/-parallel <parallelism> \u6574\u4e2amoveTo\u4efb\u52a1\u7684\u6700\u5927task\u5e76\u53d1\u5ea6\uff0c\u9ed8\u8ba4\u4e3a1\u3002 \u5426 -s/-storagePolicy <OSS storage policy> OSS-HDFS \u670d\u52a1\u4e0d\u652f\u6301\u8be5\u53c2\u6570 \u5426 -o/-overWrite \u662f\u5426\u8986\u76d6\u6700\u7ec8\u76ee\u5f55\u3002\u5206\u533a\u7ea7\u522b\u8986\u76d6\uff0c\u4e0d\u4f1a\u8986\u76d6\u672c\u6b21\u79fb\u52a8\u4e0d\u6d89\u53ca\u7684\u5206\u533a\u3002 \u5426 -r/-removeSource \u662f\u5426\u5728\u79fb\u52a8\u5b8c\u6210\u540e\u5220\u9664\u6e90\u8def\u5f84\u3002 \u5426 -skipTrash \u5982\u679c\u5220\u9664\u6e90\u8def\u5f84\uff0c\u662f\u5426\u8df3\u8fc7Trash\u3002 \u5426 -e/-explain \u5982\u679c\u6307\u5b9aexplain\u6a21\u5f0f\uff0c\u4e0d\u4f1a\u89e6\u53d1\u5b9e\u9645\u64cd\u4f5c\uff0c\u4ec5\u6253\u5370\u4f1a\u540c\u6b65\u7684\u5206\u533a\u3002 \u5426 -q/-queue <yarn queue \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684yarn\u961f\u5217\u3002 \u5426 -w/-workingDir \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684\u5de5\u4f5c\u4e34\u65f6\u76ee\u5f55\u3002 \u5426 -l/-logDir <log directory> \u672c\u5730\u65e5\u5fd7\u76ee\u5f55\uff0c\u9ed8\u8ba4\u4e3a /tmp/<current user>/ \u5426 \u4f7f\u7528\u793a\u4f8b \u00a7 \u6709\u4e00\u4e2a HDFS \u4e0a\u7684 Hive \u5206\u533a\u8868\uff0c\u5982\u4e0b\u56fe\u6240\u793a\uff1a \u60f3\u628a bbb \u548c ccc \u5206\u533a\u79fb\u52a8\u5230 OSS-HDFS \u670d\u52a1\uff0c\u5148\u7528 explain \u6a21\u5f0f\u770b\u770b\u4f1a\u79fb\u52a8\u7684\u5206\u533a\u662f\u5426\u7b26\u5408\u9884\u671f\uff0c\u53c2\u6570\u4e3a -e \u6216 -explain\uff1a \u53bb\u6389\u53c2\u6570 -e\uff0c\u771f\u6b63\u79fb\u52a8\u5206\u533a\uff1a \u6267\u884c\u5b8c\u6210\u540e\uff0c\u68c0\u67e5\u6570\u636e\uff0c\u6570\u636e\u5df2\u7ecf\u5728 OSS-HDFS \u670d\u52a1\uff1a \u518d\u79fb\u56de HDFS\uff0c\u7ed3\u679c\u5931\u8d25\u4e86\uff0c\u4f1a\u663e\u793a\u5931\u8d25\u539f\u56e0\uff0c\u539f\u6765\u662f\u76ee\u6807\u76ee\u5f55\u975e\u7a7a\uff0c\u63d0\u793a\u53ef\u4ee5\u4f7f\u7528 -overWrite \u6e05\u7a7a\u76ee\u6807\u76ee\u5f55\uff1a \u4f7f\u7528 -overWrite \u5f3a\u5236\u8986\u76d6\uff1a \u5f02\u5e38\u72b6\u6001\u5904\u7406 \u00a7 \u4e3a\u4e86\u5c3d\u53ef\u80fd\u4fdd\u8bc1\u6570\u636e\u5b89\u5168\uff0c\u9632\u6b62\u6c61\u67d3\uff0c\u8be5\u547d\u4ee4\u4f1a\u81ea\u52a8\u68c0\u67e5\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u786e\u4fdd\u4e0d\u5b58\u5728\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u76ee\u5f55\u62f7\u8d1d\u3002 \u5982\u679c\u65e0\u6cd5\u6392\u9664\uff0c\u5219\u8be5\u8868\u6216\u5206\u533a\u7684\u62f7\u8d1d\u547d\u4ee4\u5c06\u6267\u884c\u5931\u8d25\u3002 \u6b64\u65f6\uff0c\u9700\u8981\u4e3b\u52a8\u5e72\u9884\uff0c\u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u6b63\u5728\u6267\u884c\u62f7\u8d1d\u4e4b\u540e\uff0c\u6e05\u7406\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u5219\u53ef\u4ee5\u91cd\u65b0\u6267\u884c\u8be5\u547d\u4ee4\u3002 \u5bf9\u4e8e\u975e\u5206\u533a\u8868\u7684\u62f7\u8d1d\u6216\u8fc1\u79fb\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u5373\u4e3a\u8868\u7ea7\u522b\u76ee\u5f55\uff1b\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u4e3a\u5f85\u62f7\u8d1d\u6216\u79fb\u52a8\u5206\u533a\u7684\u5206\u533a\u7ea7\u76ee\u5f55\uff0c \u53ea\u9700\u5bf9\u5f85\u62f7\u8d1d\u7684\u5206\u533a\u8fdb\u884c\u6e05\u7406\u3002 \u51fa\u73b0\u539f\u56e0 \u00a7 \u5982\u679c\u8be5\u547d\u4ee4\u88ab\u5f02\u5e38\u4e2d\u6b62\uff0c\u5219\u53ef\u80fd\u51fa\u73b0\u9700\u8981\u4e3b\u52a8\u5e72\u9884\u7684\u60c5\u51b5\u3002\u6b64\u65f6\u62f7\u8d1d\u8fd8\u672a\u5b8c\u6210\uff0c\u6e90\u6570\u636e\u4e0e\u8868\u7684\u5143\u4fe1\u606f\u5747\u672a\u6539\u53d8\uff0c \u6570\u636e\u4ecd\u5904\u4e8e\u5b89\u5168\u72b6\u6001\u3002\u5e38\u89c1\u7684\u5f02\u5e38\u4e2d\u6b62\u6709\u4e24\u79cd\u60c5\u51b5\uff1a * \u7528\u6237\u5728\u547d\u4ee4\u5c1a\u672a\u7ed3\u675f\u65f6\uff0c\u4e3b\u52a8\u6740\u6389\u547d\u4ee4\u8fdb\u7a0b * \u7531\u4e8e\u5185\u5b58\u6ea2\u51fa\u7b49\u5f02\u5e38\uff0c\u8fdb\u7a0b\u5f02\u5e38\u4e2d\u6b62 \u5982\u679c\u62f7\u8d1d\u8868\u6216\u5206\u533a\u65f6\u62f7\u8d1d\u5931\u8d25\u5e76\u663e\u793a Conflicts found \uff0c\u5219\u5c5e\u4e8e\u5f02\u5e38\u72b6\u6001\uff0c\u9700\u4e3b\u52a8\u5e72\u9884\u3002 \u5904\u7406\u65b9\u6cd5 \u00a7 \u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u7684\u76ee\u6807\u8def\u5f84\u62f7\u8d1d\u6570\u636e\u3002\u7279\u522b\u662f\uff0c\u786e\u4fdd\u6ca1\u6709 DistCp/JindoDistCp \u7b49\u5206\u5e03\u5f0f\u62f7\u8d1d\u547d\u4ee4\u3002 \u5220\u9664\u76ee\u7684\u5730\u76ee\u5f55\u3002\u5bf9\u4e8e\u975e\u5206\u533a\u8868\uff0c\u5220\u9664\u8868\u4e00\u7ea7\u76ee\u5f55\u3002\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u5220\u9664\u5b58\u5728\u51b2\u7a81\u7684\u5206\u533a\u7ea7\u76ee\u5f55\u3002\u5207\u52ff\u5220\u9664\u6e90\u76ee\u5f55\u3002","title":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/jindotable/table_moveto/#jindotable-hive-oss-hdfs-jindofs","text":"HDFS \u53d7\u9650\u4e8e\u96c6\u7fa4\u89c4\u6a21\u548c\u6210\u672c\u56e0\u7d20\uff0c\u65e0\u6cd5\u65e0\u9650\u6269\u5c55\uff0c\u5bb9\u91cf\u5b58\u5728\u74f6\u9888\u3002\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\u53ef\u4ee5\u4f5c\u4e3a HDFS \u7684\u66ff\u4ee3\u6216\u8865\u5145\uff0c\u6269\u5c55\u4e91\u4e0a Hadoop \u5e73\u53f0\u7684\u5b58\u50a8\u80fd\u529b\u3002 \u672c\u5de5\u5177\u7528\u4e8e\u628a Hive \u6570\u636e\u6839\u636e\u5206\u533a\u952e\u89c4\u5219\u7b5b\u9009\uff0c\u5728 HDFS \u548c \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\u4e4b\u95f4\u8f6c\u79fb\u5206\u533a\u3002","title":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/jindotable/table_moveto/#_1","text":"","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/jindofs/jindotable/table_moveto/#jindosdk","text":"\u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8 \u3002","title":"\u5df2\u90e8\u7f72 JindoSDK"},{"location":"backup/jindofs/jindotable/table_moveto/#hadoop-hive","text":"\u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e","title":"\u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883"},{"location":"backup/jindofs/jindotable/table_moveto/#moveto-hdfs","text":"\u5728 Hadoop \u914d\u7f6e\u6587\u4ef6 core-site.xml \u6216 hdfs-site.xml \uff08\u4efb\u4e00\u5373\u53ef\uff0c\u5728 $HADOOP_CONF_DIR \u76ee\u5f55\u4e0b\uff09\u65b0\u589e\u914d\u7f6e\u9879\uff1a jindotable.moveto.tablelock.base.dir \u8be5\u914d\u7f6e\u7684\u503c\u5e94\u6307\u5411\u4e00\u4e2a HDFS \u76ee\u5f55\uff0c\u76ee\u7684\u662f\u5b58\u653e MoveTo \u5de5\u5177\u5728\u8fd0\u884c\u65f6\u81ea\u52a8\u521b\u5efa\u7684\u9501\u6587\u4ef6\u3002\u9700\u786e\u4fdd\u8be5\u76ee\u5f55\u53ea\u4f1a\u88ab MoveTo \u5de5\u5177\u8bbf\u95ee\uff0c\u5e76\u4e14\u6709\u8bbf\u95ee\u6743\u9650\u3002\u5982\u679c\u4e0d\u914d\u7f6e\uff0c\u5219\u4f7f\u7528\u7f3a\u7701\u503c hdfs:///tmp/jindotable-lock/ \uff0c\u65e0\u6743\u9650\u5219\u62a5\u9519\u3002","title":"\u914d\u7f6e MoveTo \u5de5\u5177\u5728 HDFS \u4e0b\u7684\u9501\u76ee\u5f55\uff1a"},{"location":"backup/jindofs/jindotable/table_moveto/#_2","text":"","title":"\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindotable/table_moveto/#_3","text":"\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help moveTo","title":"\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f"},{"location":"backup/jindofs/jindotable/table_moveto/#_4","text":"jindotable -moveTo -t <dbName.tableName> -d <destination path> [-c \"<condition>\" | -fullTable] [-b/-before <before days>] [-p/-parallel <parallelism>] [-s/-storagePolicy <OSS storage policy>] [-o/-overWrite] [-r/-removeSource] [-skipTrash] [-e/-explain] [-q/-queue <yarn queue>] [-w/-workingDir <working directory>][-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u8981\u79fb\u52a8\u7684\u8868\u3002 \u662f -d <destination path> \u76ee\u6807\u8def\u5f84\uff0c\u4e3a\u8868\u7ea7\u522b\u7684\u8def\u5f84\uff0c\u5206\u533a\u8def\u5f84\u4f1a\u5728\u8fd9\u4e2a\u8def\u5f84\u4e0b\u81ea\u52a8\u521b\u5efa\u3002 \u662f -c \"<condition>\" / -fullTable \u5206\u533a\u8fc7\u6ee4\u6761\u4ef6\u8868\u8fbe\u5f0f\uff0c\u652f\u6301\u57fa\u672c\u8fd0\u7b97\u7b26\uff0c\u4e0d\u652f\u6301udf\u3002 \u5426 -b/before <before days> \u6839\u636e\u5206\u533a\u521b\u5efa\u65f6\u95f4\uff0c\u521b\u5efa\u65f6\u95f4\u8d85\u8fc7\u7ed9\u5b9a\u5929\u6570\u7684\u5206\u533a\u624d\u8fdb\u884c\u79fb\u52a8\u3002 \u5426 -p/-parallel <parallelism> \u6574\u4e2amoveTo\u4efb\u52a1\u7684\u6700\u5927task\u5e76\u53d1\u5ea6\uff0c\u9ed8\u8ba4\u4e3a1\u3002 \u5426 -s/-storagePolicy <OSS storage policy> OSS-HDFS \u670d\u52a1\u4e0d\u652f\u6301\u8be5\u53c2\u6570 \u5426 -o/-overWrite \u662f\u5426\u8986\u76d6\u6700\u7ec8\u76ee\u5f55\u3002\u5206\u533a\u7ea7\u522b\u8986\u76d6\uff0c\u4e0d\u4f1a\u8986\u76d6\u672c\u6b21\u79fb\u52a8\u4e0d\u6d89\u53ca\u7684\u5206\u533a\u3002 \u5426 -r/-removeSource \u662f\u5426\u5728\u79fb\u52a8\u5b8c\u6210\u540e\u5220\u9664\u6e90\u8def\u5f84\u3002 \u5426 -skipTrash \u5982\u679c\u5220\u9664\u6e90\u8def\u5f84\uff0c\u662f\u5426\u8df3\u8fc7Trash\u3002 \u5426 -e/-explain \u5982\u679c\u6307\u5b9aexplain\u6a21\u5f0f\uff0c\u4e0d\u4f1a\u89e6\u53d1\u5b9e\u9645\u64cd\u4f5c\uff0c\u4ec5\u6253\u5370\u4f1a\u540c\u6b65\u7684\u5206\u533a\u3002 \u5426 -q/-queue <yarn queue \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684yarn\u961f\u5217\u3002 \u5426 -w/-workingDir \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684\u5de5\u4f5c\u4e34\u65f6\u76ee\u5f55\u3002 \u5426 -l/-logDir <log directory> \u672c\u5730\u65e5\u5fd7\u76ee\u5f55\uff0c\u9ed8\u8ba4\u4e3a /tmp/<current user>/ \u5426","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"backup/jindofs/jindotable/table_moveto/#_5","text":"\u6709\u4e00\u4e2a HDFS \u4e0a\u7684 Hive \u5206\u533a\u8868\uff0c\u5982\u4e0b\u56fe\u6240\u793a\uff1a \u60f3\u628a bbb \u548c ccc \u5206\u533a\u79fb\u52a8\u5230 OSS-HDFS \u670d\u52a1\uff0c\u5148\u7528 explain \u6a21\u5f0f\u770b\u770b\u4f1a\u79fb\u52a8\u7684\u5206\u533a\u662f\u5426\u7b26\u5408\u9884\u671f\uff0c\u53c2\u6570\u4e3a -e \u6216 -explain\uff1a \u53bb\u6389\u53c2\u6570 -e\uff0c\u771f\u6b63\u79fb\u52a8\u5206\u533a\uff1a \u6267\u884c\u5b8c\u6210\u540e\uff0c\u68c0\u67e5\u6570\u636e\uff0c\u6570\u636e\u5df2\u7ecf\u5728 OSS-HDFS \u670d\u52a1\uff1a \u518d\u79fb\u56de HDFS\uff0c\u7ed3\u679c\u5931\u8d25\u4e86\uff0c\u4f1a\u663e\u793a\u5931\u8d25\u539f\u56e0\uff0c\u539f\u6765\u662f\u76ee\u6807\u76ee\u5f55\u975e\u7a7a\uff0c\u63d0\u793a\u53ef\u4ee5\u4f7f\u7528 -overWrite \u6e05\u7a7a\u76ee\u6807\u76ee\u5f55\uff1a \u4f7f\u7528 -overWrite \u5f3a\u5236\u8986\u76d6\uff1a","title":"\u4f7f\u7528\u793a\u4f8b"},{"location":"backup/jindofs/jindotable/table_moveto/#_6","text":"\u4e3a\u4e86\u5c3d\u53ef\u80fd\u4fdd\u8bc1\u6570\u636e\u5b89\u5168\uff0c\u9632\u6b62\u6c61\u67d3\uff0c\u8be5\u547d\u4ee4\u4f1a\u81ea\u52a8\u68c0\u67e5\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u786e\u4fdd\u4e0d\u5b58\u5728\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u76ee\u5f55\u62f7\u8d1d\u3002 \u5982\u679c\u65e0\u6cd5\u6392\u9664\uff0c\u5219\u8be5\u8868\u6216\u5206\u533a\u7684\u62f7\u8d1d\u547d\u4ee4\u5c06\u6267\u884c\u5931\u8d25\u3002 \u6b64\u65f6\uff0c\u9700\u8981\u4e3b\u52a8\u5e72\u9884\uff0c\u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u6b63\u5728\u6267\u884c\u62f7\u8d1d\u4e4b\u540e\uff0c\u6e05\u7406\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u5219\u53ef\u4ee5\u91cd\u65b0\u6267\u884c\u8be5\u547d\u4ee4\u3002 \u5bf9\u4e8e\u975e\u5206\u533a\u8868\u7684\u62f7\u8d1d\u6216\u8fc1\u79fb\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u5373\u4e3a\u8868\u7ea7\u522b\u76ee\u5f55\uff1b\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u4e3a\u5f85\u62f7\u8d1d\u6216\u79fb\u52a8\u5206\u533a\u7684\u5206\u533a\u7ea7\u76ee\u5f55\uff0c \u53ea\u9700\u5bf9\u5f85\u62f7\u8d1d\u7684\u5206\u533a\u8fdb\u884c\u6e05\u7406\u3002","title":"\u5f02\u5e38\u72b6\u6001\u5904\u7406"},{"location":"backup/jindofs/jindotable/table_moveto/#_7","text":"\u5982\u679c\u8be5\u547d\u4ee4\u88ab\u5f02\u5e38\u4e2d\u6b62\uff0c\u5219\u53ef\u80fd\u51fa\u73b0\u9700\u8981\u4e3b\u52a8\u5e72\u9884\u7684\u60c5\u51b5\u3002\u6b64\u65f6\u62f7\u8d1d\u8fd8\u672a\u5b8c\u6210\uff0c\u6e90\u6570\u636e\u4e0e\u8868\u7684\u5143\u4fe1\u606f\u5747\u672a\u6539\u53d8\uff0c \u6570\u636e\u4ecd\u5904\u4e8e\u5b89\u5168\u72b6\u6001\u3002\u5e38\u89c1\u7684\u5f02\u5e38\u4e2d\u6b62\u6709\u4e24\u79cd\u60c5\u51b5\uff1a * \u7528\u6237\u5728\u547d\u4ee4\u5c1a\u672a\u7ed3\u675f\u65f6\uff0c\u4e3b\u52a8\u6740\u6389\u547d\u4ee4\u8fdb\u7a0b * \u7531\u4e8e\u5185\u5b58\u6ea2\u51fa\u7b49\u5f02\u5e38\uff0c\u8fdb\u7a0b\u5f02\u5e38\u4e2d\u6b62 \u5982\u679c\u62f7\u8d1d\u8868\u6216\u5206\u533a\u65f6\u62f7\u8d1d\u5931\u8d25\u5e76\u663e\u793a Conflicts found \uff0c\u5219\u5c5e\u4e8e\u5f02\u5e38\u72b6\u6001\uff0c\u9700\u4e3b\u52a8\u5e72\u9884\u3002","title":"\u51fa\u73b0\u539f\u56e0"},{"location":"backup/jindofs/jindotable/table_moveto/#_8","text":"\u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u7684\u76ee\u6807\u8def\u5f84\u62f7\u8d1d\u6570\u636e\u3002\u7279\u522b\u662f\uff0c\u786e\u4fdd\u6ca1\u6709 DistCp/JindoDistCp \u7b49\u5206\u5e03\u5f0f\u62f7\u8d1d\u547d\u4ee4\u3002 \u5220\u9664\u76ee\u7684\u5730\u76ee\u5f55\u3002\u5bf9\u4e8e\u975e\u5206\u533a\u8868\uff0c\u5220\u9664\u8868\u4e00\u7ea7\u76ee\u5f55\u3002\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u5220\u9664\u5b58\u5728\u51b2\u7a81\u7684\u5206\u533a\u7ea7\u76ee\u5f55\u3002\u5207\u52ff\u5220\u9664\u6e90\u76ee\u5f55\u3002","title":"\u5904\u7406\u65b9\u6cd5"},{"location":"backup/jindofs/jindotable/table_shell/","text":"\u4f7f\u7528 JindoTable \u547d\u4ee4\u5b9e\u73b0 Hive \u8868\u548c\u5206\u533a\u7684\u6570\u636e\u6cbb\u7406 \u00a7 JindoTable \u63d0\u4f9b\u591a\u4e2a\u547d\u4ee4\u884c\u547d\u4ee4\uff0c\u53ef\u4ee5\u4fbf\u6377\u67e5\u8be2 Hive \u8868\u548c\u5206\u533a\u7684\u57fa\u672c\u4fe1\u606f\u6216\u7a7a\u95f4\u5360\u7528\uff0c \u6216\u8005\u5b9e\u73b0\u6570\u636e\u7684\u8fc1\u79fb\u3001\u5f52\u6863\u3002\u672c\u6587\u4ecb\u7ecd\u8fd9\u4e9b\u547d\u4ee4\u884c\u547d\u4ee4\u7684\u7528\u6cd5\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u90e8\u7f72 JindoSDK \u00a7 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8 \u3002 \u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883 \u00a7 \u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e \u4f7f\u7528\u8bf4\u660e \u00a7 \u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f \u00a7 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u67e5\u770b JindoTable \u652f\u6301\u7684\u6240\u6709\u547d\u4ee4\uff1a jindotable -help \u5982\u679c\u60f3\u67e5\u770b\u547d\u4ee4\u7684\u5177\u4f53\u7528\u6cd5\uff0c\u6267\u884c\uff1a jindotable -help <cmd> \u5176\u4e2d\uff0c <cmd> \u53ef\u4ee5\u662f showTable \u3001 moveTo \u3001 archiveTable \u7b49\u3002 showTable \u547d\u4ee4 \u00a7 jindotable -showTable -t <dbName.tableName> \u5176\u4e2d <dbName.tableName> \u4e3a\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a \u5e93\u540d.\u8868\u540d \u529f\u80fd\uff1a\u5982\u679c\u662f\u5206\u533a\u8868\uff0c\u5219\u5c55\u793a\u6240\u6709\u5206\u533a\uff1b\u5982\u679c\u662f\u975e\u5206\u533a\u8868\uff0c\u5219\u8fd4\u56de\u8868\u7684\u5b58\u50a8\u60c5\u51b5\u3002 \u793a\u4f8b\uff1a\u5c55\u793a db1.t1 \u5206\u533a\u8868\u7684\u6240\u6709\u5206\u533a\u3002 jindo table -showTable -t db1.t1 showPartition \u547d\u4ee4 \u00a7 jindotable -showPartition -t <dbName.tableName> -p <partitionSpec> \u5176\u4e2d <dbName.tableName> \u4e3a\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a \u5e93\u540d.\u8868\u540d \uff1b <partitionSpec> \u4e3a\u5206\u533a\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a col1=v1,col2=v2 \u529f\u80fd\uff1a\u8fd4\u56de\u5206\u533a\u7684\u5b58\u50a8\u60c5\u51b5\u3002 \u793a\u4f8b\uff1a\u8fd4\u56de\u5206\u533a\u8868 db1.t1 \u5728 2020-10-12 \u65e5\u7684\u5b58\u50a8\u60c5\u51b5\u3002 jindo table -showPartition -t db1.t1 -p date=2020-10-12 listTables \u547d\u4ee4 \u00a7 jindotable -listTables [-db] <dbName> \u5176\u4e2d <dbName> \u4e3a\u6570\u636e\u5e93\u540d\u79f0\uff0c\u53ef\u7701\u7565\uff0c\u7701\u7565\u5219\u4e3a\u9ed8\u8ba4\u6570\u636e\u5e93\u3002 \u529f\u80fd\uff1a\u5c55\u793a\u6307\u5b9a\u6570\u636e\u5e93\u4e2d\u7684\u6240\u6709\u8868\u3002\u4e0d\u6307\u5b9a -db \u65f6\u9ed8\u8ba4\u5c55\u793a default \u5e93\u4e2d\u7684\u8868\u3002 \u793a\u4f8b\uff1a\u5217\u51fa\u6570\u636e\u5e93 db1 \u4e2d\u7684\u8868\u3002 jindo table -listTables -db db1 moveTo \u547d\u4ee4 \u00a7 \u8bf7\u53c2\u8003 \u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1 archiveTable/unarchiveTable \u547d\u4ee4 \u00a7 \u6682\u4e0d\u652f\u6301\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff0c\u5bf9\u4e8e OSS \u670d\u52a1\uff0c\u8bf7\u53c2\u8003 JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a cacheTable/uncacheTable \u547d\u4ee4 \u00a7 \u4ec5\u652f\u6301\u914d\u5907 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7684\u573a\u666f\uff0c\u8bf7\u53c2\u8003 \u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e","title":"\u4f7f\u7528 JindoTable \u547d\u4ee4\u5b9e\u73b0 Hive \u8868\u548c\u5206\u533a\u7684\u6570\u636e\u6cbb\u7406"},{"location":"backup/jindofs/jindotable/table_shell/#jindotable-hive","text":"JindoTable \u63d0\u4f9b\u591a\u4e2a\u547d\u4ee4\u884c\u547d\u4ee4\uff0c\u53ef\u4ee5\u4fbf\u6377\u67e5\u8be2 Hive \u8868\u548c\u5206\u533a\u7684\u57fa\u672c\u4fe1\u606f\u6216\u7a7a\u95f4\u5360\u7528\uff0c \u6216\u8005\u5b9e\u73b0\u6570\u636e\u7684\u8fc1\u79fb\u3001\u5f52\u6863\u3002\u672c\u6587\u4ecb\u7ecd\u8fd9\u4e9b\u547d\u4ee4\u884c\u547d\u4ee4\u7684\u7528\u6cd5\u3002","title":"\u4f7f\u7528 JindoTable \u547d\u4ee4\u5b9e\u73b0 Hive \u8868\u548c\u5206\u533a\u7684\u6570\u636e\u6cbb\u7406"},{"location":"backup/jindofs/jindotable/table_shell/#_1","text":"","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/jindofs/jindotable/table_shell/#jindosdk","text":"\u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8 \u3002","title":"\u5df2\u90e8\u7f72 JindoSDK"},{"location":"backup/jindofs/jindotable/table_shell/#hadoop-hive","text":"\u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e","title":"\u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883"},{"location":"backup/jindofs/jindotable/table_shell/#_2","text":"","title":"\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindotable/table_shell/#_3","text":"\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u67e5\u770b JindoTable \u652f\u6301\u7684\u6240\u6709\u547d\u4ee4\uff1a jindotable -help \u5982\u679c\u60f3\u67e5\u770b\u547d\u4ee4\u7684\u5177\u4f53\u7528\u6cd5\uff0c\u6267\u884c\uff1a jindotable -help <cmd> \u5176\u4e2d\uff0c <cmd> \u53ef\u4ee5\u662f showTable \u3001 moveTo \u3001 archiveTable \u7b49\u3002","title":"\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f"},{"location":"backup/jindofs/jindotable/table_shell/#showtable","text":"jindotable -showTable -t <dbName.tableName> \u5176\u4e2d <dbName.tableName> \u4e3a\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a \u5e93\u540d.\u8868\u540d \u529f\u80fd\uff1a\u5982\u679c\u662f\u5206\u533a\u8868\uff0c\u5219\u5c55\u793a\u6240\u6709\u5206\u533a\uff1b\u5982\u679c\u662f\u975e\u5206\u533a\u8868\uff0c\u5219\u8fd4\u56de\u8868\u7684\u5b58\u50a8\u60c5\u51b5\u3002 \u793a\u4f8b\uff1a\u5c55\u793a db1.t1 \u5206\u533a\u8868\u7684\u6240\u6709\u5206\u533a\u3002 jindo table -showTable -t db1.t1","title":"showTable \u547d\u4ee4"},{"location":"backup/jindofs/jindotable/table_shell/#showpartition","text":"jindotable -showPartition -t <dbName.tableName> -p <partitionSpec> \u5176\u4e2d <dbName.tableName> \u4e3a\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a \u5e93\u540d.\u8868\u540d \uff1b <partitionSpec> \u4e3a\u5206\u533a\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a col1=v1,col2=v2 \u529f\u80fd\uff1a\u8fd4\u56de\u5206\u533a\u7684\u5b58\u50a8\u60c5\u51b5\u3002 \u793a\u4f8b\uff1a\u8fd4\u56de\u5206\u533a\u8868 db1.t1 \u5728 2020-10-12 \u65e5\u7684\u5b58\u50a8\u60c5\u51b5\u3002 jindo table -showPartition -t db1.t1 -p date=2020-10-12","title":"showPartition \u547d\u4ee4"},{"location":"backup/jindofs/jindotable/table_shell/#listtables","text":"jindotable -listTables [-db] <dbName> \u5176\u4e2d <dbName> \u4e3a\u6570\u636e\u5e93\u540d\u79f0\uff0c\u53ef\u7701\u7565\uff0c\u7701\u7565\u5219\u4e3a\u9ed8\u8ba4\u6570\u636e\u5e93\u3002 \u529f\u80fd\uff1a\u5c55\u793a\u6307\u5b9a\u6570\u636e\u5e93\u4e2d\u7684\u6240\u6709\u8868\u3002\u4e0d\u6307\u5b9a -db \u65f6\u9ed8\u8ba4\u5c55\u793a default \u5e93\u4e2d\u7684\u8868\u3002 \u793a\u4f8b\uff1a\u5217\u51fa\u6570\u636e\u5e93 db1 \u4e2d\u7684\u8868\u3002 jindo table -listTables -db db1","title":"listTables \u547d\u4ee4"},{"location":"backup/jindofs/jindotable/table_shell/#moveto","text":"\u8bf7\u53c2\u8003 \u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1","title":"moveTo \u547d\u4ee4"},{"location":"backup/jindofs/jindotable/table_shell/#archivetableunarchivetable","text":"\u6682\u4e0d\u652f\u6301\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff0c\u5bf9\u4e8e OSS \u670d\u52a1\uff0c\u8bf7\u53c2\u8003 JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a","title":"archiveTable/unarchiveTable \u547d\u4ee4"},{"location":"backup/jindofs/jindotable/table_shell/#cachetableuncachetable","text":"\u4ec5\u652f\u6301\u914d\u5907 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7684\u573a\u666f\uff0c\u8bf7\u53c2\u8003 \u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e","title":"cacheTable/uncacheTable \u547d\u4ee4"},{"location":"backup/jindofs/presto/jindosdk_on_presto/","text":"Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u6570\u636e \u00a7 Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002 \u6b65\u9aa4 \u00a7 1. \u5b89\u88c5 jar \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/ 2. \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> 3. \u914d\u7f6e OSS Access Key \u00a7 \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002 4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u00a7 \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 5. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 \u4f7f\u7528\u793a\u4f8b \u00a7 \u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236\u53f0 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='oss://<Bucket>.<Endpoint>/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofs/presto/jindosdk_on_presto/#presto-jindosdk-oss-hdfs","text":"Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002","title":"Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofs/presto/jindosdk_on_presto/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofs/presto/jindosdk_on_presto/#1-jar","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/","title":"1. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofs/presto/jindosdk_on_presto/#2-jindosdk-oss","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration>","title":"2. \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b"},{"location":"backup/jindofs/presto/jindosdk_on_presto/#3-oss-access-key","text":"\u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002","title":"3. \u914d\u7f6e OSS Access Key"},{"location":"backup/jindofs/presto/jindosdk_on_presto/#4-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002","title":"4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint"},{"location":"backup/jindofs/presto/jindosdk_on_presto/#5-presto","text":"","title":"5. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"backup/jindofs/presto/jindosdk_on_presto/#_2","text":"\u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236\u53f0 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='oss://<Bucket>.<Endpoint>/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"\u4f7f\u7528\u793a\u4f8b"},{"location":"backup/jindofs/security/jindosdk_credential_provider_bucket_dls/","text":"JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u6309 bucket \u914d\u7f6e\u4f7f\u7528\u8bf4\u660e \u00a7 \u6309\u7167 bucket \u914d\u7f6e Credential Provider \u00a7 <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleAliyunCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.AliyunCredentialsProvider\u7684\u5b9e\u73b0\u7c7b\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u8bf4\u660e XXX \u4e3a OSS-HDFS \u670d\u52a1 bucket \u540d\u79f0\u3002 Credential Provider \u7c7b\u578b \u00a7 1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Secret</value> </property> <property> <name>fs.oss.bucket.XXX.securityToken</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration> 2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Secret</value> </property> </configuration> 3. EnvironmentVariableCredentialsProvider \u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6 AccessKey\u3002 \u00a7 \u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Id OSS_ACCESS_KEY_SECRET OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Secret OSS_SECURITY_TOKEN OSS-HDFS \u670d\u52a1 bucket \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548Token\u65f6\u9700\u8981\u3002 4. CommonCredentialsProvider \u4e3a\u901a\u7528\u914d\u7f6e\u3002 \u00a7 \u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration>","title":"JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u6309 bucket \u914d\u7f6e\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/security/jindosdk_credential_provider_bucket_dls/#jindosdk-oss-hdfs-credential-provider-bucket","text":"","title":"JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u6309 bucket \u914d\u7f6e\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/security/jindosdk_credential_provider_bucket_dls/#bucket-credential-provider","text":"<configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleAliyunCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.AliyunCredentialsProvider\u7684\u5b9e\u73b0\u7c7b\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u8bf4\u660e XXX \u4e3a OSS-HDFS \u670d\u52a1 bucket \u540d\u79f0\u3002","title":"\u6309\u7167 bucket \u914d\u7f6e Credential Provider"},{"location":"backup/jindofs/security/jindosdk_credential_provider_bucket_dls/#credential-provider","text":"","title":"Credential Provider \u7c7b\u578b"},{"location":"backup/jindofs/security/jindosdk_credential_provider_bucket_dls/#1-temporarycredentialsprovider-accesskey-securitytoken-oss-hdfs","text":"\u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Secret</value> </property> <property> <name>fs.oss.bucket.XXX.securityToken</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration>","title":"1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u60c5\u51b5\u3002"},{"location":"backup/jindofs/security/jindosdk_credential_provider_bucket_dls/#2-simplecredentialsprovider-accesskey-oss-hdfs","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Secret</value> </property> </configuration>","title":"2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u60c5\u51b5\u3002"},{"location":"backup/jindofs/security/jindosdk_credential_provider_bucket_dls/#3-environmentvariablecredentialsprovider-accesskey","text":"\u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Id OSS_ACCESS_KEY_SECRET OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Secret OSS_SECURITY_TOKEN OSS-HDFS \u670d\u52a1 bucket \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548Token\u65f6\u9700\u8981\u3002","title":"3. EnvironmentVariableCredentialsProvider \u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6 AccessKey\u3002"},{"location":"backup/jindofs/security/jindosdk_credential_provider_bucket_dls/#4-commoncredentialsprovider","text":"\u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration>","title":"4. CommonCredentialsProvider \u4e3a\u901a\u7528\u914d\u7f6e\u3002"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/","text":"JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u00a7 \u57fa\u672c\u914d\u7f6e\u65b9\u5f0f \u00a7 \u60a8\u53ef\u4ee5\u5c06\u5df2\u5f00\u542f\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 \u7684 Access Key ID \u3001 Access Secret \u3001 Endpoint \u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \uff0c\u914d\u7f6e\u9879\u5982\u4e0b\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>cn-xxx.oss-dls.aliyuncs.com</value> </property> </configuration> \u4f7f\u7528 Hadoop Credential Providers \u5b58\u50a8 AccessKey \u4fe1\u606f \u00a7 \u4e0a\u9762\u7684 fs.oss.accessKeyId \u548c fs.oss.accessKeySecret \u5c06\u4f1a\u660e\u6587\u914d\u7f6e\u5728 core-site.xml \u4e2d\uff0c\u53ef\u4ee5\u5c06\u5176\u4ee5\u52a0\u5bc6\u5bf9\u65b9\u5f0f\u5b58\u50a8\u81f3 Hadoop Credential Providers\u6587\u4ef6\u4e2d\u3002 \u4f7f\u7528Hadoop\u63d0\u4f9b\u7684\u547d\u4ee4\uff0c\u5b58\u50a8AccessKey\u548cSecurityToken\u4fe1\u606f\u81f3Credential\u6587\u4ef6\u4e2d\u3002\u547d\u4ee4\u683c\u5f0f\u5982\u4e0b\u3002 hadoop credential <subcommand> [options] \u4f8b\u5982\uff0c\u5b58\u50a8AccessKey\u548cToken\u4fe1\u606f\u81f3JECKS\u6587\u4ef6\u4e2d\uff0c\u9664\u4e86\u4f7f\u7528\u6587\u4ef6\u6743\u9650\u4fdd\u62a4\u8be5\u6587\u4ef6\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u6307\u5b9a\u5bc6\u7801\u52a0\u5bc6\u5b58\u50a8\u4fe1\u606f\uff0c\u5982\u679c\u4e0d\u6307\u5b9a\u5bc6\u7801\u5219\u4f7f\u7528\u9ed8\u8ba4\u5b57\u7b26\u4e32\u52a0\u5bc6\u3002 hadoop credential create fs.oss.accessKeyId -value AAA -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.accessKeySecret -value BBB -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.securityToken -value CCC -provider jceks://file/root/oss.jceks \u751f\u6210Credential\u6587\u4ef6\u540e\uff0c\u60a8\u9700\u8981\u914d\u7f6e\u4e0b\u9762\u7684\u53c2\u6570\u6765\u6307\u5b9aProvider\u7684\u7c7b\u578b\u548c\u4f4d\u7f6e\u3002 <configuration> <property> <name>fs.oss.security.credential.provider.path</name> <value>jceks://file/root/oss.jceks</value> <description>\u914d\u7f6e\u5b58\u50a8AK\u7684Credential\u6587\u4ef6\u3002\u4f8b\u5982\uff0cjceks://file/${user.home}/oss.jceks\u4e3aHOME\u4e0b\u7684oss.jceks\u6587\u4ef6</description> </property> </configuration> \u4f7f\u7528 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u00a7 \u9ed8\u8ba4\u4f1a\u914d\u7f6e SimpleCredentialsProvider\uff0cEnvironmentVariableCredentialsProvider\uff0cCommonCredentialsProvider \u8fd9\u4e09\u4e2aCredential Provider, \u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6 Credential \u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684 Credential\u3002 <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.JindoCredentialsProvider\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u60a8\u53ef\u4ee5\u6839\u636e\u60c5\u51b5\uff0c\u9009\u62e9\u4e0d\u540c\u7684 Credential Provider\uff0c\u652f\u6301\u5982\u4e0bProvider\uff1a 1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Secret</value> </property> <property> <name>fs.oss.securityToken</name> <value>OSS-HDFS \u670d\u52a1\u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration> 2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Secret</value> </property> </configuration> 3. EnvironmentVariableCredentialsProvider\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6AK\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS-HDFS \u670d\u52a1\u7684 AccessKey Id OSS_ACCESS_KEY_SECRET OSS-HDFS \u670d\u52a1\u7684 AccessKey Secret OSS_SECURITY_TOKEN OSS-HDFS \u670d\u52a1\u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002 4. CommonCredentialsProvider \u4e3a\u901a\u7528\u914d\u7f6e\u3002 \u00a7 \u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS-HDFS \u670d\u52a1\u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration> 5. CustomCredentialsProvider \u5bf9\u63a5\u5b9a\u5236\u7684\u514d\u5bc6\u670d\u52a1\u3002 \u00a7 \u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CustomCredentialsProvider</value> </property> </configuration> \u914d\u7f6e\u514d\u5bc6\u670d\u52a1\u5730\u5740 <configuration> <property> <name>aliyun.oss.provider.url</name> <value>\u514d\u5bc6\u670d\u52a1\u5730\u5740</value> </property> </configuration> aliyun.oss.provider.url \u652f\u6301 http(s) \u534f\u8bae \u548c secrets \u534f\u8bae\uff1a a. http(s) \u534f\u8bae http(s) \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a http://localhost:1234/sts , http \u514d\u5bc6\u534f\u8bae\u8981\u6c42\u8fd4\u56de\u7ed3\u679c\u4e3a Json \u683c\u5f0f\uff0c\u5982\u679c\u60a8\u9700\u8981\u5bf9\u63a5\u60a8\u7684 http \u514d\u5bc6\u670d\u52a1\uff0c\u514d\u5bc6\u670d\u52a1\u7684\u8bbe\u8ba1\u53c2\u8003 \u300a\u901a\u8fc7API\u4f7f\u7528\u5b9e\u4f8bRAM\u89d2\u8272#\u4e34\u65f6\u6388\u6743Token\u300b { \"AccessKeyId\" : \"XXXXXXXXX\", \"AccessKeySecret\" : \"XXXXXXXXX\", \"Expiration\" : \"2020-11-01T05:20:01Z\", \"SecurityToken\" : \"XXXXXXXXX\", \"LastUpdated\" : \"2020-10-31T23:20:01Z\", \"Code\" : \"Success\" } b. Secrets \u534f\u8bae Secrets \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a secrets:///local_path_prefix \uff0c\u5e38\u89c1\u4f7f\u7528\u4e8e k8s \u573a\u666f \uff0c \u5176\u4e2d local_path_prefix \u4e3a\u8def\u5f84\u524d\u7f00\uff0c\u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOssAccessKeyId /secret/JindoOssAccessKeySecret /secret/JindoOssSecurityToken \u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss/ \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOss/AccessKeyId /secret/JindoOss/AccessKeySecret /secret/JindoOss/SecurityToken JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 OSS-HDFS \u670d\u52a1 bucket \u914d\u7f6e\u4e0d\u540c\u7684 Credential Provider \u00a7 \u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u6309 bucket \u914d\u7f6e\u4f7f\u7528\u8bf4\u660e \u3002","title":"JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/#jindosdk-oss-hdfs-credential-provider","text":"","title":"JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/#_1","text":"\u60a8\u53ef\u4ee5\u5c06\u5df2\u5f00\u542f\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 \u7684 Access Key ID \u3001 Access Secret \u3001 Endpoint \u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \uff0c\u914d\u7f6e\u9879\u5982\u4e0b\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>cn-xxx.oss-dls.aliyuncs.com</value> </property> </configuration>","title":"\u57fa\u672c\u914d\u7f6e\u65b9\u5f0f"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/#hadoop-credential-providers-accesskey","text":"\u4e0a\u9762\u7684 fs.oss.accessKeyId \u548c fs.oss.accessKeySecret \u5c06\u4f1a\u660e\u6587\u914d\u7f6e\u5728 core-site.xml \u4e2d\uff0c\u53ef\u4ee5\u5c06\u5176\u4ee5\u52a0\u5bc6\u5bf9\u65b9\u5f0f\u5b58\u50a8\u81f3 Hadoop Credential Providers\u6587\u4ef6\u4e2d\u3002 \u4f7f\u7528Hadoop\u63d0\u4f9b\u7684\u547d\u4ee4\uff0c\u5b58\u50a8AccessKey\u548cSecurityToken\u4fe1\u606f\u81f3Credential\u6587\u4ef6\u4e2d\u3002\u547d\u4ee4\u683c\u5f0f\u5982\u4e0b\u3002 hadoop credential <subcommand> [options] \u4f8b\u5982\uff0c\u5b58\u50a8AccessKey\u548cToken\u4fe1\u606f\u81f3JECKS\u6587\u4ef6\u4e2d\uff0c\u9664\u4e86\u4f7f\u7528\u6587\u4ef6\u6743\u9650\u4fdd\u62a4\u8be5\u6587\u4ef6\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u6307\u5b9a\u5bc6\u7801\u52a0\u5bc6\u5b58\u50a8\u4fe1\u606f\uff0c\u5982\u679c\u4e0d\u6307\u5b9a\u5bc6\u7801\u5219\u4f7f\u7528\u9ed8\u8ba4\u5b57\u7b26\u4e32\u52a0\u5bc6\u3002 hadoop credential create fs.oss.accessKeyId -value AAA -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.accessKeySecret -value BBB -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.securityToken -value CCC -provider jceks://file/root/oss.jceks \u751f\u6210Credential\u6587\u4ef6\u540e\uff0c\u60a8\u9700\u8981\u914d\u7f6e\u4e0b\u9762\u7684\u53c2\u6570\u6765\u6307\u5b9aProvider\u7684\u7c7b\u578b\u548c\u4f4d\u7f6e\u3002 <configuration> <property> <name>fs.oss.security.credential.provider.path</name> <value>jceks://file/root/oss.jceks</value> <description>\u914d\u7f6e\u5b58\u50a8AK\u7684Credential\u6587\u4ef6\u3002\u4f8b\u5982\uff0cjceks://file/${user.home}/oss.jceks\u4e3aHOME\u4e0b\u7684oss.jceks\u6587\u4ef6</description> </property> </configuration>","title":"\u4f7f\u7528 Hadoop Credential Providers \u5b58\u50a8 AccessKey \u4fe1\u606f"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/#jindosdk-oss-hdfs-credential-provider_1","text":"\u9ed8\u8ba4\u4f1a\u914d\u7f6e SimpleCredentialsProvider\uff0cEnvironmentVariableCredentialsProvider\uff0cCommonCredentialsProvider \u8fd9\u4e09\u4e2aCredential Provider, \u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6 Credential \u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684 Credential\u3002 <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.JindoCredentialsProvider\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u60a8\u53ef\u4ee5\u6839\u636e\u60c5\u51b5\uff0c\u9009\u62e9\u4e0d\u540c\u7684 Credential Provider\uff0c\u652f\u6301\u5982\u4e0bProvider\uff1a","title":"\u4f7f\u7528 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/#1-temporarycredentialsprovider-accesskey-securitytoken-oss","text":"\u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Secret</value> </property> <property> <name>fs.oss.securityToken</name> <value>OSS-HDFS \u670d\u52a1\u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration>","title":"1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/#2-simplecredentialsprovider-accesskey-oss","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Secret</value> </property> </configuration>","title":"2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/#3-environmentvariablecredentialsproviderak","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS-HDFS \u670d\u52a1\u7684 AccessKey Id OSS_ACCESS_KEY_SECRET OSS-HDFS \u670d\u52a1\u7684 AccessKey Secret OSS_SECURITY_TOKEN OSS-HDFS \u670d\u52a1\u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002","title":"3. EnvironmentVariableCredentialsProvider\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6AK\u3002"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/#4-commoncredentialsprovider","text":"\u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS-HDFS \u670d\u52a1\u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration>","title":"4. CommonCredentialsProvider \u4e3a\u901a\u7528\u914d\u7f6e\u3002"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/#5-customcredentialsprovider","text":"\u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CustomCredentialsProvider</value> </property> </configuration> \u914d\u7f6e\u514d\u5bc6\u670d\u52a1\u5730\u5740 <configuration> <property> <name>aliyun.oss.provider.url</name> <value>\u514d\u5bc6\u670d\u52a1\u5730\u5740</value> </property> </configuration> aliyun.oss.provider.url \u652f\u6301 http(s) \u534f\u8bae \u548c secrets \u534f\u8bae\uff1a a. http(s) \u534f\u8bae http(s) \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a http://localhost:1234/sts , http \u514d\u5bc6\u534f\u8bae\u8981\u6c42\u8fd4\u56de\u7ed3\u679c\u4e3a Json \u683c\u5f0f\uff0c\u5982\u679c\u60a8\u9700\u8981\u5bf9\u63a5\u60a8\u7684 http \u514d\u5bc6\u670d\u52a1\uff0c\u514d\u5bc6\u670d\u52a1\u7684\u8bbe\u8ba1\u53c2\u8003 \u300a\u901a\u8fc7API\u4f7f\u7528\u5b9e\u4f8bRAM\u89d2\u8272#\u4e34\u65f6\u6388\u6743Token\u300b { \"AccessKeyId\" : \"XXXXXXXXX\", \"AccessKeySecret\" : \"XXXXXXXXX\", \"Expiration\" : \"2020-11-01T05:20:01Z\", \"SecurityToken\" : \"XXXXXXXXX\", \"LastUpdated\" : \"2020-10-31T23:20:01Z\", \"Code\" : \"Success\" } b. Secrets \u534f\u8bae Secrets \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a secrets:///local_path_prefix \uff0c\u5e38\u89c1\u4f7f\u7528\u4e8e k8s \u573a\u666f \uff0c \u5176\u4e2d local_path_prefix \u4e3a\u8def\u5f84\u524d\u7f00\uff0c\u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOssAccessKeyId /secret/JindoOssAccessKeySecret /secret/JindoOssSecurityToken \u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss/ \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOss/AccessKeyId /secret/JindoOss/AccessKeySecret /secret/JindoOss/SecurityToken","title":"5. CustomCredentialsProvider \u5bf9\u63a5\u5b9a\u5236\u7684\u514d\u5bc6\u670d\u52a1\u3002"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/#jindosdk-oss-hdfs-bucket-credential-provider","text":"\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u6309 bucket \u914d\u7f6e\u4f7f\u7528\u8bf4\u660e \u3002","title":"JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 OSS-HDFS \u670d\u52a1 bucket \u914d\u7f6e\u4e0d\u540c\u7684 Credential Provider"},{"location":"backup/jindofs/spark/jindosdk_ide_spark/","text":"Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 \u00a7 \u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindosdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.spark.sql.SparkSession; public class TestJindoSDK { public static void main(String[] args) throws Exception { SparkSession spark = SparkSession .builder() .config(\"spark.hadoop.fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\") .config(\"spark.hadoop.fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\") .config(\"spark.hadoop.fs.oss.accessKeyId\", \"xxx\") .config(\"spark.hadoop.fs.oss.accessKeySecret\", \"xxx\") // set accessKey, secret, endpoint and so on. .appName(\"TestJindoSDK\") .getOrCreate(); spark.read().parquet(\"oss://bucket.endpoint/xxx\").count(); spark.stop(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency>","title":"Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5"},{"location":"backup/jindofs/spark/jindosdk_ide_spark/#spark-jindosdk-ide","text":"\u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindosdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.spark.sql.SparkSession; public class TestJindoSDK { public static void main(String[] args) throws Exception { SparkSession spark = SparkSession .builder() .config(\"spark.hadoop.fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\") .config(\"spark.hadoop.fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\") .config(\"spark.hadoop.fs.oss.accessKeyId\", \"xxx\") .config(\"spark.hadoop.fs.oss.accessKeySecret\", \"xxx\") // set accessKey, secret, endpoint and so on. .appName(\"TestJindoSDK\") .getOrCreate(); spark.read().parquet(\"oss://bucket.endpoint/xxx\").count(); spark.stop(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency>","title":"Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5"},{"location":"backup/jindofs/spark/jindosdk_on_spark/","text":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e \u00a7 JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411Hadoop/Spark\u751f\u6001\u7684OSS\u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684Hadoop FileSystem\u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fddSpark\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/ 3. \u914d\u7f6e JindoSDK \u00a7 \u5168\u5c40\u914d\u7f6e, \u4fee\u6539 Spark \u4f7f\u7528\u7684 core-site.xml \u00a7 \u914d\u7f6e JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u4efb\u52a1\u7ea7\u522b\u914d\u7f6e \u00a7 \u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx 4. \u4f7f\u7528 Spark \u8bbf\u95ee OSS \u00a7 \u521b\u5efa\u8868 create table test_oss (c1 string) location \"oss://bucket.endpoint/dir\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss; 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofs/spark/jindosdk_on_spark/#spark-jindosdk-oss-hdfs-jindofs","text":"JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411Hadoop/Spark\u751f\u6001\u7684OSS\u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684Hadoop FileSystem\u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofs/spark/jindosdk_on_spark/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofs/spark/jindosdk_on_spark/#1-spark-hadoop","text":"\u786e\u4fddSpark\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"backup/jindofs/spark/jindosdk_on_spark/#2-spark-classpath-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/","title":"2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK"},{"location":"backup/jindofs/spark/jindosdk_on_spark/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofs/spark/jindosdk_on_spark/#spark-core-sitexml","text":"\u914d\u7f6e JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002","title":"\u5168\u5c40\u914d\u7f6e, \u4fee\u6539 Spark \u4f7f\u7528\u7684core-site.xml"},{"location":"backup/jindofs/spark/jindosdk_on_spark/#_2","text":"\u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx","title":"\u4efb\u52a1\u7ea7\u522b\u914d\u7f6e"},{"location":"backup/jindofs/spark/jindosdk_on_spark/#4-spark-oss","text":"\u521b\u5efa\u8868 create table test_oss (c1 string) location \"oss://bucket.endpoint/dir\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss;","title":"4. \u4f7f\u7528 Spark \u8bbf\u95ee OSS"},{"location":"backup/jindofs/spark/jindosdk_on_spark/#5","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofs/sqoop/kitesdk_on_sqoop/","text":"Sqoop \u4f7f\u7528 Kite SDK \u8bfb\u5199\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 \u5728\u96c6\u7fa4\u4e0a\u6709\u5f00\u6e90\u7248\u672c Sqoop \u8f6f\u4ef6\uff0c\u7248\u672c\u4e0d\u4f4e\u4e8e 1.4.7\u3002 Sqoop \u4f9d\u8d56\u7684 Hadoop \u73af\u5883\u53ef\u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff0c\u53ef\u53c2\u8003 Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u4f7f\u7528 Kite SDK \u8bfb\u5199 OSS-HDFS \u670d\u52a1 \u00a7 Sqoop \u672c\u8eab\u5e76\u4e0d\u652f\u6301\u5bf9 OSS \u7684\u8bfb\u5199\uff0c\u9700\u8981\u4f7f\u7528\u7b2c\u4e09\u65b9 Kite SDK \u8fdb\u884c URI \u7684\u8f6c\u6362\u4ece\u800c\u8fdb\u884c\u6570\u636e\u4ea4\u4e92\u3002 \u6b65\u9aa4 \u00a7 1. \u5b89\u88c5 jar \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 jar \u5305 kite-data-oss-3.4.0.jar ( \u4e0b\u8f7d\u9875\u9762 ))\uff0c\u5c06 sdk \u5305\u5b89\u88c5\u5230 Sqoop \u7684 classpath \u76ee\u5f55\u4e0b\u3002 cp ./kite-data-oss-3.4.0.jar <SQOOP_HOME>/lib/kite-data-oss-3.4.0.jar 2.\u6388\u4e88\u5bf9 JAR \u7684\u6743\u9650 \u00a7 sudo chmod 755 kite-data-oss-3.4.0.jar 3.\u4f7f\u7528 oss \u8fde\u63a5\u5668\u5bfc\u5165 mysql \u4e0a\u7684\u6570\u636e \u00a7 sqoop import --connect jdbc:mysql://<host>:<port>/database --username username --password password --table yourtable --target-dir \"oss://yourbucket/dir/\" --as-parquetfile -m 5","title":"Sqoop \u4f7f\u7528 Kite SDK \u8bfb\u5199\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/sqoop/kitesdk_on_sqoop/#sqoop-kite-sdk-oss-hdfs-jindofs","text":"","title":"Sqoop \u4f7f\u7528 Kite SDK \u8bfb\u5199\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/sqoop/kitesdk_on_sqoop/#_1","text":"\u5728\u96c6\u7fa4\u4e0a\u6709\u5f00\u6e90\u7248\u672c Sqoop \u8f6f\u4ef6\uff0c\u7248\u672c\u4e0d\u4f4e\u4e8e 1.4.7\u3002 Sqoop \u4f9d\u8d56\u7684 Hadoop \u73af\u5883\u53ef\u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff0c\u53ef\u53c2\u8003 Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS-HDFS \u670d\u52a1","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/jindofs/sqoop/kitesdk_on_sqoop/#kite-sdk-oss-hdfs","text":"Sqoop \u672c\u8eab\u5e76\u4e0d\u652f\u6301\u5bf9 OSS \u7684\u8bfb\u5199\uff0c\u9700\u8981\u4f7f\u7528\u7b2c\u4e09\u65b9 Kite SDK \u8fdb\u884c URI \u7684\u8f6c\u6362\u4ece\u800c\u8fdb\u884c\u6570\u636e\u4ea4\u4e92\u3002","title":"\u4f7f\u7528 Kite SDK \u8bfb\u5199 OSS-HDFS \u670d\u52a1"},{"location":"backup/jindofs/sqoop/kitesdk_on_sqoop/#_2","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofs/sqoop/kitesdk_on_sqoop/#1-jar","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 jar \u5305 kite-data-oss-3.4.0.jar ( \u4e0b\u8f7d\u9875\u9762 ))\uff0c\u5c06 sdk \u5305\u5b89\u88c5\u5230 Sqoop \u7684 classpath \u76ee\u5f55\u4e0b\u3002 cp ./kite-data-oss-3.4.0.jar <SQOOP_HOME>/lib/kite-data-oss-3.4.0.jar","title":"1. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofs/sqoop/kitesdk_on_sqoop/#2-jar","text":"sudo chmod 755 kite-data-oss-3.4.0.jar","title":"2.\u6388\u4e88\u5bf9 JAR \u7684\u6743\u9650"},{"location":"backup/jindofs/sqoop/kitesdk_on_sqoop/#3-oss-mysql","text":"sqoop import --connect jdbc:mysql://<host>:<port>/database --username username --password password --table yourtable --target-dir \"oss://yourbucket/dir/\" --as-parquetfile -m 5","title":"3.\u4f7f\u7528 oss \u8fde\u63a5\u5668\u5bfc\u5165 mysql \u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofsx/jindofsx_cache/","text":"JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u00a7 \u7f13\u5b58\u52a0\u901f\u662fJindoFSx\u4e00\u4e2a\u6838\u5fc3\u529f\u80fd\uff0c\u5305\u62ec\u5143\u6570\u636e\u7f13\u5b58\u548c\u6570\u636e\u7f13\u5b58\uff0c\u53ef\u4ee5\u5bf9\u4e0d\u540c\u7684\u5b58\u50a8\u540e\u7aef\u63d0\u4f9b\u8bbf\u95ee\u52a0\u901f\u80fd\u529b\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u542f\u7528\u7f13\u5b58\u52a0\u901f\u529f\u80fd \u542f\u7528\u7f13\u5b58\u4f1a\u5229\u7528\u672c\u5730\u78c1\u76d8\u5bf9\u8bbf\u95ee\u7684\u70ed\u6570\u636e\u5757\u8fdb\u884c\u7f13\u5b58\uff0c\u9ed8\u8ba4\u72b6\u6001\u4e3a\u7981\u7528\uff0c\u5373\u6240\u6709OSS\u8bfb\u53d6\u90fd\u76f4\u63a5\u8bbf\u95eeOSS\u4e0a\u7684\u6570\u636e\u3002 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> <property> <!-- \u5143\u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.meta.cache.enable</name> <value>false</value> </property> <property> <!-- \u5c0f\u6587\u4ef6\u7f13\u5b58\u4f18\u5316\u5f00\u5173 --> <name>fs.jindofsx.slice.cache.enable</name> <value>false</value> </property> <property> <!-- \u5185\u5b58\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.ram.cache.enable</name> <value>false</value> </property> <property> <!-- \u77ed\u8def\u8bfb\u5f00\u5173 --> <name>fs.jindofsx.short.circuit.enable</name> <value>true</value> </property> </configuration> \u5b8c\u6210\u4ee5\u4e0a\u914d\u7f6e\u540e\uff0c\u4f5c\u4e1a\u8bfb\u53d6 OSS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002 \u6ce8\u610f\uff1a\u6b64\u914d\u7f6e\u4e3a\u5ba2\u6237\u7aef\u914d\u7f6e\uff0c\u4e0d\u9700\u8981\u91cd\u542f JindoFSx \u670d\u52a1\u3002 \u78c1\u76d8\u7a7a\u95f4\u6c34\u4f4d\u63a7\u5236 \u00a7 \u7f13\u5b58\u542f\u7528\u540e\uff0cJindoFSx \u670d\u52a1\u4f1a\u81ea\u52a8\u7ba1\u7406\u672c\u5730\u7f13\u5b58\u5907\u4efd\uff0c\u901a\u8fc7\u6c34\u4f4d\u6e05\u7406\u672c\u5730\u7f13\u5b58\uff0c\u8bf7\u60a8\u6839\u636e\u9700\u6c42\u914d\u7f6e\u4e00\u5b9a\u7684\u6bd4\u4f8b\u7528\u4e8e\u7f13\u5b58\u3002JindoFSx \u540e\u7aef\u57fa\u4e8e OSS\uff0c\u53ef\u4ee5\u63d0\u4f9b\u6d77\u91cf\u7684\u5b58\u50a8\uff0c\u4f46\u662f\u672c\u5730\u76d8\u7684\u5bb9\u91cf\u662f\u6709\u9650\u7684\uff0c\u56e0\u6b64 JindoFSx \u4f1a\u81ea\u52a8\u6dd8\u6c70\u672c\u5730\u8f83\u51b7\u7684\u6570\u636e\u5907\u4efd \u6211\u4eec\u63d0\u4f9b\u4e86 storage.watermark.high.ratio \u548c storage.watermark.low.ratio \u4e24\u4e2a\u53c2\u6570\u6765\u8c03\u8282\u672c\u5730\u5b58\u50a8\u7684\u4f7f\u7528\u5bb9\u91cf\uff0c\u503c\u5747\u4e3a0\uff5e1\u7684\u5c0f\u6570\uff0c\u8868\u793a\u4f7f\u7528\u78c1\u76d8\u7a7a\u95f4\u7684\u6bd4\u4f8b\u3002 \u4fee\u6539 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u5e76\u66f4\u65b0\u81f3\u6240\u6709\u670d\u52a1\u8282\u70b9\u3002 [jindofsx-storage]# Storage service \u914d\u7f6e storage.data-dirs=/mnt/disk1/jindofsx,/mnt/disk2/jindofsx # \u8868\u793a\u7528\u4ee5\u7f13\u5b58\u6570\u636e\u7684\u78c1\u76d8\u76ee\u5f55 storage.data-dirs.capacities=500G,500G # \u8868\u793a\u6bcf\u4e2a\u78c1\u76d8\u76ee\u5f55\u7684\u603b\u5bb9\u91cf storage.watermark.high.ratio=0.8 # \u8868\u793a\u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0a\u6c34\u4f4d\u6bd4\u4f8b\uff0c\u6bcf\u5757\u6570\u636e\u76d8\u7684 JindoFSx \u6570\u636e\u76ee\u5f55\u5360\u7528\u7684\u78c1\u76d8\u7a7a\u95f4\u5230\u8fbe\u4e0a\u6c34\u4f4d\u5373\u4f1a\u89e6\u53d1\u6e05\u7406\u3002\u9ed8\u8ba4\u503c\uff1a0.8\u3002 storage.watermark.low.ratio=0.6 # \u8868\u793a\u4f7f\u7528\u91cf\u7684\u4e0b\u6c34\u4f4d\u6bd4\u4f8b\uff0c\u89e6\u53d1\u6e05\u7406\u540e\u4f1a\u81ea\u52a8\u6e05\u7406\u51b7\u6570\u636e\uff0c\u5c06JindoFS\u6570\u636e\u76ee\u5f55\u5360\u7528\u7a7a\u95f4\u6e05\u7406\u5230\u4e0b\u6c34\u4f4d\u3002\u9ed8\u8ba4\u503c\uff1a0.6\u3002 \u8bf4\u660e: \u60a8\u53ef\u4ee5\u901a\u8fc7\u8bbe\u7f6e\u4e0a\u6c34\u4f4d\u6bd4\u4f8b\u8c03\u8282\u671f\u671b\u5206\u7ed9 JindoFSx \u7684\u78c1\u76d8\u7a7a\u95f4\uff0c\u4e0b\u6c34\u4f4d\u5fc5\u987b\u5c0f\u4e8e\u4e0a\u6c34\u4f4d\uff0c\u8bbe\u7f6e\u5408\u7406\u7684\u503c\u5373\u53ef\u3002 \u6ce8\u610f: \u914d\u7f6e\u5b8c\u6210\u540e\u9700\u91cd\u542f JindoFSx \u670d\u52a1\u3002","title":"JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofsx/jindofsx_cache/#jindofsx","text":"\u7f13\u5b58\u52a0\u901f\u662fJindoFSx\u4e00\u4e2a\u6838\u5fc3\u529f\u80fd\uff0c\u5305\u62ec\u5143\u6570\u636e\u7f13\u5b58\u548c\u6570\u636e\u7f13\u5b58\uff0c\u53ef\u4ee5\u5bf9\u4e0d\u540c\u7684\u5b58\u50a8\u540e\u7aef\u63d0\u4f9b\u8bbf\u95ee\u52a0\u901f\u80fd\u529b\u3002","title":"JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofsx/jindofsx_cache/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindofsx_cache/#jindosdk","text":"\u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u542f\u7528\u7f13\u5b58\u52a0\u901f\u529f\u80fd \u542f\u7528\u7f13\u5b58\u4f1a\u5229\u7528\u672c\u5730\u78c1\u76d8\u5bf9\u8bbf\u95ee\u7684\u70ed\u6570\u636e\u5757\u8fdb\u884c\u7f13\u5b58\uff0c\u9ed8\u8ba4\u72b6\u6001\u4e3a\u7981\u7528\uff0c\u5373\u6240\u6709OSS\u8bfb\u53d6\u90fd\u76f4\u63a5\u8bbf\u95eeOSS\u4e0a\u7684\u6570\u636e\u3002 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> <property> <!-- \u5143\u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.meta.cache.enable</name> <value>false</value> </property> <property> <!-- \u5c0f\u6587\u4ef6\u7f13\u5b58\u4f18\u5316\u5f00\u5173 --> <name>fs.jindofsx.slice.cache.enable</name> <value>false</value> </property> <property> <!-- \u5185\u5b58\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.ram.cache.enable</name> <value>false</value> </property> <property> <!-- \u77ed\u8def\u8bfb\u5f00\u5173 --> <name>fs.jindofsx.short.circuit.enable</name> <value>true</value> </property> </configuration> \u5b8c\u6210\u4ee5\u4e0a\u914d\u7f6e\u540e\uff0c\u4f5c\u4e1a\u8bfb\u53d6 OSS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002 \u6ce8\u610f\uff1a\u6b64\u914d\u7f6e\u4e3a\u5ba2\u6237\u7aef\u914d\u7f6e\uff0c\u4e0d\u9700\u8981\u91cd\u542f JindoFSx \u670d\u52a1\u3002","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/jindofsx_cache/#_2","text":"\u7f13\u5b58\u542f\u7528\u540e\uff0cJindoFSx \u670d\u52a1\u4f1a\u81ea\u52a8\u7ba1\u7406\u672c\u5730\u7f13\u5b58\u5907\u4efd\uff0c\u901a\u8fc7\u6c34\u4f4d\u6e05\u7406\u672c\u5730\u7f13\u5b58\uff0c\u8bf7\u60a8\u6839\u636e\u9700\u6c42\u914d\u7f6e\u4e00\u5b9a\u7684\u6bd4\u4f8b\u7528\u4e8e\u7f13\u5b58\u3002JindoFSx \u540e\u7aef\u57fa\u4e8e OSS\uff0c\u53ef\u4ee5\u63d0\u4f9b\u6d77\u91cf\u7684\u5b58\u50a8\uff0c\u4f46\u662f\u672c\u5730\u76d8\u7684\u5bb9\u91cf\u662f\u6709\u9650\u7684\uff0c\u56e0\u6b64 JindoFSx \u4f1a\u81ea\u52a8\u6dd8\u6c70\u672c\u5730\u8f83\u51b7\u7684\u6570\u636e\u5907\u4efd \u6211\u4eec\u63d0\u4f9b\u4e86 storage.watermark.high.ratio \u548c storage.watermark.low.ratio \u4e24\u4e2a\u53c2\u6570\u6765\u8c03\u8282\u672c\u5730\u5b58\u50a8\u7684\u4f7f\u7528\u5bb9\u91cf\uff0c\u503c\u5747\u4e3a0\uff5e1\u7684\u5c0f\u6570\uff0c\u8868\u793a\u4f7f\u7528\u78c1\u76d8\u7a7a\u95f4\u7684\u6bd4\u4f8b\u3002 \u4fee\u6539 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u5e76\u66f4\u65b0\u81f3\u6240\u6709\u670d\u52a1\u8282\u70b9\u3002 [jindofsx-storage]# Storage service \u914d\u7f6e storage.data-dirs=/mnt/disk1/jindofsx,/mnt/disk2/jindofsx # \u8868\u793a\u7528\u4ee5\u7f13\u5b58\u6570\u636e\u7684\u78c1\u76d8\u76ee\u5f55 storage.data-dirs.capacities=500G,500G # \u8868\u793a\u6bcf\u4e2a\u78c1\u76d8\u76ee\u5f55\u7684\u603b\u5bb9\u91cf storage.watermark.high.ratio=0.8 # \u8868\u793a\u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0a\u6c34\u4f4d\u6bd4\u4f8b\uff0c\u6bcf\u5757\u6570\u636e\u76d8\u7684 JindoFSx \u6570\u636e\u76ee\u5f55\u5360\u7528\u7684\u78c1\u76d8\u7a7a\u95f4\u5230\u8fbe\u4e0a\u6c34\u4f4d\u5373\u4f1a\u89e6\u53d1\u6e05\u7406\u3002\u9ed8\u8ba4\u503c\uff1a0.8\u3002 storage.watermark.low.ratio=0.6 # \u8868\u793a\u4f7f\u7528\u91cf\u7684\u4e0b\u6c34\u4f4d\u6bd4\u4f8b\uff0c\u89e6\u53d1\u6e05\u7406\u540e\u4f1a\u81ea\u52a8\u6e05\u7406\u51b7\u6570\u636e\uff0c\u5c06JindoFS\u6570\u636e\u76ee\u5f55\u5360\u7528\u7a7a\u95f4\u6e05\u7406\u5230\u4e0b\u6c34\u4f4d\u3002\u9ed8\u8ba4\u503c\uff1a0.6\u3002 \u8bf4\u660e: \u60a8\u53ef\u4ee5\u901a\u8fc7\u8bbe\u7f6e\u4e0a\u6c34\u4f4d\u6bd4\u4f8b\u8c03\u8282\u671f\u671b\u5206\u7ed9 JindoFSx \u7684\u78c1\u76d8\u7a7a\u95f4\uff0c\u4e0b\u6c34\u4f4d\u5fc5\u987b\u5c0f\u4e8e\u4e0a\u6c34\u4f4d\uff0c\u8bbe\u7f6e\u5408\u7406\u7684\u503c\u5373\u53ef\u3002 \u6ce8\u610f: \u914d\u7f6e\u5b8c\u6210\u540e\u9700\u91cd\u542f JindoFSx \u670d\u52a1\u3002","title":"\u78c1\u76d8\u7a7a\u95f4\u6c34\u4f4d\u63a7\u5236"},{"location":"backup/jindofsx/jindofsx_local_cache/","text":"\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u00a7 \u6570\u636e\u7f13\u5b58\u52a0\u901f\u4f5c\u4e3aJindoFSx\u7684\u6838\u5fc3\u529f\u80fd\u4e4b\u4e00\uff0c4.4.0\u7248\u672c\u5f15\u5165\u4e86\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\uff08Local cache\uff09\u7684\u7f13\u5b58\u4f7f\u7528\u6a21\u5f0f\uff0c\u63d0\u4f9b\u4e00\u79cd\u7eaf\u5ba2\u6237\u7aef\u7684\u7f13\u5b58\u52a0\u901f\u80fd\u529b\u3002 \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u7b80\u4ecb \u00a7 JindoFSx\u5ba2\u6237\u7aef\u63d0\u4f9b\u4e86\u4e24\u79cd\u6570\u636e\u7f13\u5b58\u4f7f\u7528\u6a21\u5f0f\uff0c\u5206\u5e03\u5f0f\u7f13\u5b58\uff08Dist cache\uff09\u548c\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\uff08Local cache\uff09\u3002\u5206\u5e03\u5f0f\u7f13\u5b58\u4f5c\u4e3aJindoFSx\u7684\u9ed8\u8ba4\u4f7f\u7528\u6a21\u5f0f\uff0c\u901a\u8fc7\u8bbf\u95eeJindoFSx\u5206\u5e03\u5f0f\u7f13\u5b58\u670d\u52a1\u5b9e\u73b0\u5bf9\u7f13\u5b58\u6570\u636e\u7684\u8bfb\u5199\uff1b 4.4.0\u989d\u5916\u5f15\u5165\u4e86\u672c\u5730\u7f13\u5b58\u7684\u4f7f\u7528\u6a21\u5f0f\uff0c\u5728\u5ba2\u6237\u7aef\u8fdb\u7a0b\u5185\u5d4c\u4e86\u7f13\u5b58\u7ba1\u7406\u6a21\u5757\uff0c\u4e0d\u9700\u8981\u4f9d\u8d56\u5206\u5e03\u5f0f\u670d\u52a1\uff0c\u76f4\u63a5\u5229\u7528\u672c\u5730\u7684\u78c1\u76d8\u6216\u8005\u5185\u5b58\u5bf9\u6570\u636e\u8fdb\u884c\u7f13\u5b58\u52a0\u901f\u3002 \u4f7f\u7528\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58 \u00a7 \u672c\u6587\u4ee5 JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u4e3a\u4f8b\u3002 Hadoop\u5ba2\u6237\u7aef\u4f7f\u7528\u672c\u5730\u7f13\u5b58 \u00a7 \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1 OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06 OSS \u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u914d\u7f6e\u672c\u5730\u7f13\u5b58\u52a8\u6001\u5e93\u8def\u5f84 \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u529f\u80fd\u901a\u8fc7\u63d2\u4ef6\u5f62\u5f0f\u63d0\u4f9b\u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728jindosdk-4.6.12/plugins/\u8def\u5f84\u4e0b\uff0c\u5c06\u8be5\u8def\u5f84\u7684\u7edd\u5bf9\u8def\u5f84\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.jdo.plugin.dir</name> <value>${JINDOSDK_HOME}/plugins</value> </property> </configuration> \u542f\u7528\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58 \u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\uff0c\u542f\u7528\u5e76\u914d\u7f6e\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u7684\u5fc5\u8981\u914d\u7f6e\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u603b\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>false</value> </property> <property> <!-- \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.local.enable</name> <value>false</value> </property> <property> <!-- \u672c\u5730\u7f13\u5b58\u78c1\u76d8\u8def\u5f84 --> <name>fs.jindofsx.data.cache.local.data-dirs</name> <value>/mnt/disk1/localcache,/mnt/disk2/localcache</value> </property> <property> <!-- \u672c\u5730\u7f13\u5b58\u78c1\u76d8\u5927\u5c0f --> <name>fs.jindofsx.data.cache.local.data-dirs.capacities</name> <value>50G,50G</value> </property> <property> <!-- \u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0a\u6c34\u4f4d\u6bd4\u4f8b --> <name>fs.jindofsx.data.cache.local.watermark.high.ratio</name> <value>0.8</value> </property> <property> <!-- \u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0b\u6c34\u4f4d\u6bd4\u4f8b --> <name>fs.jindofsx.data.cache.local.watermark.low.ratio</name> <value>0.4</value> </property> </configuration> Fuse\u5ba2\u6237\u7aef\u4f7f\u7528\u672c\u5730\u7f13\u5b58 \u00a7 JindoFuse \u8be6\u7ec6\u4f7f\u7528\u6587\u6863\u53c2\u8003 JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f OSS \u3002 \u5bf9\u4e8e\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u65e0\u9700\u90e8\u7f72\u542f\u52a8 JindoFSx \u7f13\u5b58\u52a0\u901f\u670d\u52a1\uff0c\u4ec5\u9700\u5ba2\u6237\u7aef\u914d\u7f6e\u672c\u5730\u7f13\u5b58\u76f8\u5173\u53c2\u6570\uff0c\u542f\u52a8fuse\u5373\u53ef\uff0c\u5177\u4f53\u5982\u4e0b\u3002 * \u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf \u914d\u7f6e\u6587\u4ef6 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u914d\u7f6e\u963f\u91cc\u4e91 OSS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou-internal.aliyuncs.com\u3002 fs.oss.endpoint= <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1\u7684 AccessKey ID \u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7 AccessKey \u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable = true # \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u529f\u80fd\u901a\u8fc7\u63d2\u4ef6\u5f62\u5f0f\u63d0\u4f9b\u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728jindosdk-4.6.12/plugins/\u8def\u5f84\u4e0b fs.jdo.plugin.dir=${JINDOSDK_HOME}/plugins # \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58 fs.jindofsx.data.cache.local.enable=true # \u672c\u5730\u7f13\u5b58\u78c1\u76d8\u8def\u5f84 fs.jindofsx.data.cache.local.data-dirs=/mnt/disk1/localcache/,/mnt/disk2/localcache/ # \u672c\u5730\u7f13\u5b58\u78c1\u76d8\u5927\u5c0f fs.jindofsx.data.cache.local.data-dirs.capacities=80G,80G # \u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0a\u6c34\u4f4d\u6bd4\u4f8b fs.jindofsx.data.cache.local.watermark.high.ratio=0.8 # \u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0b\u6c34\u4f4d\u6bd4\u4f8b fs.jindofsx.data.cache.local.watermark.low.ratio=0.4 \u6ce8\u610f\u4e8b\u9879 \u00a7 \u5f53\u524d\u7248\u672c\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u4e3a\u8fdb\u7a0b\u72ec\u5360\u78c1\u76d8\u8def\u5f84\uff0c\u540c\u4e00\u4e2a\u8def\u5f84\u53ea\u80fd\u7531\u4e00\u4e2a\u8fdb\u7a0b\u5360\u7528\uff0c\u4e14\u4e0d\u540c\u8fdb\u7a0b\u95f4\u4e0d\u5171\u4eab\u7f13\u5b58\u6570\u636e\uff1b \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u9002\u7528\u4e8e\u4e00\u4e9b\u5e38\u9a7b\u8fdb\u7a0b\u7684\u670d\u52a1\u4e2d\u4f7f\u7528\uff0c\u8fdb\u7a0b\u72ec\u7acb\u4eab\u6709\u7f13\u5b58\u3002","title":"\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofsx/jindofsx_local_cache/#_1","text":"\u6570\u636e\u7f13\u5b58\u52a0\u901f\u4f5c\u4e3aJindoFSx\u7684\u6838\u5fc3\u529f\u80fd\u4e4b\u4e00\uff0c4.4.0\u7248\u672c\u5f15\u5165\u4e86\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\uff08Local cache\uff09\u7684\u7f13\u5b58\u4f7f\u7528\u6a21\u5f0f\uff0c\u63d0\u4f9b\u4e00\u79cd\u7eaf\u5ba2\u6237\u7aef\u7684\u7f13\u5b58\u52a0\u901f\u80fd\u529b\u3002","title":"\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofsx/jindofsx_local_cache/#_2","text":"JindoFSx\u5ba2\u6237\u7aef\u63d0\u4f9b\u4e86\u4e24\u79cd\u6570\u636e\u7f13\u5b58\u4f7f\u7528\u6a21\u5f0f\uff0c\u5206\u5e03\u5f0f\u7f13\u5b58\uff08Dist cache\uff09\u548c\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\uff08Local cache\uff09\u3002\u5206\u5e03\u5f0f\u7f13\u5b58\u4f5c\u4e3aJindoFSx\u7684\u9ed8\u8ba4\u4f7f\u7528\u6a21\u5f0f\uff0c\u901a\u8fc7\u8bbf\u95eeJindoFSx\u5206\u5e03\u5f0f\u7f13\u5b58\u670d\u52a1\u5b9e\u73b0\u5bf9\u7f13\u5b58\u6570\u636e\u7684\u8bfb\u5199\uff1b 4.4.0\u989d\u5916\u5f15\u5165\u4e86\u672c\u5730\u7f13\u5b58\u7684\u4f7f\u7528\u6a21\u5f0f\uff0c\u5728\u5ba2\u6237\u7aef\u8fdb\u7a0b\u5185\u5d4c\u4e86\u7f13\u5b58\u7ba1\u7406\u6a21\u5757\uff0c\u4e0d\u9700\u8981\u4f9d\u8d56\u5206\u5e03\u5f0f\u670d\u52a1\uff0c\u76f4\u63a5\u5229\u7528\u672c\u5730\u7684\u78c1\u76d8\u6216\u8005\u5185\u5b58\u5bf9\u6570\u636e\u8fdb\u884c\u7f13\u5b58\u52a0\u901f\u3002","title":"\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u7b80\u4ecb"},{"location":"backup/jindofsx/jindofsx_local_cache/#_3","text":"\u672c\u6587\u4ee5 JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u4e3a\u4f8b\u3002","title":"\u4f7f\u7528\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58"},{"location":"backup/jindofsx/jindofsx_local_cache/#hadoop","text":"\u914d\u7f6e OSS \u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1 OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06 OSS \u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u914d\u7f6e\u672c\u5730\u7f13\u5b58\u52a8\u6001\u5e93\u8def\u5f84 \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u529f\u80fd\u901a\u8fc7\u63d2\u4ef6\u5f62\u5f0f\u63d0\u4f9b\u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728jindosdk-4.6.12/plugins/\u8def\u5f84\u4e0b\uff0c\u5c06\u8be5\u8def\u5f84\u7684\u7edd\u5bf9\u8def\u5f84\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.jdo.plugin.dir</name> <value>${JINDOSDK_HOME}/plugins</value> </property> </configuration> \u542f\u7528\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58 \u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\uff0c\u542f\u7528\u5e76\u914d\u7f6e\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u7684\u5fc5\u8981\u914d\u7f6e\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u603b\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>false</value> </property> <property> <!-- \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.local.enable</name> <value>false</value> </property> <property> <!-- \u672c\u5730\u7f13\u5b58\u78c1\u76d8\u8def\u5f84 --> <name>fs.jindofsx.data.cache.local.data-dirs</name> <value>/mnt/disk1/localcache,/mnt/disk2/localcache</value> </property> <property> <!-- \u672c\u5730\u7f13\u5b58\u78c1\u76d8\u5927\u5c0f --> <name>fs.jindofsx.data.cache.local.data-dirs.capacities</name> <value>50G,50G</value> </property> <property> <!-- \u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0a\u6c34\u4f4d\u6bd4\u4f8b --> <name>fs.jindofsx.data.cache.local.watermark.high.ratio</name> <value>0.8</value> </property> <property> <!-- \u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0b\u6c34\u4f4d\u6bd4\u4f8b --> <name>fs.jindofsx.data.cache.local.watermark.low.ratio</name> <value>0.4</value> </property> </configuration>","title":"Hadoop\u5ba2\u6237\u7aef\u4f7f\u7528\u672c\u5730\u7f13\u5b58"},{"location":"backup/jindofsx/jindofsx_local_cache/#fuse","text":"JindoFuse \u8be6\u7ec6\u4f7f\u7528\u6587\u6863\u53c2\u8003 JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f OSS \u3002 \u5bf9\u4e8e\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u65e0\u9700\u90e8\u7f72\u542f\u52a8 JindoFSx \u7f13\u5b58\u52a0\u901f\u670d\u52a1\uff0c\u4ec5\u9700\u5ba2\u6237\u7aef\u914d\u7f6e\u672c\u5730\u7f13\u5b58\u76f8\u5173\u53c2\u6570\uff0c\u542f\u52a8fuse\u5373\u53ef\uff0c\u5177\u4f53\u5982\u4e0b\u3002 * \u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf \u914d\u7f6e\u6587\u4ef6 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u914d\u7f6e\u963f\u91cc\u4e91 OSS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou-internal.aliyuncs.com\u3002 fs.oss.endpoint= <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1\u7684 AccessKey ID \u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7 AccessKey \u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable = true # \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u529f\u80fd\u901a\u8fc7\u63d2\u4ef6\u5f62\u5f0f\u63d0\u4f9b\u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728jindosdk-4.6.12/plugins/\u8def\u5f84\u4e0b fs.jdo.plugin.dir=${JINDOSDK_HOME}/plugins # \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58 fs.jindofsx.data.cache.local.enable=true # \u672c\u5730\u7f13\u5b58\u78c1\u76d8\u8def\u5f84 fs.jindofsx.data.cache.local.data-dirs=/mnt/disk1/localcache/,/mnt/disk2/localcache/ # \u672c\u5730\u7f13\u5b58\u78c1\u76d8\u5927\u5c0f fs.jindofsx.data.cache.local.data-dirs.capacities=80G,80G # \u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0a\u6c34\u4f4d\u6bd4\u4f8b fs.jindofsx.data.cache.local.watermark.high.ratio=0.8 # \u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0b\u6c34\u4f4d\u6bd4\u4f8b fs.jindofsx.data.cache.local.watermark.low.ratio=0.4","title":"Fuse\u5ba2\u6237\u7aef\u4f7f\u7528\u672c\u5730\u7f13\u5b58"},{"location":"backup/jindofsx/jindofsx_local_cache/#_4","text":"\u5f53\u524d\u7248\u672c\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u4e3a\u8fdb\u7a0b\u72ec\u5360\u78c1\u76d8\u8def\u5f84\uff0c\u540c\u4e00\u4e2a\u8def\u5f84\u53ea\u80fd\u7531\u4e00\u4e2a\u8fdb\u7a0b\u5360\u7528\uff0c\u4e14\u4e0d\u540c\u8fdb\u7a0b\u95f4\u4e0d\u5171\u4eab\u7f13\u5b58\u6570\u636e\uff1b \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u9002\u7528\u4e8e\u4e00\u4e9b\u5e38\u9a7b\u8fdb\u7a0b\u7684\u670d\u52a1\u4e2d\u4f7f\u7528\uff0c\u8fdb\u7a0b\u72ec\u7acb\u4eab\u6709\u7f13\u5b58\u3002","title":"\u6ce8\u610f\u4e8b\u9879"},{"location":"backup/jindofsx/jindofsx_metrics/","text":"\u57fa\u4e8e Prometheus + Grafana \u642d\u5efa JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u53ef\u89c6\u5316\u6307\u6807\u89c2\u6d4b\u5e73\u53f0 \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 JindoFSx >= 4.1.0 Prometheus >= 2.10 Grafana >= 7.3.7 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u3002 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK\u3002 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u4ee5\u7b80\u5355\u65b9\u5f0f\u67e5\u770b metrics \u6307\u6807 \u00a7 \u5728\u5df2\u7ecf\u6210\u529f\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7684\u524d\u63d0\u4e0b\uff0c\u627e\u5230\u670d\u52a1 namesapce \u6240\u5728\u8282\u70b9\u7684 ip(localhost) \u548c\u670d\u52a1\u542f\u52a8\u7aef\u53e3 port(8101)\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 curl/wget \u7b49\u547d\u4ee4\u6765\u83b7\u53d6\u5bf9\u5e94\u7684 metrics \u8f93\u51fa\u7ed3\u679c\u3002\u5982 curl localhost:8101/jindo_metrics/prometheus \u6307\u6807\u8be6\u7ec6\u540d\u79f0\u548c\u542b\u4e49\u8bf7\u53c2\u8003\u6587\u7ae0\u672b\u5c3e 1\u3001Prometheus \u5b89\u88c5 \u00a7 \u9996\u5148\u9700\u8981\u5728\u73af\u5883\u91cc\u5b89\u88c5 Prometheus \u548c Grafana \u670d\u52a1\uff0c\u4e0b\u9762\u4ee5 centos7 \u73af\u5883\u4e3a\u4f8b * Prometheus \u76f8\u5173\u4ecb\u7ecd * Grafana \u76f8\u5173\u4ecb\u7ecd 1\u3001\u5b89\u88c5 Prometheus \u670d\u52a1 #\u4e0b\u8f7d\u5b89\u88c5\u5305 wget https://github.com/prometheus/prometheus/releases/download/v2.30.1/prometheus-2.30.1.linux-amd64.tar.gz #\u89e3\u538b\u5b89\u88c5\u5305 tar -zxvf prometheus-2.30.1.linux-amd64.tar.gz #\u91cd\u547d\u540d mv prometheus-2.30.1.linux-amd64 prometheus-server 2\u3001\u4fee\u6539\u89e3\u538b\u5b89\u88c5\u5305\u5185\u7684 prometheus.yml \u6587\u4ef6\uff0c\u66ff\u6362\u4e3a global: scrape_interval: 15s # By default, scrape targets every 15 seconds. # Attach these labels to any time series or alerts when communicating with # external systems (federation, remote storage, Alertmanager). external_labels: monitor: 'codelab-monitor' # A scrape configuration containing exactly one endpoint to scrape: # Here it's Prometheus itself. scrape_configs: # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config. - job_name: 'namespace-prometheus' scrape_interval: 5s metrics_path: /jindo_metrics/prometheus static_configs: - targets: ['<ns Ip>:<ns Port>'] # your ns Ip and ns port eg:localhost:8101 \u7b80\u5355\u4f7f\u7528\u60a8\u53ea\u9700\u586b\u5199 targets: ['<ns Ip>:<ns Port>'] \u5373\u53ef\uff1a \u5982 targets: ['localhost:8101'] \uff0c\u4fdd\u8bc1\u7f51\u7edc\u548c\u7aef\u53e3\u53ef\u8bbf\u95ee 3\u3001\u542f\u52a8 prometheus \u670d\u52a1 #\u5b89\u88c5\u76ee\u5f55\u4e0b cd prometheus-server #\u540e\u53f0\u542f\u52a8server nohup ./prometheus > myout.file 2>&1 & prometheus \u9ed8\u8ba4\u542f\u52a8\u57289090\u7aef\u53e3 $ netstat -ntlp | grep prometheus tcp 0 0 0.0.0.0:9090 0.0.0.0:* LISTEN 17448/./prometheus \u4ece\u6d4f\u89c8\u5668\u6253\u5f00 prometheus UI\u754c\u9762 \u5230\u6b64\uff0cprometheus \u670d\u52a1\u5b89\u88c5\u6210\u529f\uff0c\u63a5\u4e0b\u6765\u6211\u4eec\u901a\u8fc7 Grafana \u670d\u52a1\u6765\u5bf9\u63a5 Prometheus \u6570\u636e\u6e90\u8fdb\u884c metrics \u7684\u5c55\u793a\u5de5\u4f5c 2\u3001Grafana \u5b89\u88c5 \u00a7 1\u3001\u4e0b\u8f7d\u5b89\u88c5 Grafana \u670d\u52a1 # \u4e0b\u8f7d grafana 7.3.7 wget https://dl.grafana.com/enterprise/release/grafana-enterprise-7.3.7.linux-amd64.tar.gz # \u89e3\u538b tar -zxvf grafana-enterprise-7.3.7.linux-amd64.tar.gz 2\u3001\u542f\u52a8 grafana \u670d\u52a1 \u6700\u5c0f\u5316\u542f\u52a8 grafana \u670d\u52a1\uff0c\u5176\u4ed6\u9009\u9879\u53ef\u53c2\u8003 grafana \u5b98\u65b9\u6587\u6863 cd grafana-7.3.7/bin # ls -l \u603b\u7528\u91cf 85416 -rwxr-xr-x 1 root root 25691632 1\u6708 14 2021 grafana-cli -rw-r--r-- 1 root root 33 1\u6708 14 2021 grafana-cli.md5 -rwxr-xr-x 1 root root 61760192 1\u6708 14 2021 grafana-server -rw-r--r-- 1 root root 33 1\u6708 14 2021 grafana-server.md5 \u540e\u53f0\u542f\u52a8 grafana \u670d\u52a1 nohup ./grafana-server > myout.file 2>&1 & \u6253\u5f00 grafana UI \u754c\u9762\uff0c\u9ed8\u8ba4 3000 \u7aef\u53e3 $ netstat -ntlp | grep grafana tcp 0 0 0.0.0.0:3000 0.0.0.0:* LISTEN 27223/./grafan-ser \u5982\u6253\u5f00 localhost:3000 \uff0c\u521d\u59cb\u5316\u7528\u6237\u540d/\u5bc6\u7801\uff1aadmin/admin\uff0c\u70b9\u51fb login \u767b\u9646 \u6dfb\u52a0 prometheus \u6570\u636e\u6e90 \u586b\u5199 Prometheus \u7684 HTTP URL\uff0c\u8fdb\u884c Save&Test\uff0c\u6d4b\u8bd5\u901a\u8fc7\u5373\u4e3a\u6dfb\u52a0\u6210\u529f \u6dfb\u52a0 JindoFS cache \u6a21\u5f0f\u7684 JSON \u5c55\u793a\u6a21\u7248\uff0c \u70b9\u51fb\u8fd9\u91cc\u4e0b\u8f7d JSON \u6587\u4ef6 \uff0c\u4e0b\u8f7d\u5230\u672c\u5730\u540e\uff0c\u70b9\u51fb Upload JSON file \u70b9\u51fb import \u540e\u5373\u53ef\u770b\u5230 cache \u6a21\u5f0f\u7684 UI Prometheus \u6307\u6807\u9884\u89c8(\u66f4\u65b0\u4e2d) \u00a7 \u4e0b\u9762\u5bf9 JindoRuntime \u5f53\u524d\u6536\u96c6\u7684\u6307\u6807\u8fdb\u884c\u9884\u89c8\u548c\u63cf\u8ff0\uff0c\u60a8\u53ef\u4ee5\u57fa\u4e8e\u5df2\u6536\u96c6\u6307\u6807\u5728 Grafana \u4e2d\u8fdb\u884c\u7ec4\u5408\u548c\u56fe\u8868\u5c55\u793a\uff0c\u5982\u60a8\u9700\u8981\u989d\u5916\u7684\u57fa\u672c metrics \u5c55\u793a\uff0c\u8bf7\u5f00 issue \u8054\u7cfb\u6211\u4eec # HELP jindofsx_ns_backend_read_bytes_time_total_window # TYPE jindofsx_ns_backend_read_bytes_time_total_window gauge jindofsx_ns_backend_read_bytes_time_total_window 20367553 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_read_bytes_total # TYPE jindofsx_ns_backend_read_bytes_total gauge jindofsx_ns_backend_read_bytes_total 166703347021 #\u603b\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_bytes_total_window # TYPE jindofsx_ns_backend_read_bytes_total_window gauge jindofsx_ns_backend_read_bytes_total_window 1174671374 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_time_total # TYPE jindofsx_ns_backend_read_time_total gauge jindofsx_ns_backend_read_time_total 8717632094 #\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_readop_num_total # TYPE jindofsx_ns_backend_readop_num_total gauge jindofsx_ns_backend_readop_num_total 181951 #\u5e95\u5c42\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFS \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_local_read_bytes_time_total_window # TYPE jindofsx_ns_local_read_bytes_time_total_window gauge jindofsx_ns_local_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_bytes_total # TYPE jindofsx_ns_local_read_bytes_total gauge jindofsx_ns_local_read_bytes_total 0 #\u603b\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_local_read_bytes_total_window # TYPE jindofsx_ns_local_read_bytes_total_window gauge jindofsx_ns_local_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_time_total # TYPE jindofsx_ns_local_read_time_total gauge jindofsx_ns_local_read_time_total 0 #\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_readop_num_total # TYPE jindofsx_ns_local_readop_num_total gauge jindofsx_ns_local_readop_num_total 0 #\u77ed\u8def\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFS \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_remote_read_bytes_time_total_window # TYPE jindofsx_ns_remote_read_bytes_time_total_window gauge jindofsx_ns_remote_read_bytes_time_total_window 73714663 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_read_bytes_total # TYPE jindofsx_ns_remote_read_bytes_total gauge jindofsx_ns_remote_read_bytes_total 112108045498 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_bytes_total_window # TYPE jindofsx_ns_remote_read_bytes_total_window gauge jindofsx_ns_remote_read_bytes_total_window 13380884043 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_time_total # TYPE jindofsx_ns_remote_read_time_total gauge jindofsx_ns_remote_read_time_total 765272359 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u65f6\u95f4\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_readop_num_total # TYPE jindofsx_ns_remote_readop_num_total gauge jindofsx_ns_remote_readop_num_total 107314 #\u8fdc\u7aef\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFS \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_slicelet_read_bytes_time_total_window # TYPE jindofsx_ns_slicelet_read_bytes_time_total_window gauge jindofsx_ns_slicelet_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_read_bytes_total # TYPE jindofsx_ns_slicelet_read_bytes_total gauge jindofsx_ns_slicelet_read_bytes_total 0 #\u603b\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_bytes_total_window # TYPE jindofsx_ns_slicelet_read_bytes_total_window gauge jindofsx_ns_slicelet_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_time_total # TYPE jindofsx_ns_slicelet_read_time_total gauge jindofsx_ns_slicelet_read_time_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_readop_num_total # TYPE jindofsx_ns_slicelet_readop_num_total gauge jindofsx_ns_slicelet_readop_num_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFS \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_total_disk_cap # TYPE jindofsx_ns_total_disk_cap gauge jindofsx_ns_total_disk_cap 840739848192 #\u78c1\u76d8\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_mem_cap # TYPE jindofsx_ns_total_mem_cap gauge jindofsx_ns_total_mem_cap 0 #\u5185\u5b58\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_stsnodes_num # TYPE jindofsx_ns_total_stsnodes_num gauge jindofsx_ns_total_stsnodes_num 3 #alive worker \u8282\u70b9\u7684\u6570\u91cf # HELP jindofsx_ns_total_used_disk_cap # TYPE jindofsx_ns_total_used_disk_cap gauge jindofsx_ns_total_used_disk_cap 153423446016 #\u5df2\u7528\u78c1\u76d8\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_used_mem_cap # TYPE jindofsx_ns_total_used_mem_cap gauge jindofsx_ns_total_used_mem_cap 0 #\u5df2\u7528\u5185\u5b58\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte","title":"Jindofsx metrics"},{"location":"backup/jindofsx/jindofsx_metrics/#prometheus-grafana-jindofsx","text":"","title":"\u57fa\u4e8e Prometheus + Grafana \u642d\u5efa JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u53ef\u89c6\u5316\u6307\u6807\u89c2\u6d4b\u5e73\u53f0"},{"location":"backup/jindofsx/jindofsx_metrics/#_1","text":"JindoFSx >= 4.1.0 Prometheus >= 2.10 Grafana >= 7.3.7","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/jindofsx/jindofsx_metrics/#_2","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u3002 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK\u3002 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindofsx_metrics/#metrics","text":"\u5728\u5df2\u7ecf\u6210\u529f\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7684\u524d\u63d0\u4e0b\uff0c\u627e\u5230\u670d\u52a1 namesapce \u6240\u5728\u8282\u70b9\u7684 ip(localhost) \u548c\u670d\u52a1\u542f\u52a8\u7aef\u53e3 port(8101)\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 curl/wget \u7b49\u547d\u4ee4\u6765\u83b7\u53d6\u5bf9\u5e94\u7684 metrics \u8f93\u51fa\u7ed3\u679c\u3002\u5982 curl localhost:8101/jindo_metrics/prometheus \u6307\u6807\u8be6\u7ec6\u540d\u79f0\u548c\u542b\u4e49\u8bf7\u53c2\u8003\u6587\u7ae0\u672b\u5c3e","title":"\u4ee5\u7b80\u5355\u65b9\u5f0f\u67e5\u770b metrics \u6307\u6807"},{"location":"backup/jindofsx/jindofsx_metrics/#1prometheus","text":"\u9996\u5148\u9700\u8981\u5728\u73af\u5883\u91cc\u5b89\u88c5 Prometheus \u548c Grafana \u670d\u52a1\uff0c\u4e0b\u9762\u4ee5 centos7 \u73af\u5883\u4e3a\u4f8b * Prometheus \u76f8\u5173\u4ecb\u7ecd * Grafana \u76f8\u5173\u4ecb\u7ecd 1\u3001\u5b89\u88c5 Prometheus \u670d\u52a1 #\u4e0b\u8f7d\u5b89\u88c5\u5305 wget https://github.com/prometheus/prometheus/releases/download/v2.30.1/prometheus-2.30.1.linux-amd64.tar.gz #\u89e3\u538b\u5b89\u88c5\u5305 tar -zxvf prometheus-2.30.1.linux-amd64.tar.gz #\u91cd\u547d\u540d mv prometheus-2.30.1.linux-amd64 prometheus-server 2\u3001\u4fee\u6539\u89e3\u538b\u5b89\u88c5\u5305\u5185\u7684 prometheus.yml \u6587\u4ef6\uff0c\u66ff\u6362\u4e3a global: scrape_interval: 15s # By default, scrape targets every 15 seconds. # Attach these labels to any time series or alerts when communicating with # external systems (federation, remote storage, Alertmanager). external_labels: monitor: 'codelab-monitor' # A scrape configuration containing exactly one endpoint to scrape: # Here it's Prometheus itself. scrape_configs: # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config. - job_name: 'namespace-prometheus' scrape_interval: 5s metrics_path: /jindo_metrics/prometheus static_configs: - targets: ['<ns Ip>:<ns Port>'] # your ns Ip and ns port eg:localhost:8101 \u7b80\u5355\u4f7f\u7528\u60a8\u53ea\u9700\u586b\u5199 targets: ['<ns Ip>:<ns Port>'] \u5373\u53ef\uff1a \u5982 targets: ['localhost:8101'] \uff0c\u4fdd\u8bc1\u7f51\u7edc\u548c\u7aef\u53e3\u53ef\u8bbf\u95ee 3\u3001\u542f\u52a8 prometheus \u670d\u52a1 #\u5b89\u88c5\u76ee\u5f55\u4e0b cd prometheus-server #\u540e\u53f0\u542f\u52a8server nohup ./prometheus > myout.file 2>&1 & prometheus \u9ed8\u8ba4\u542f\u52a8\u57289090\u7aef\u53e3 $ netstat -ntlp | grep prometheus tcp 0 0 0.0.0.0:9090 0.0.0.0:* LISTEN 17448/./prometheus \u4ece\u6d4f\u89c8\u5668\u6253\u5f00 prometheus UI\u754c\u9762 \u5230\u6b64\uff0cprometheus \u670d\u52a1\u5b89\u88c5\u6210\u529f\uff0c\u63a5\u4e0b\u6765\u6211\u4eec\u901a\u8fc7 Grafana \u670d\u52a1\u6765\u5bf9\u63a5 Prometheus \u6570\u636e\u6e90\u8fdb\u884c metrics \u7684\u5c55\u793a\u5de5\u4f5c","title":"1\u3001Prometheus \u5b89\u88c5"},{"location":"backup/jindofsx/jindofsx_metrics/#2grafana","text":"1\u3001\u4e0b\u8f7d\u5b89\u88c5 Grafana \u670d\u52a1 # \u4e0b\u8f7d grafana 7.3.7 wget https://dl.grafana.com/enterprise/release/grafana-enterprise-7.3.7.linux-amd64.tar.gz # \u89e3\u538b tar -zxvf grafana-enterprise-7.3.7.linux-amd64.tar.gz 2\u3001\u542f\u52a8 grafana \u670d\u52a1 \u6700\u5c0f\u5316\u542f\u52a8 grafana \u670d\u52a1\uff0c\u5176\u4ed6\u9009\u9879\u53ef\u53c2\u8003 grafana \u5b98\u65b9\u6587\u6863 cd grafana-7.3.7/bin # ls -l \u603b\u7528\u91cf 85416 -rwxr-xr-x 1 root root 25691632 1\u6708 14 2021 grafana-cli -rw-r--r-- 1 root root 33 1\u6708 14 2021 grafana-cli.md5 -rwxr-xr-x 1 root root 61760192 1\u6708 14 2021 grafana-server -rw-r--r-- 1 root root 33 1\u6708 14 2021 grafana-server.md5 \u540e\u53f0\u542f\u52a8 grafana \u670d\u52a1 nohup ./grafana-server > myout.file 2>&1 & \u6253\u5f00 grafana UI \u754c\u9762\uff0c\u9ed8\u8ba4 3000 \u7aef\u53e3 $ netstat -ntlp | grep grafana tcp 0 0 0.0.0.0:3000 0.0.0.0:* LISTEN 27223/./grafan-ser \u5982\u6253\u5f00 localhost:3000 \uff0c\u521d\u59cb\u5316\u7528\u6237\u540d/\u5bc6\u7801\uff1aadmin/admin\uff0c\u70b9\u51fb login \u767b\u9646 \u6dfb\u52a0 prometheus \u6570\u636e\u6e90 \u586b\u5199 Prometheus \u7684 HTTP URL\uff0c\u8fdb\u884c Save&Test\uff0c\u6d4b\u8bd5\u901a\u8fc7\u5373\u4e3a\u6dfb\u52a0\u6210\u529f \u6dfb\u52a0 JindoFS cache \u6a21\u5f0f\u7684 JSON \u5c55\u793a\u6a21\u7248\uff0c \u70b9\u51fb\u8fd9\u91cc\u4e0b\u8f7d JSON \u6587\u4ef6 \uff0c\u4e0b\u8f7d\u5230\u672c\u5730\u540e\uff0c\u70b9\u51fb Upload JSON file \u70b9\u51fb import \u540e\u5373\u53ef\u770b\u5230 cache \u6a21\u5f0f\u7684 UI","title":"2\u3001Grafana \u5b89\u88c5"},{"location":"backup/jindofsx/jindofsx_metrics/#prometheus","text":"\u4e0b\u9762\u5bf9 JindoRuntime \u5f53\u524d\u6536\u96c6\u7684\u6307\u6807\u8fdb\u884c\u9884\u89c8\u548c\u63cf\u8ff0\uff0c\u60a8\u53ef\u4ee5\u57fa\u4e8e\u5df2\u6536\u96c6\u6307\u6807\u5728 Grafana \u4e2d\u8fdb\u884c\u7ec4\u5408\u548c\u56fe\u8868\u5c55\u793a\uff0c\u5982\u60a8\u9700\u8981\u989d\u5916\u7684\u57fa\u672c metrics \u5c55\u793a\uff0c\u8bf7\u5f00 issue \u8054\u7cfb\u6211\u4eec # HELP jindofsx_ns_backend_read_bytes_time_total_window # TYPE jindofsx_ns_backend_read_bytes_time_total_window gauge jindofsx_ns_backend_read_bytes_time_total_window 20367553 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_read_bytes_total # TYPE jindofsx_ns_backend_read_bytes_total gauge jindofsx_ns_backend_read_bytes_total 166703347021 #\u603b\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_bytes_total_window # TYPE jindofsx_ns_backend_read_bytes_total_window gauge jindofsx_ns_backend_read_bytes_total_window 1174671374 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_time_total # TYPE jindofsx_ns_backend_read_time_total gauge jindofsx_ns_backend_read_time_total 8717632094 #\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_readop_num_total # TYPE jindofsx_ns_backend_readop_num_total gauge jindofsx_ns_backend_readop_num_total 181951 #\u5e95\u5c42\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFS \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_local_read_bytes_time_total_window # TYPE jindofsx_ns_local_read_bytes_time_total_window gauge jindofsx_ns_local_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_bytes_total # TYPE jindofsx_ns_local_read_bytes_total gauge jindofsx_ns_local_read_bytes_total 0 #\u603b\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_local_read_bytes_total_window # TYPE jindofsx_ns_local_read_bytes_total_window gauge jindofsx_ns_local_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_time_total # TYPE jindofsx_ns_local_read_time_total gauge jindofsx_ns_local_read_time_total 0 #\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_readop_num_total # TYPE jindofsx_ns_local_readop_num_total gauge jindofsx_ns_local_readop_num_total 0 #\u77ed\u8def\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFS \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_remote_read_bytes_time_total_window # TYPE jindofsx_ns_remote_read_bytes_time_total_window gauge jindofsx_ns_remote_read_bytes_time_total_window 73714663 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_read_bytes_total # TYPE jindofsx_ns_remote_read_bytes_total gauge jindofsx_ns_remote_read_bytes_total 112108045498 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_bytes_total_window # TYPE jindofsx_ns_remote_read_bytes_total_window gauge jindofsx_ns_remote_read_bytes_total_window 13380884043 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_time_total # TYPE jindofsx_ns_remote_read_time_total gauge jindofsx_ns_remote_read_time_total 765272359 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u65f6\u95f4\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_readop_num_total # TYPE jindofsx_ns_remote_readop_num_total gauge jindofsx_ns_remote_readop_num_total 107314 #\u8fdc\u7aef\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFS \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_slicelet_read_bytes_time_total_window # TYPE jindofsx_ns_slicelet_read_bytes_time_total_window gauge jindofsx_ns_slicelet_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_read_bytes_total # TYPE jindofsx_ns_slicelet_read_bytes_total gauge jindofsx_ns_slicelet_read_bytes_total 0 #\u603b\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_bytes_total_window # TYPE jindofsx_ns_slicelet_read_bytes_total_window gauge jindofsx_ns_slicelet_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_time_total # TYPE jindofsx_ns_slicelet_read_time_total gauge jindofsx_ns_slicelet_read_time_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_readop_num_total # TYPE jindofsx_ns_slicelet_readop_num_total gauge jindofsx_ns_slicelet_readop_num_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFS \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_total_disk_cap # TYPE jindofsx_ns_total_disk_cap gauge jindofsx_ns_total_disk_cap 840739848192 #\u78c1\u76d8\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_mem_cap # TYPE jindofsx_ns_total_mem_cap gauge jindofsx_ns_total_mem_cap 0 #\u5185\u5b58\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_stsnodes_num # TYPE jindofsx_ns_total_stsnodes_num gauge jindofsx_ns_total_stsnodes_num 3 #alive worker \u8282\u70b9\u7684\u6570\u91cf # HELP jindofsx_ns_total_used_disk_cap # TYPE jindofsx_ns_total_used_disk_cap gauge jindofsx_ns_total_used_disk_cap 153423446016 #\u5df2\u7528\u78c1\u76d8\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_used_mem_cap # TYPE jindofsx_ns_total_used_mem_cap gauge jindofsx_ns_total_used_mem_cap 0 #\u5df2\u7528\u5185\u5b58\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte","title":"Prometheus \u6307\u6807\u9884\u89c8(\u66f4\u65b0\u4e2d)"},{"location":"backup/jindofsx/jindofsx_multi_mount/","text":"\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u6df7\u5408\u6302\u8f7d \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u4e86\u5bf9\u591a\u79cd\u6570\u636e\u6e90\u7684\u7f13\u5b58\u52a0\u901f\u529f\u80fd\uff0c\u800c\u4e14\u53ef\u4ee5\u5c06\u4e0d\u540c\u6570\u636e\u6e90\u7edf\u4e00\u7ba1\u7406\uff0c\u653e\u5230\u540c\u4e00\u4e2a\u547d\u540d\u7a7a\u95f4\u4e0b\u7edf\u4e00\u8bbf\u95ee\u3002 \u524d\u63d0 \u00a7 1\u3001\u5df2\u7ecf\u90e8\u7f72 jindofsx \u670d\u52a1\uff0c\u4e14\u5728\u670d\u52a1\u7aef\u914d\u7f6e\u597d\u5404\u79cd\u6570\u636e\u6e90\u7684\u8bbf\u95ee\u5bc6\u94a5\u3002 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u5feb\u901f\u5165\u95e8 2\u3001\u5728 core-site \u91cc\u6dfb\u52a0 jindo \u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u53c2\u6570 <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> \u6302\u8f7d OSS \u6570\u636e\u6e90 \u00a7 1\u3001\u5df2\u7ecf\u5c06 OSS \u76f8\u5173 ak/endpoint \u4fe1\u606f\u914d\u7f6e\u5230 core-site.xml \u91cc \u53ef\u53c2\u8003 JindoFSx \u6302\u8f7d OSS \u6570\u636e\u6e90 2\u3001\u6267\u884c\u6302\u8f7d\u547d\u4ee4 jindo admin -mount /oss oss://<bucket>/<dir> /oss\uff1a\u6302\u8f7d\u5230\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u4e0b\u7684\u8bbf\u95ee\u8def\u5f84 \u540c\u6837\u53ef\u4ee5\u628a\u591a\u4e2a OSS bucket \u6302\u8f7d\u5230\u4e0d\u540c\u7684\u8def\u5f84\u4e0b\u9762\u3002 \u6302\u8f7d OSS-HDFS \u670d\u52a1\u6570\u636e\u6e90 \u00a7 1\u3001\u5df2\u7ecf\u5c06 OSS-HDFS \u8bbf\u95ee\u4fe1\u606f\u914d\u7f6e\u5230 core-site.xml \u91cc \u53ef\u53c2\u8003 JindoFSx \u6302\u8f7d OSS-HDFS \u6570\u636e\u6e90 2\u3001\u6267\u884c\u6302\u8f7d\u547d\u4ee4 jindo admin -mount /oss-hdfs oss://<bucket>/<dir> /oss-hdfs\uff1a\u6302\u8f7d\u5230\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u4e0b\u7684\u8bbf\u95ee\u8def\u5f84 \u540c\u6837\u53ef\u4ee5\u628a\u591a\u4e2a OSS-HDFS bucket \u6302\u8f7d\u5230\u4e0d\u540c\u7684\u8def\u5f84\u4e0b\u9762\u3002 \u4f7f\u7528\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u8bbf\u95ee \u00a7 $ hadoop fs -ls jindo:/// drwxrwxr-x - root root 0 1970-01-01 00:00 jindo:///oss drwxrwxr-x - root root 0 1970-01-01 00:00 jindo:///hdfs $ hadoop fs -ls jindo:///oss/ drwxrwxr-x - root root 0 1970-01-01 00:00 jindo:///oss/test_a_dir -rwxrwxr-x 0 root root 19564111 2022-04-11 10:25 jindo:///oss/tmp.txt $ hadoop fs -ls jindo:///hdfs/ drwxrwxr-x - root root 0 1970-01-01 00:00 jindo:///hdfs/test_a_dir -rwxrwxr-x 0 root root 19564111 2022-04-11 10:25 jindo:///hdfs/tmp.txt \u6211\u4eec\u8fd8\u652f\u6301\u5176\u4ed6\u5176\u4ed6\u6570\u636e\u6e90\u7684\u6302\u8f7d\uff0c\u6bd4\u5982 S3\u3001HDFS\u3001NAS\u7b49\uff0c\u53ef\u53c2\u8003\u4ee5\u4e0b\u6587\u6863 JindoFSx \u6302\u8f7d S3 \u6570\u636e\u6e90 JindoFSx \u6302\u8f7d HDFS \u6570\u636e\u6e90 JindoFSx \u6302\u8f7d NAS \u6570\u636e\u6e90","title":"\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u6df7\u5408\u6302\u8f7d"},{"location":"backup/jindofsx/jindofsx_multi_mount/#_1","text":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u4e86\u5bf9\u591a\u79cd\u6570\u636e\u6e90\u7684\u7f13\u5b58\u52a0\u901f\u529f\u80fd\uff0c\u800c\u4e14\u53ef\u4ee5\u5c06\u4e0d\u540c\u6570\u636e\u6e90\u7edf\u4e00\u7ba1\u7406\uff0c\u653e\u5230\u540c\u4e00\u4e2a\u547d\u540d\u7a7a\u95f4\u4e0b\u7edf\u4e00\u8bbf\u95ee\u3002","title":"\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u6df7\u5408\u6302\u8f7d"},{"location":"backup/jindofsx/jindofsx_multi_mount/#_2","text":"1\u3001\u5df2\u7ecf\u90e8\u7f72 jindofsx \u670d\u52a1\uff0c\u4e14\u5728\u670d\u52a1\u7aef\u914d\u7f6e\u597d\u5404\u79cd\u6570\u636e\u6e90\u7684\u8bbf\u95ee\u5bc6\u94a5\u3002 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u5feb\u901f\u5165\u95e8 2\u3001\u5728 core-site \u91cc\u6dfb\u52a0 jindo \u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u53c2\u6570 <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property>","title":"\u524d\u63d0"},{"location":"backup/jindofsx/jindofsx_multi_mount/#oss","text":"1\u3001\u5df2\u7ecf\u5c06 OSS \u76f8\u5173 ak/endpoint \u4fe1\u606f\u914d\u7f6e\u5230 core-site.xml \u91cc \u53ef\u53c2\u8003 JindoFSx \u6302\u8f7d OSS \u6570\u636e\u6e90 2\u3001\u6267\u884c\u6302\u8f7d\u547d\u4ee4 jindo admin -mount /oss oss://<bucket>/<dir> /oss\uff1a\u6302\u8f7d\u5230\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u4e0b\u7684\u8bbf\u95ee\u8def\u5f84 \u540c\u6837\u53ef\u4ee5\u628a\u591a\u4e2a OSS bucket \u6302\u8f7d\u5230\u4e0d\u540c\u7684\u8def\u5f84\u4e0b\u9762\u3002","title":"\u6302\u8f7d OSS \u6570\u636e\u6e90"},{"location":"backup/jindofsx/jindofsx_multi_mount/#oss-hdfs","text":"1\u3001\u5df2\u7ecf\u5c06 OSS-HDFS \u8bbf\u95ee\u4fe1\u606f\u914d\u7f6e\u5230 core-site.xml \u91cc \u53ef\u53c2\u8003 JindoFSx \u6302\u8f7d OSS-HDFS \u6570\u636e\u6e90 2\u3001\u6267\u884c\u6302\u8f7d\u547d\u4ee4 jindo admin -mount /oss-hdfs oss://<bucket>/<dir> /oss-hdfs\uff1a\u6302\u8f7d\u5230\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u4e0b\u7684\u8bbf\u95ee\u8def\u5f84 \u540c\u6837\u53ef\u4ee5\u628a\u591a\u4e2a OSS-HDFS bucket \u6302\u8f7d\u5230\u4e0d\u540c\u7684\u8def\u5f84\u4e0b\u9762\u3002","title":"\u6302\u8f7d OSS-HDFS \u670d\u52a1\u6570\u636e\u6e90"},{"location":"backup/jindofsx/jindofsx_multi_mount/#_3","text":"$ hadoop fs -ls jindo:/// drwxrwxr-x - root root 0 1970-01-01 00:00 jindo:///oss drwxrwxr-x - root root 0 1970-01-01 00:00 jindo:///hdfs $ hadoop fs -ls jindo:///oss/ drwxrwxr-x - root root 0 1970-01-01 00:00 jindo:///oss/test_a_dir -rwxrwxr-x 0 root root 19564111 2022-04-11 10:25 jindo:///oss/tmp.txt $ hadoop fs -ls jindo:///hdfs/ drwxrwxr-x - root root 0 1970-01-01 00:00 jindo:///hdfs/test_a_dir -rwxrwxr-x 0 root root 19564111 2022-04-11 10:25 jindo:///hdfs/tmp.txt \u6211\u4eec\u8fd8\u652f\u6301\u5176\u4ed6\u5176\u4ed6\u6570\u636e\u6e90\u7684\u6302\u8f7d\uff0c\u6bd4\u5982 S3\u3001HDFS\u3001NAS\u7b49\uff0c\u53ef\u53c2\u8003\u4ee5\u4e0b\u6587\u6863 JindoFSx \u6302\u8f7d S3 \u6570\u636e\u6e90 JindoFSx \u6302\u8f7d HDFS \u6570\u636e\u6e90 JindoFSx \u6302\u8f7d NAS \u6570\u636e\u6e90","title":"\u4f7f\u7528\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u8bbf\u95ee"},{"location":"backup/jindofsx/jindofsx_p2p/","text":"P2P \u5206\u5e03\u5f0f\u4e0b\u8f7d\u7f13\u5b58\u4f7f\u7528 \u00a7 JindoFSx \u5ba2\u6237\u7aef P2P \u53ef\u4ee5\u88ab\u89c6\u4f5c\u4e00\u79cd\u672c\u5730\u7f13\u5b58\uff08LocalCache\uff09\u3002\u4e0e\u539f\u6709\u7684 LocalCache \u76f8\u6bd4\uff0cP2P \u7f13\u5b58\u4e2d\u7684\u672c\u5730\u6570\u636e\u5757\u4f1a\u4f18\u5148\u4ece\u5176\u4ed6\u6301\u6709\u8be5\u6570\u636e\u7684\u5ba2\u6237\u7aef\u62c9\u53d6\uff0c\u53ea\u6709\u65e0\u6cd5\u5411\u5176\u4ed6\u5ba2\u6237\u7aef\u8bf7\u6c42\u65f6\uff0c\u624d\u4f1a\u4ece STS \u6216\u8fdc\u7aef\u8bfb\u53d6\u3002\u4ece\u6982\u5ff5\u4e0a\u6765\u8bb2\uff0c\u53ef\u4ee5\u5c06 P2P \u7f13\u5b58\u89c6\u4f5c \u201c\u5ba2\u6237\u7aef\u4e4b\u95f4\u53ef\u4ee5\u4ea4\u4e92\u4f20\u8f93\u6570\u636e\u201d \u7684 \u201c\u672c\u5730\u7f13\u5b58\u201d\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u670d\u52a1\u7aef\u914d\u7f6e \u00a7 \u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 [jindofsx-common] jindofsx.p2p.tracker.thread.number = 1 # TrackerService \u7684\u5904\u7406\u7ebf\u7a0b\u6570\uff0c\u82e5\u8981\u5f00\u542f P2P \u529f\u80fd\uff0c\u5219\u5fc5\u987b\u8bbe\u7f6e\u5927\u4e8e 1 \u7684\u503c\u3002\u5982\u679c\u5c0f\u4e8e 1\uff0c\u5c31\u4e0d\u4f1a\u521b\u5efa TrackerService\uff0c\u4e5f\u5c31\u4e0d\u4f1a\u5f00\u542f P2P \u529f\u80fd\u3002 jindofsx.p2p.file.prefix = oss://bucket1/data-dir1/,oss://bucket2/data-dir2/ # \u4f7f\u7528 P2P \u4e0b\u8f7d\u7684\u524d\u7f00\u5217\u8868\uff0c\u7528\u534a\u89d2\u9017\u53f7\u9694\u5f00\uff0c\u6587\u4ef6\u8def\u5f84\u53ea\u6709\u5339\u914d\u5230\u5176\u4e2d\u4efb\u4e00\u4e2a\u524d\u7f00\uff0c\u624d\u4f1a\u4ee5 P2P \u65b9\u5f0f\u4e0b\u8f7d\u3002\u5728\u5e94\u7528\u5c42\u4f7f\u7528\u7edf\u4e00\u6302\u8f7d\u8def\u5f84\u8fdb\u884c\u4e0b\u8f7d\u65f6\uff0c\u8fd9\u91cc\u4ecd\u5e94\u914d\u7f6e\u4e3a\u771f\u5b9e\u7684\u5bf9\u8c61\u8def\u5f84\u3002 \u5e76\u5c06\u914d\u7f6e\u6587\u4ef6\u66f4\u65b0\u5230\u6240\u6709\u670d\u52a1\u8282\u70b9\uff08Namespace Service \u4e0e Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97 P2P \u914d\u7f6e\u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u5ba2\u6237\u7aef\u914d\u7f6e \u00a7 \u5728 $JINDOSDK_HOME/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6\uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [jindosdk] # P2P \u4e0b\u8f7d\u6700\u591a\u5360\u7528\u7684\u7f13\u5b58\u5927\u5c0f\uff0c\u5355\u4f4d\u4e3a\u5b57\u8282\uff0c\u9ed8\u8ba4\u4e3a 5GB\uff0c\u6700\u5c0f\u503c\u4e3a 1GB fs.jindofsx.p2p.cache.capacity.limit = 5 * 1024 * 1024 * 1024 # \u4e0b\u8f7d\u5355\u4e2a\u6587\u4ef6\u4f7f\u7528\u7684\u5e76\u53d1\u6570\u3002 fs.jindofsx.p2p.download.parallelism.per.file = 5 # P2P \u4e0b\u8f7d\u4f7f\u7528\u7684\u7ebf\u7a0b\u6c60\u603b\u5927\u5c0f\u3002 fs.jindofsx.p2p.download.thread.pool.size = 5 \u5e76\u5c06\u914d\u7f6e\u66f4\u65b0\u5230\u6240\u9700\u8282\u70b9\u3002 \u5b8c\u6210\u4ee5\u4e0a\u914d\u7f6e\u540e\uff0c\u4f5c\u4e1a\u81ea\u52a8\u4f7f\u7528 P2P \u7684\u65b9\u5f0f\u8bfb\u53d6\u3002","title":"P2P \u5206\u5e03\u5f0f\u4e0b\u8f7d\u7f13\u5b58\u4f7f\u7528"},{"location":"backup/jindofsx/jindofsx_p2p/#p2p","text":"JindoFSx \u5ba2\u6237\u7aef P2P \u53ef\u4ee5\u88ab\u89c6\u4f5c\u4e00\u79cd\u672c\u5730\u7f13\u5b58\uff08LocalCache\uff09\u3002\u4e0e\u539f\u6709\u7684 LocalCache \u76f8\u6bd4\uff0cP2P \u7f13\u5b58\u4e2d\u7684\u672c\u5730\u6570\u636e\u5757\u4f1a\u4f18\u5148\u4ece\u5176\u4ed6\u6301\u6709\u8be5\u6570\u636e\u7684\u5ba2\u6237\u7aef\u62c9\u53d6\uff0c\u53ea\u6709\u65e0\u6cd5\u5411\u5176\u4ed6\u5ba2\u6237\u7aef\u8bf7\u6c42\u65f6\uff0c\u624d\u4f1a\u4ece STS \u6216\u8fdc\u7aef\u8bfb\u53d6\u3002\u4ece\u6982\u5ff5\u4e0a\u6765\u8bb2\uff0c\u53ef\u4ee5\u5c06 P2P \u7f13\u5b58\u89c6\u4f5c \u201c\u5ba2\u6237\u7aef\u4e4b\u95f4\u53ef\u4ee5\u4ea4\u4e92\u4f20\u8f93\u6570\u636e\u201d \u7684 \u201c\u672c\u5730\u7f13\u5b58\u201d\u3002","title":"P2P \u5206\u5e03\u5f0f\u4e0b\u8f7d\u7f13\u5b58\u4f7f\u7528"},{"location":"backup/jindofsx/jindofsx_p2p/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindofsx_p2p/#_2","text":"\u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 [jindofsx-common] jindofsx.p2p.tracker.thread.number = 1 # TrackerService \u7684\u5904\u7406\u7ebf\u7a0b\u6570\uff0c\u82e5\u8981\u5f00\u542f P2P \u529f\u80fd\uff0c\u5219\u5fc5\u987b\u8bbe\u7f6e\u5927\u4e8e 1 \u7684\u503c\u3002\u5982\u679c\u5c0f\u4e8e 1\uff0c\u5c31\u4e0d\u4f1a\u521b\u5efa TrackerService\uff0c\u4e5f\u5c31\u4e0d\u4f1a\u5f00\u542f P2P \u529f\u80fd\u3002 jindofsx.p2p.file.prefix = oss://bucket1/data-dir1/,oss://bucket2/data-dir2/ # \u4f7f\u7528 P2P \u4e0b\u8f7d\u7684\u524d\u7f00\u5217\u8868\uff0c\u7528\u534a\u89d2\u9017\u53f7\u9694\u5f00\uff0c\u6587\u4ef6\u8def\u5f84\u53ea\u6709\u5339\u914d\u5230\u5176\u4e2d\u4efb\u4e00\u4e2a\u524d\u7f00\uff0c\u624d\u4f1a\u4ee5 P2P \u65b9\u5f0f\u4e0b\u8f7d\u3002\u5728\u5e94\u7528\u5c42\u4f7f\u7528\u7edf\u4e00\u6302\u8f7d\u8def\u5f84\u8fdb\u884c\u4e0b\u8f7d\u65f6\uff0c\u8fd9\u91cc\u4ecd\u5e94\u914d\u7f6e\u4e3a\u771f\u5b9e\u7684\u5bf9\u8c61\u8def\u5f84\u3002 \u5e76\u5c06\u914d\u7f6e\u6587\u4ef6\u66f4\u65b0\u5230\u6240\u6709\u670d\u52a1\u8282\u70b9\uff08Namespace Service \u4e0e Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97 P2P \u914d\u7f6e\u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u670d\u52a1\u7aef\u914d\u7f6e"},{"location":"backup/jindofsx/jindofsx_p2p/#_3","text":"\u5728 $JINDOSDK_HOME/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6\uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [jindosdk] # P2P \u4e0b\u8f7d\u6700\u591a\u5360\u7528\u7684\u7f13\u5b58\u5927\u5c0f\uff0c\u5355\u4f4d\u4e3a\u5b57\u8282\uff0c\u9ed8\u8ba4\u4e3a 5GB\uff0c\u6700\u5c0f\u503c\u4e3a 1GB fs.jindofsx.p2p.cache.capacity.limit = 5 * 1024 * 1024 * 1024 # \u4e0b\u8f7d\u5355\u4e2a\u6587\u4ef6\u4f7f\u7528\u7684\u5e76\u53d1\u6570\u3002 fs.jindofsx.p2p.download.parallelism.per.file = 5 # P2P \u4e0b\u8f7d\u4f7f\u7528\u7684\u7ebf\u7a0b\u6c60\u603b\u5927\u5c0f\u3002 fs.jindofsx.p2p.download.thread.pool.size = 5 \u5e76\u5c06\u914d\u7f6e\u66f4\u65b0\u5230\u6240\u9700\u8282\u70b9\u3002 \u5b8c\u6210\u4ee5\u4e0a\u914d\u7f6e\u540e\uff0c\u4f5c\u4e1a\u81ea\u52a8\u4f7f\u7528 P2P \u7684\u65b9\u5f0f\u8bfb\u53d6\u3002","title":"\u5ba2\u6237\u7aef\u914d\u7f6e"},{"location":"backup/jindofsx/jindofsx_quickstart/","text":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u5feb\u901f\u5165\u95e8 \u00a7 JindoFSx \u662f JindoData \u63d0\u4f9b\u7684\u4e91\u6570\u636e\u8bbf\u95ee\u4e2d\u5fc3\uff0c\u4e3a\u7528\u6237\u7ba1\u7406\u4e91\u5e73\u53f0\u5bf9\u4e8e\u4e91\u5b58\u50a8\u7684\u4f7f\u7528\uff0c\u9488\u5bf9\u5404\u79cd\u8bbf\u95ee\u573a\u666f\uff0c\u63d0\u4f9b\u4e00\u7ad9\u5f0f\u7684\u5b58\u50a8\u4f18\u5316\u670d\u52a1\u3002JindoFSx \u53ef\u4ee5\u4e3a\u7528\u6237\u540c\u65f6\u7ba1\u7406\u591a\u4e2a\u540e\u7aef\u5b58\u50a8\u7cfb\u7edf\u3002\u540c\u65f6\uff0cJindoFSx \u4f1a\u4e3a\u4e0a\u5c42\u5e94\u7528\u6784\u5efa\u57fa\u4e8e\u591a\u79cd\u5b58\u50a8\u4ecb\u8d28\u7684\u6570\u636e\u7f13\u5b58\uff0c\u5e76\u89c4\u5212\u9ad8\u6548\u7684\u6570\u636e\u8bbf\u95ee\u7b56\u7565\uff0c\u4fdd\u969c\u7528\u6237\u4f5c\u4e1a\u9ad8\u901f\u3001\u7a33\u5b9a\u8fd0\u884c\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u3002 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK\u3002 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u6587\u4ef6\u4ee5\u5bf9\u8c61\u7684\u5f62\u5f0f\u5b58\u50a8\u5728 OSS \u4e0a\u3002 \u672c\u6587\u4ee5 JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u4e3a\u4f8b\u3002\u5229\u7528\u96c6\u7fa4\u672c\u8eab\u7684\u5b58\u50a8\u8d44\u6e90\u7f13\u5b58 OSS \u6587\u4ef6\uff0c\u4ece\u800c\u52a0\u901f\u4f5c\u4e1a\u8bbf\u95ee OSS\u3002 \u670d\u52a1\u7aef\u914d\u7f6e\u963f\u91cc\u4e91 OSS AccessKey \u00a7 JindoFSx \u82e5\u8981\u63d0\u4f9b\u7f13\u5b58\u52a0\u901f\u670d\u52a1\uff0c\u9700\u914d\u7f6e\u5177\u6709\u8bbf\u95ee OSS \u6743\u9650\u7684 AccessKey\u3002 \u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u3002 [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = oss-cn-xxx-internal.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002 \u5c06\u914d\u7f6e\u6587\u4ef6\u90e8\u7f72\u5230\u6240\u9700\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1 OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06 OSS \u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.jindofsx.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.jindofsx.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.jindofsx.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u5b8c\u6210\u4ee5\u4e0a\u914d\u7f6e\u540e\uff0c\u4f5c\u4e1a\u8bbf\u95ee OSS \u5373\u53ef\u5229\u7528\u4e0a\u7f13\u5b58\uff0cJindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u4e86\u900f\u660e\u7f13\u5b58\u7684\u4f7f\u7528\u65b9\u5f0f\uff0c\u4f5c\u4e1a\u8bbf\u95ee OSS \u7684\u65b9\u5f0f\u65e0\u9700\u505a\u4efb\u4f55\u4fee\u6539\u3002 \u4f5c\u4e1a\u8bfb\u53d6 OSS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002 \u6ce8\u610f\uff1a\u6b64\u914d\u7f6e\u4e3a\u5ba2\u6237\u7aef\u914d\u7f6e\uff0c\u4e0d\u9700\u8981\u91cd\u542f JindoFSx \u670d\u52a1\u3002 \u4f7f\u7528 Prometheus + Grafana \u53ef\u89c6\u5316\u6307\u6807\u89c2\u6d4b\u5e73\u53f0 \u00a7 \u5728\u5b8c\u6210 JindoFSx \u7f13\u5b58\u96c6\u7fa4\u7684\u542f\u52a8\u540e\uff0cJindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u57fa\u4e8e Prometheus + Grafana \u7684\u53ef\u89c6\u5316\u6307\u6807\u89c2\u6d4b\u5e73\u53f0\uff0c\u4f60\u53ef\u4ee5\u53c2\u8003\u5982\u4e0b\u6587\u6863\u8fdb\u884c\u5b89\u88c5\u548c\u4f7f\u7528\u8be5\u529f\u80fd \u57fa\u4e8e Prometheus + Grafana \u7684\u53ef\u89c6\u5316\u6307\u6807\u89c2\u6d4b\u5e73\u53f0 \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u5feb\u901f\u5165\u95e8"},{"location":"backup/jindofsx/jindofsx_quickstart/#jindofsx","text":"JindoFSx \u662f JindoData \u63d0\u4f9b\u7684\u4e91\u6570\u636e\u8bbf\u95ee\u4e2d\u5fc3\uff0c\u4e3a\u7528\u6237\u7ba1\u7406\u4e91\u5e73\u53f0\u5bf9\u4e8e\u4e91\u5b58\u50a8\u7684\u4f7f\u7528\uff0c\u9488\u5bf9\u5404\u79cd\u8bbf\u95ee\u573a\u666f\uff0c\u63d0\u4f9b\u4e00\u7ad9\u5f0f\u7684\u5b58\u50a8\u4f18\u5316\u670d\u52a1\u3002JindoFSx \u53ef\u4ee5\u4e3a\u7528\u6237\u540c\u65f6\u7ba1\u7406\u591a\u4e2a\u540e\u7aef\u5b58\u50a8\u7cfb\u7edf\u3002\u540c\u65f6\uff0cJindoFSx \u4f1a\u4e3a\u4e0a\u5c42\u5e94\u7528\u6784\u5efa\u57fa\u4e8e\u591a\u79cd\u5b58\u50a8\u4ecb\u8d28\u7684\u6570\u636e\u7f13\u5b58\uff0c\u5e76\u89c4\u5212\u9ad8\u6548\u7684\u6570\u636e\u8bbf\u95ee\u7b56\u7565\uff0c\u4fdd\u969c\u7528\u6237\u4f5c\u4e1a\u9ad8\u901f\u3001\u7a33\u5b9a\u8fd0\u884c\u3002","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u5feb\u901f\u5165\u95e8"},{"location":"backup/jindofsx/jindofsx_quickstart/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u3002 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK\u3002 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u6587\u4ef6\u4ee5\u5bf9\u8c61\u7684\u5f62\u5f0f\u5b58\u50a8\u5728 OSS \u4e0a\u3002 \u672c\u6587\u4ee5 JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u4e3a\u4f8b\u3002\u5229\u7528\u96c6\u7fa4\u672c\u8eab\u7684\u5b58\u50a8\u8d44\u6e90\u7f13\u5b58 OSS \u6587\u4ef6\uff0c\u4ece\u800c\u52a0\u901f\u4f5c\u4e1a\u8bbf\u95ee OSS\u3002","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindofsx_quickstart/#oss-accesskey","text":"JindoFSx \u82e5\u8981\u63d0\u4f9b\u7f13\u5b58\u52a0\u901f\u670d\u52a1\uff0c\u9700\u914d\u7f6e\u5177\u6709\u8bbf\u95ee OSS \u6743\u9650\u7684 AccessKey\u3002 \u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u3002 [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = oss-cn-xxx-internal.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002 \u5c06\u914d\u7f6e\u6587\u4ef6\u90e8\u7f72\u5230\u6240\u9700\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002","title":"\u670d\u52a1\u7aef\u914d\u7f6e\u963f\u91cc\u4e91 OSS AccessKey"},{"location":"backup/jindofsx/jindofsx_quickstart/#jindofsx_1","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/jindofsx_quickstart/#jindosdk","text":"\u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1 OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06 OSS \u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.jindofsx.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.jindofsx.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.jindofsx.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u5b8c\u6210\u4ee5\u4e0a\u914d\u7f6e\u540e\uff0c\u4f5c\u4e1a\u8bbf\u95ee OSS \u5373\u53ef\u5229\u7528\u4e0a\u7f13\u5b58\uff0cJindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u4e86\u900f\u660e\u7f13\u5b58\u7684\u4f7f\u7528\u65b9\u5f0f\uff0c\u4f5c\u4e1a\u8bbf\u95ee OSS \u7684\u65b9\u5f0f\u65e0\u9700\u505a\u4efb\u4f55\u4fee\u6539\u3002 \u4f5c\u4e1a\u8bfb\u53d6 OSS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002 \u6ce8\u610f\uff1a\u6b64\u914d\u7f6e\u4e3a\u5ba2\u6237\u7aef\u914d\u7f6e\uff0c\u4e0d\u9700\u8981\u91cd\u542f JindoFSx \u670d\u52a1\u3002","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/jindofsx_quickstart/#prometheus-grafana","text":"\u5728\u5b8c\u6210 JindoFSx \u7f13\u5b58\u96c6\u7fa4\u7684\u542f\u52a8\u540e\uff0cJindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u57fa\u4e8e Prometheus + Grafana \u7684\u53ef\u89c6\u5316\u6307\u6807\u89c2\u6d4b\u5e73\u53f0\uff0c\u4f60\u53ef\u4ee5\u53c2\u8003\u5982\u4e0b\u6587\u6863\u8fdb\u884c\u5b89\u88c5\u548c\u4f7f\u7528\u8be5\u529f\u80fd \u57fa\u4e8e Prometheus + Grafana \u7684\u53ef\u89c6\u5316\u6307\u6807\u89c2\u6d4b\u5e73\u53f0","title":"\u4f7f\u7528 Prometheus + Grafana \u53ef\u89c6\u5316\u6307\u6807\u89c2\u6d4b\u5e73\u53f0"},{"location":"backup/jindofsx/jindofsx_quickstart/#_2","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/jindoshell_jindofsx_howto/","text":"JindoShell CLI \u652f\u6301 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4f7f\u7528\u8bf4\u660e \u00a7 JindoShell CLI \u652f\u6301\u64cd\u4f5c JindoFSx \u6570\u636e\u7f13\u5b58\u3001\u5143\u6570\u636e\u7f13\u5b58\u548c\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u7b49\u547d\u4ee4\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u6570\u636e\u7f13\u5b58\u547d\u4ee4 \u00a7 \u6570\u636e\u7f13\u5b58\u547d\u4ee4\u53ef\u4ee5\u5907\u4efd\u5bf9\u5e94\u8def\u5f84\u7684\u6570\u636e\u81f3\u672c\u96c6\u7fa4\u7684\u78c1\u76d8\uff0c\u4ee5\u4fbf\u4e8e\u540e\u7eed\u53ef\u4ee5\u8bfb\u53d6\u672c\u5730\u6570\u636e\uff0c\u65e0\u9700\u8bfb\u53d6OSS\u7b49\u540e\u7aef\u4e0a\u7684\u6570\u636e\u3002 jindo fs -load -data <options> <path> options \u5404\u79cd\u53ef\u9009\u53c2\u6570 path \u6570\u636e\u7f13\u5b58\u8def\u5f84 \u5176\u4e2d options \u53ef\u9009\u53c2\u6570\u6709: \u53c2\u6570 \u8bf4\u660e -s \u8868\u793a\u7f13\u5b58\u8fc7\u7a0b\u540c\u6b65\u6267\u884c\uff0c\u6253\u5370\u8fdb\u5ea6\u548c\u6267\u884c\u4fe1\u606f\uff0c\u63a8\u8350\u5f00\u542f -replica \u7f13\u5b58\u526f\u672c\u6570\u91cf\uff0c\u9ed8\u8ba4\u7f13\u5b581\u4e2a\u526f\u672c -R \u9012\u5f52\u7f13\u5b58\u6587\u4ef6\uff0c\u5f53 path \u662f\u6587\u4ef6\u5939\u662f\u5f00\u542f -cachelist \u63a5\u53d7\u672c\u5730\u6587\u4ef6\uff0c\u6587\u4ef6\u5185\u5bb9\u4e3acache\u5217\u8868 \u63a8\u8350\u4f7f\u7528\u547d\u4ee4\u7ec4\u5408 jindo fs -load -data -s -R <path> \u5143\u6570\u636e\u7f13\u5b58\u547d\u4ee4 \u00a7 \u5143\u6570\u636e\u7f13\u5b58\u547d\u4ee4\u53ef\u4ee5\u5907\u4efd\u8fdc\u7aef\u6587\u4ef6\u7684\u5143\u6570\u636e\u4fe1\u606f\uff0c\u4ece\u800c\u540e\u7eed\u65e0\u9700\u4eceOSS\u7b49\u540e\u7aef\u8bfb\u53d6\u6587\u4ef6\u5143\u6570\u636e\u4fe1\u606f\u3002 jindo fs -load -meta <options> <path> options \u5404\u79cd\u53ef\u9009\u53c2\u6570 path \u6570\u636e\u7f13\u5b58\u8def\u5f84 \u5176\u4e2d options \u53ef\u9009\u53c2\u6570\u6709: \u53c2\u6570 \u8bf4\u660e -s \u8868\u793a\u5143\u7f13\u5b58\u8fc7\u7a0b\u540c\u6b65\u6267\u884c\uff0c\u6253\u5370\u8fdb\u5ea6\u548c\u6267\u884c\u4fe1\u606f\uff0c\u63a8\u8350\u5f00\u542f -R \u9012\u5f52\u7f13\u5b58\u6587\u4ef6\uff0c\u5f53 path \u662f\u6587\u4ef6\u5939\u662f\u5f00\u542f \u63a8\u8350\u4f7f\u7528\u547d\u4ee4\u7ec4\u5408 jindo fs -load -meta -s -R <path> \u6570\u636e\u7f13\u5b58\u548c\u5143\u6570\u636e\u7f13\u5b58\u53ef\u4ee5\u7ec4\u5408\u4f7f\u7528\uff0c\u5f53\u9700\u8981\u540c\u65f6\u8fdb\u884c\u4e8c\u8005\u7f13\u5b58\u65f6\uff0c\u53ef\u642d\u914d\u4e0a\u9762\u53c2\u6570\u4f7f\u7528\u3002 \u63a8\u8350\u4f7f\u7528\u547d\u4ee4\u7ec4\u5408 jindo fs -load -meta -data -s -R <path> \u901a\u8fc7\u901a\u914d\u7b26\u5339\u914d\u6765\u6307\u5b9a\u7f13\u5b58\u6587\u4ef6 \u00a7 \u901a\u8fc7\u6307\u5b9a\u901a\u914d\u7b26\u6765\u9009\u62e9\u5728\u5177\u4f53\u7684 path \u4e0b\u9700\u8981\u7f13\u5b58\u7684\u6587\u4ef6\uff0c\u7c7b\u4f3c\u767d\u540d\u5355\u529f\u80fd jindo fs -load -meta -data -s -R -filter <wildcard> -dryrun <path> \u53c2\u6570 \u8bf4\u660e -filter \u8868\u793a\u5177\u4f53\u7684\u901a\u914d\u7b26\uff0c\u7528\u6765\u5bf9path\u4e0b\u7684\u6240\u6709\u6587\u4ef6\u8fdb\u884c\u5339\u914d\u3002\u6bd4\u5982 .log,\u5982\u679c\u6709\u591a\u4e2a\u901a\u914d\u7b26\u7528,\u9694\u5f00,\u6bd4\u5982 .log,*.dat -dryrun \u8868\u793a\u5355\u7eaf\u8f93\u51fa\u7b26\u5408\u5339\u914d\u7684\u7ed3\u679c\uff0c\u4e0d\u771f\u6b63\u8fdb\u884c\u7f13\u5b58\uff0c\u7528\u4e8e\u8fdb\u884c\u5224\u65ad\u901a\u914d\u7b26\u662f\u5426\u6b63\u786e -thread \u8868\u793a\u5ba2\u6237\u7aef\u5bf9\u8fc7\u6ee4\u51fa\u6765\u7684\u6587\u4ef6\u8fdb\u884c\u7f13\u5b58\u7684\u7ebf\u7a0b\u6570\uff0c\u9ed8\u8ba4\u662f10\u4e2a \u4f7f\u7528\u793a\u4f8b jindo fs -load -meta -data -s -R -filter *.log oss://bucket/dir \u6e05\u7406\u7f13\u5b58\u547d\u4ee4 \u00a7 uncache\u547d\u4ee4\u53ef\u4ee5\u5220\u9664\u672c\u5730\u96c6\u7fa4\u4e2d\u7684\u672c\u5730\u5907\u4efd\uff0c\u53ea\u5b58\u50a8\u6570\u636e\u5728OSS\u6807\u51c6\u5b58\u50a8\u4e0a\uff0c\u4ee5\u4fbf\u4e8e\u540e\u7eed\u8bfb\u53d6OSS\u4e0a\u7684\u6570\u636e\u3002 jindo fs -uncache <path> \u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u547d\u4ee4 \u00a7 \u6dfb\u52a0\u4e00\u4e2a\u6302\u8f7d\u70b9\u3002 jindo admin -mount <path> <realpath> \u79fb\u9664\u4e00\u4e2a\u6302\u8f7d\u70b9\u3002 jindo admin -unmount <path> \u5176\u4ed6\u547d\u4ee4 \u00a7 \u8f93\u51fa\u5f53\u524dJindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7684\u4e00\u4e9b\u4fe1\u606f\uff0c\u6bd4\u5982\u7f13\u5b58\u5927\u5c0f\uff0c\u7f13\u5b58\u5bb9\u91cf\u7b49\u3002 jindo fs -report \u8f93\u51fa\u683c\u5f0f\u5982\u4e0b\uff1a Namespace Address: 127.0.0.1:8101 Rpc Port: 8101 Started: Mon Jan 10 15:23:51 2022 Version: 4.1.0 Live Nodes: 2 Decommission Nodes: 0 Total Disk Capacity: 438.17GB Used Disk Capacity: 5120.00MB Total MEM Capacity: 4096.00MB Used MEM Capacity: 0B","title":"JindoShell CLI \u652f\u6301 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofsx/jindoshell_jindofsx_howto/#jindoshell-cli-jindofsx","text":"JindoShell CLI \u652f\u6301\u64cd\u4f5c JindoFSx \u6570\u636e\u7f13\u5b58\u3001\u5143\u6570\u636e\u7f13\u5b58\u548c\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u7b49\u547d\u4ee4\u3002","title":"JindoShell CLI \u652f\u6301 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofsx/jindoshell_jindofsx_howto/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindoshell_jindofsx_howto/#_2","text":"\u6570\u636e\u7f13\u5b58\u547d\u4ee4\u53ef\u4ee5\u5907\u4efd\u5bf9\u5e94\u8def\u5f84\u7684\u6570\u636e\u81f3\u672c\u96c6\u7fa4\u7684\u78c1\u76d8\uff0c\u4ee5\u4fbf\u4e8e\u540e\u7eed\u53ef\u4ee5\u8bfb\u53d6\u672c\u5730\u6570\u636e\uff0c\u65e0\u9700\u8bfb\u53d6OSS\u7b49\u540e\u7aef\u4e0a\u7684\u6570\u636e\u3002 jindo fs -load -data <options> <path> options \u5404\u79cd\u53ef\u9009\u53c2\u6570 path \u6570\u636e\u7f13\u5b58\u8def\u5f84 \u5176\u4e2d options \u53ef\u9009\u53c2\u6570\u6709: \u53c2\u6570 \u8bf4\u660e -s \u8868\u793a\u7f13\u5b58\u8fc7\u7a0b\u540c\u6b65\u6267\u884c\uff0c\u6253\u5370\u8fdb\u5ea6\u548c\u6267\u884c\u4fe1\u606f\uff0c\u63a8\u8350\u5f00\u542f -replica \u7f13\u5b58\u526f\u672c\u6570\u91cf\uff0c\u9ed8\u8ba4\u7f13\u5b581\u4e2a\u526f\u672c -R \u9012\u5f52\u7f13\u5b58\u6587\u4ef6\uff0c\u5f53 path \u662f\u6587\u4ef6\u5939\u662f\u5f00\u542f -cachelist \u63a5\u53d7\u672c\u5730\u6587\u4ef6\uff0c\u6587\u4ef6\u5185\u5bb9\u4e3acache\u5217\u8868 \u63a8\u8350\u4f7f\u7528\u547d\u4ee4\u7ec4\u5408 jindo fs -load -data -s -R <path>","title":"\u6570\u636e\u7f13\u5b58\u547d\u4ee4"},{"location":"backup/jindofsx/jindoshell_jindofsx_howto/#_3","text":"\u5143\u6570\u636e\u7f13\u5b58\u547d\u4ee4\u53ef\u4ee5\u5907\u4efd\u8fdc\u7aef\u6587\u4ef6\u7684\u5143\u6570\u636e\u4fe1\u606f\uff0c\u4ece\u800c\u540e\u7eed\u65e0\u9700\u4eceOSS\u7b49\u540e\u7aef\u8bfb\u53d6\u6587\u4ef6\u5143\u6570\u636e\u4fe1\u606f\u3002 jindo fs -load -meta <options> <path> options \u5404\u79cd\u53ef\u9009\u53c2\u6570 path \u6570\u636e\u7f13\u5b58\u8def\u5f84 \u5176\u4e2d options \u53ef\u9009\u53c2\u6570\u6709: \u53c2\u6570 \u8bf4\u660e -s \u8868\u793a\u5143\u7f13\u5b58\u8fc7\u7a0b\u540c\u6b65\u6267\u884c\uff0c\u6253\u5370\u8fdb\u5ea6\u548c\u6267\u884c\u4fe1\u606f\uff0c\u63a8\u8350\u5f00\u542f -R \u9012\u5f52\u7f13\u5b58\u6587\u4ef6\uff0c\u5f53 path \u662f\u6587\u4ef6\u5939\u662f\u5f00\u542f \u63a8\u8350\u4f7f\u7528\u547d\u4ee4\u7ec4\u5408 jindo fs -load -meta -s -R <path> \u6570\u636e\u7f13\u5b58\u548c\u5143\u6570\u636e\u7f13\u5b58\u53ef\u4ee5\u7ec4\u5408\u4f7f\u7528\uff0c\u5f53\u9700\u8981\u540c\u65f6\u8fdb\u884c\u4e8c\u8005\u7f13\u5b58\u65f6\uff0c\u53ef\u642d\u914d\u4e0a\u9762\u53c2\u6570\u4f7f\u7528\u3002 \u63a8\u8350\u4f7f\u7528\u547d\u4ee4\u7ec4\u5408 jindo fs -load -meta -data -s -R <path>","title":"\u5143\u6570\u636e\u7f13\u5b58\u547d\u4ee4"},{"location":"backup/jindofsx/jindoshell_jindofsx_howto/#_4","text":"\u901a\u8fc7\u6307\u5b9a\u901a\u914d\u7b26\u6765\u9009\u62e9\u5728\u5177\u4f53\u7684 path \u4e0b\u9700\u8981\u7f13\u5b58\u7684\u6587\u4ef6\uff0c\u7c7b\u4f3c\u767d\u540d\u5355\u529f\u80fd jindo fs -load -meta -data -s -R -filter <wildcard> -dryrun <path> \u53c2\u6570 \u8bf4\u660e -filter \u8868\u793a\u5177\u4f53\u7684\u901a\u914d\u7b26\uff0c\u7528\u6765\u5bf9path\u4e0b\u7684\u6240\u6709\u6587\u4ef6\u8fdb\u884c\u5339\u914d\u3002\u6bd4\u5982 .log,\u5982\u679c\u6709\u591a\u4e2a\u901a\u914d\u7b26\u7528,\u9694\u5f00,\u6bd4\u5982 .log,*.dat -dryrun \u8868\u793a\u5355\u7eaf\u8f93\u51fa\u7b26\u5408\u5339\u914d\u7684\u7ed3\u679c\uff0c\u4e0d\u771f\u6b63\u8fdb\u884c\u7f13\u5b58\uff0c\u7528\u4e8e\u8fdb\u884c\u5224\u65ad\u901a\u914d\u7b26\u662f\u5426\u6b63\u786e -thread \u8868\u793a\u5ba2\u6237\u7aef\u5bf9\u8fc7\u6ee4\u51fa\u6765\u7684\u6587\u4ef6\u8fdb\u884c\u7f13\u5b58\u7684\u7ebf\u7a0b\u6570\uff0c\u9ed8\u8ba4\u662f10\u4e2a \u4f7f\u7528\u793a\u4f8b jindo fs -load -meta -data -s -R -filter *.log oss://bucket/dir","title":"\u901a\u8fc7\u901a\u914d\u7b26\u5339\u914d\u6765\u6307\u5b9a\u7f13\u5b58\u6587\u4ef6"},{"location":"backup/jindofsx/jindoshell_jindofsx_howto/#_5","text":"uncache\u547d\u4ee4\u53ef\u4ee5\u5220\u9664\u672c\u5730\u96c6\u7fa4\u4e2d\u7684\u672c\u5730\u5907\u4efd\uff0c\u53ea\u5b58\u50a8\u6570\u636e\u5728OSS\u6807\u51c6\u5b58\u50a8\u4e0a\uff0c\u4ee5\u4fbf\u4e8e\u540e\u7eed\u8bfb\u53d6OSS\u4e0a\u7684\u6570\u636e\u3002 jindo fs -uncache <path>","title":"\u6e05\u7406\u7f13\u5b58\u547d\u4ee4"},{"location":"backup/jindofsx/jindoshell_jindofsx_howto/#_6","text":"\u6dfb\u52a0\u4e00\u4e2a\u6302\u8f7d\u70b9\u3002 jindo admin -mount <path> <realpath> \u79fb\u9664\u4e00\u4e2a\u6302\u8f7d\u70b9\u3002 jindo admin -unmount <path>","title":"\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u547d\u4ee4"},{"location":"backup/jindofsx/jindoshell_jindofsx_howto/#_7","text":"\u8f93\u51fa\u5f53\u524dJindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7684\u4e00\u4e9b\u4fe1\u606f\uff0c\u6bd4\u5982\u7f13\u5b58\u5927\u5c0f\uff0c\u7f13\u5b58\u5bb9\u91cf\u7b49\u3002 jindo fs -report \u8f93\u51fa\u683c\u5f0f\u5982\u4e0b\uff1a Namespace Address: 127.0.0.1:8101 Rpc Port: 8101 Started: Mon Jan 10 15:23:51 2022 Version: 4.1.0 Live Nodes: 2 Decommission Nodes: 0 Total Disk Capacity: 438.17GB Used Disk Capacity: 5120.00MB Total MEM Capacity: 4096.00MB Used MEM Capacity: 0B","title":"\u5176\u4ed6\u547d\u4ee4"},{"location":"backup/jindofsx/outline/","text":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7528\u6237\u6587\u6863 \u00a7 \u5feb\u901f\u5165\u95e8 \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u5feb\u901f\u5165\u95e8 \u96c6\u7fa4\u90e8\u7f72 \u00a7 \u5feb\u901f\u90e8\u7f72\u4e00\u4e2a\u7b80\u5355\u7684 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u90e8\u7f72\u9ad8\u53ef\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u57fa\u4e8e Prometheus + Grafana \u642d\u5efa JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u53ef\u89c6\u5316\u6307\u6807\u89c2\u6d4b\u5e73\u53f0 \u57fa\u672c\u529f\u80fd \u00a7 \u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u6df7\u5408\u6302\u8f7d JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS \u7f13\u5b58\u52a0\u901f \u00a7 \u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f \u963f\u91cc\u4e91 OSS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7f13\u5b58\u52a0\u901f \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u900f\u660e\u7f13\u5b58\u52a0\u901f \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f Apache HDFS \u7f13\u5b58\u52a0\u901f \u00a7 Apache HDFS \u900f\u660e\u7f13\u5b58\u52a0\u901f Apache HDFS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f \u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8 NAS \u7f13\u5b58\u52a0\u901f \u00a7 \u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8 NAS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f \u591a\u4e91\u5b58\u50a8\u652f\u6301 [NEW] \u00a7 S3\u3001COS\u3001OBS \u591a\u4e91\u5b58\u50a8\u652f\u6301 JindoFSx \u5ba2\u6237\u7aef\u4fa7 P2P \u52a0\u901f \u00a7 P2P \u5206\u5e03\u5f0f\u4e0b\u8f7d\u7f13\u5b58\u4f7f\u7528\u8bf4\u660e JindoShell CLI \u652f\u6301 \u00a7 JindoShell CLI \u652f\u6301 JindoFSx \u4f7f\u7528\u8bf4\u660e JindoFSx \u652f\u6301 Kerberos + Ranger \u9274\u6743 \u00a7 \u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u6027\u80fd\u6d4b\u8bd5 \u00a7 \u5927\u6570\u636e\u751f\u6001 \u00a7 Hadoop \u7ec4\u4ef6 \u00a7 Hadoop \u8bbf\u95ee\u963f\u91cc\u4e91 OSS + JindoFSx \u900f\u660e\u52a0\u901f Hadoop \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09+ JindoFSx \u900f\u660e\u52a0\u901f Hadoop \u8bbf\u95ee JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e Spark \u7ec4\u4ef6 \u00a7 Spark \u5904\u7406\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Spark \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Spark \u5904\u7406 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e Hive \u7ec4\u4ef6 \u00a7 Hive \u5904\u7406\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Hive \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Hive \u5904\u7406 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e Presto \u7ec4\u4ef6 \u00a7 Presto \u67e5\u8be2\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Presto \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Presto \u67e5\u8be2 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e Impala \u7ec4\u4ef6 \u00a7 Impala \u67e5\u8be2\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Impala \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Impala \u67e5\u8be2 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e HBase \u7ec4\u4ef6 \u00a7 HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 + JindoFSx \u900f\u660e\u52a0\u901f HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 + JindoFSx \u900f\u660e\u52a0\u901f HBase \u4f7f\u7528 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e AI \u751f\u6001 \u00a7 JindoFuse \u4f7f\u7528 \u00a7 JindoFuse \u8bfb\u5199 JindoFSx \u7f13\u5b58(oss://\u8def\u5f84\u6302\u8f7d) JindoFuse \u8bfb\u5199 JindoFSx \u7f13\u5b58(jindo://\u8def\u5f84\u6302\u8f7d) Fluid + JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e91\u539f\u751f \u00a7 Fluid \u7ed3\u5408 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4ecb\u7ecd\u548c\u4f7f\u7528 \u6700\u4f73\u5b9e\u8df5 \u00a7 \u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e \u00a7 \u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e \u5386\u53f2\u7248\u672c \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.6.0 \u7528\u6237\u6587\u6863 \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.5.1 \u7528\u6237\u6587\u6863 \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.5.0 \u7528\u6237\u6587\u6863 \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.4.0 \u7528\u6237\u6587\u6863 \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.3.0 \u7528\u6237\u6587\u6863 \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.2.0 \u7528\u6237\u6587\u6863 \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.1.0 \u7528\u6237\u6587\u6863 \u00a7","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7528\u6237\u6587\u6863"},{"location":"backup/jindofsx/outline/#jindofsx","text":"","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7528\u6237\u6587\u6863"},{"location":"backup/jindofsx/outline/#_1","text":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u5feb\u901f\u5165\u95e8","title":"\u5feb\u901f\u5165\u95e8"},{"location":"backup/jindofsx/outline/#_2","text":"\u5feb\u901f\u90e8\u7f72\u4e00\u4e2a\u7b80\u5355\u7684 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u90e8\u7f72\u9ad8\u53ef\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u57fa\u4e8e Prometheus + Grafana \u642d\u5efa JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u53ef\u89c6\u5316\u6307\u6807\u89c2\u6d4b\u5e73\u53f0","title":"\u96c6\u7fa4\u90e8\u7f72"},{"location":"backup/jindofsx/outline/#_3","text":"\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u6df7\u5408\u6302\u8f7d JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e","title":"\u57fa\u672c\u529f\u80fd"},{"location":"backup/jindofsx/outline/#oss","text":"\u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f \u963f\u91cc\u4e91 OSS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f","title":"\u963f\u91cc\u4e91 OSS \u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/outline/#oss-hdfs-jindofs","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u900f\u660e\u7f13\u5b58\u52a0\u901f \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/outline/#apache-hdfs","text":"Apache HDFS \u900f\u660e\u7f13\u5b58\u52a0\u901f Apache HDFS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f","title":"Apache HDFS \u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/outline/#nas","text":"\u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8 NAS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f","title":"\u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8 NAS \u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/outline/#new","text":"S3\u3001COS\u3001OBS \u591a\u4e91\u5b58\u50a8\u652f\u6301","title":"\u591a\u4e91\u5b58\u50a8\u652f\u6301 [NEW]"},{"location":"backup/jindofsx/outline/#jindofsx-p2p","text":"P2P \u5206\u5e03\u5f0f\u4e0b\u8f7d\u7f13\u5b58\u4f7f\u7528\u8bf4\u660e","title":"JindoFSx \u5ba2\u6237\u7aef\u4fa7 P2P \u52a0\u901f"},{"location":"backup/jindofsx/outline/#jindoshell-cli","text":"JindoShell CLI \u652f\u6301 JindoFSx \u4f7f\u7528\u8bf4\u660e","title":"JindoShell CLI \u652f\u6301"},{"location":"backup/jindofsx/outline/#jindofsx-kerberos-ranger","text":"\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848","title":"JindoFSx \u652f\u6301 Kerberos + Ranger \u9274\u6743"},{"location":"backup/jindofsx/outline/#_4","text":"","title":"\u6027\u80fd\u6d4b\u8bd5"},{"location":"backup/jindofsx/outline/#_5","text":"","title":"\u5927\u6570\u636e\u751f\u6001"},{"location":"backup/jindofsx/outline/#hadoop","text":"Hadoop \u8bbf\u95ee\u963f\u91cc\u4e91 OSS + JindoFSx \u900f\u660e\u52a0\u901f Hadoop \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09+ JindoFSx \u900f\u660e\u52a0\u901f Hadoop \u8bbf\u95ee JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e","title":"Hadoop \u7ec4\u4ef6"},{"location":"backup/jindofsx/outline/#spark","text":"Spark \u5904\u7406\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Spark \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Spark \u5904\u7406 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e","title":"Spark \u7ec4\u4ef6"},{"location":"backup/jindofsx/outline/#hive","text":"Hive \u5904\u7406\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Hive \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Hive \u5904\u7406 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e","title":"Hive \u7ec4\u4ef6"},{"location":"backup/jindofsx/outline/#presto","text":"Presto \u67e5\u8be2\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Presto \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Presto \u67e5\u8be2 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e","title":"Presto \u7ec4\u4ef6"},{"location":"backup/jindofsx/outline/#impala","text":"Impala \u67e5\u8be2\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Impala \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Impala \u67e5\u8be2 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e","title":"Impala \u7ec4\u4ef6"},{"location":"backup/jindofsx/outline/#hbase","text":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 + JindoFSx \u900f\u660e\u52a0\u901f HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 + JindoFSx \u900f\u660e\u52a0\u901f HBase \u4f7f\u7528 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e","title":"HBase \u7ec4\u4ef6"},{"location":"backup/jindofsx/outline/#ai","text":"","title":"AI \u751f\u6001"},{"location":"backup/jindofsx/outline/#jindofuse","text":"JindoFuse \u8bfb\u5199 JindoFSx \u7f13\u5b58(oss://\u8def\u5f84\u6302\u8f7d) JindoFuse \u8bfb\u5199 JindoFSx \u7f13\u5b58(jindo://\u8def\u5f84\u6302\u8f7d)","title":"JindoFuse \u4f7f\u7528"},{"location":"backup/jindofsx/outline/#fluid-jindofsx","text":"Fluid \u7ed3\u5408 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4ecb\u7ecd\u548c\u4f7f\u7528","title":"Fluid + JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e91\u539f\u751f"},{"location":"backup/jindofsx/outline/#_6","text":"","title":"\u6700\u4f73\u5b9e\u8df5"},{"location":"backup/jindofsx/outline/#jindotable-hive","text":"\u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e","title":"\u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e"},{"location":"backup/jindofsx/outline/#_7","text":"","title":"\u5386\u53f2\u7248\u672c"},{"location":"backup/jindofsx/outline/#jindofsx-460","text":"","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.6.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofsx/outline/#jindofsx-451","text":"","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.5.1 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofsx/outline/#jindofsx-450","text":"","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.5.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofsx/outline/#jindofsx-440","text":"","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.4.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofsx/outline/#jindofsx-430","text":"","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.3.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofsx/outline/#jindofsx-420","text":"","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.2.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofsx/outline/#jindofsx-410","text":"","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.1.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofsx/configuration/jindosdk_configuration_list/","text":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e \u00a7 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- xengine \u5f00\u5173 --> <name>fs.xengine</name> <value>jindofsx</value> </property> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>false</value> </property> <property> <!-- \u5143\u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.meta.cache.enable</name> <value>false</value> </property> <property> <!-- \u5c0f\u6587\u4ef6\u7f13\u5b58\u4f18\u5316\u5f00\u5173 --> <name>fs.jindofsx.slice.cache.enable</name> <value>false</value> </property> <property> <!-- \u5185\u5b58\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.ram.cache.enable</name> <value>false</value> </property> <property> <!-- \u662f\u5426\u76f4\u63a5\u8fde\u63a5\u672c\u5730 storageservice --> <name>fs.jindofsx.storage.connect.enable</name> <value>true</value> </property> <property> <!-- \u77ed\u8def\u8bfb\u5f00\u5173\uff0c\u5728\u7f13\u5b58\u7ed3\u70b9\u4e0e\u5ba2\u6237\u7aef\u5728\u540c\u4e00\u53f0\u673a\u5668\u4e0a\u7684\u65f6\u5019\u53ef\u4ee5\u76f4\u63a5\u8bfb\u672c\u5730\u6587\u4ef6\uff0c\u8981\u6c42\u6253\u5f00\u672c\u5730 storageservice \u8fde\u63a5 --> <name>fs.jindofsx.short.circuit.enable</name> <value>true</value> </property> </configuration> RPC \u4f18\u5316\u76f8\u5173\u53c2\u6570 \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- brpc\u7ebf\u7a0b\u6c60\u5927\u5c0f --> <name>fs.jindofsx.rpc.backup.threads</name> <value>100</value> </property> <property> <!-- \u5e76\u53d1\u7ebf\u7a0b\u6570\u91cf --> <name>fs.jindofsx.rpc.thread.concurrency</name> <value>10</value> </property> <property> <!-- \u6d88\u606f\u4f53\u5927\u5c0f --> <name>fs.jindofsx.rpc.body.size</name> <value>256 * 1024 * 1024</value> </property> <property> <!-- \u8d85\u65f6\u65f6\u95f4\u5927\u5c0f\uff08\u6beb\u79d2\uff09--> <name>fs.jindofsx.rpc.timeout</name> <value>12 * 60 * 60 * 1000</value> </property> <property> <!-- \u6d88\u606f\u961f\u5217\u5927\u5c0f --> <name>fs.jindofsx.rpc.max.unwritten.size</name> <value>256 * 1024 * 1024</value> </property> </configuration> \u7f13\u5b58 Metrics \u76f8\u5173\u53c2\u6570 \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u5ba2\u6237\u7aefmetrics\u5f00\u5173 --> <name>fs.jindofsx.client.metrics.enable</name> <value>false</value> </property> </configuration> \u3010\u6ce8\uff1a4.3.0 \u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u3011","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/jindofsx/configuration/jindosdk_configuration_list/#jindosdk","text":"","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/jindofsx/configuration/jindosdk_configuration_list/#_1","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- xengine \u5f00\u5173 --> <name>fs.xengine</name> <value>jindofsx</value> </property> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>false</value> </property> <property> <!-- \u5143\u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.meta.cache.enable</name> <value>false</value> </property> <property> <!-- \u5c0f\u6587\u4ef6\u7f13\u5b58\u4f18\u5316\u5f00\u5173 --> <name>fs.jindofsx.slice.cache.enable</name> <value>false</value> </property> <property> <!-- \u5185\u5b58\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.ram.cache.enable</name> <value>false</value> </property> <property> <!-- \u662f\u5426\u76f4\u63a5\u8fde\u63a5\u672c\u5730 storageservice --> <name>fs.jindofsx.storage.connect.enable</name> <value>true</value> </property> <property> <!-- \u77ed\u8def\u8bfb\u5f00\u5173\uff0c\u5728\u7f13\u5b58\u7ed3\u70b9\u4e0e\u5ba2\u6237\u7aef\u5728\u540c\u4e00\u53f0\u673a\u5668\u4e0a\u7684\u65f6\u5019\u53ef\u4ee5\u76f4\u63a5\u8bfb\u672c\u5730\u6587\u4ef6\uff0c\u8981\u6c42\u6253\u5f00\u672c\u5730 storageservice \u8fde\u63a5 --> <name>fs.jindofsx.short.circuit.enable</name> <value>true</value> </property> </configuration>","title":"\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570"},{"location":"backup/jindofsx/configuration/jindosdk_configuration_list/#rpc","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- brpc\u7ebf\u7a0b\u6c60\u5927\u5c0f --> <name>fs.jindofsx.rpc.backup.threads</name> <value>100</value> </property> <property> <!-- \u5e76\u53d1\u7ebf\u7a0b\u6570\u91cf --> <name>fs.jindofsx.rpc.thread.concurrency</name> <value>10</value> </property> <property> <!-- \u6d88\u606f\u4f53\u5927\u5c0f --> <name>fs.jindofsx.rpc.body.size</name> <value>256 * 1024 * 1024</value> </property> <property> <!-- \u8d85\u65f6\u65f6\u95f4\u5927\u5c0f\uff08\u6beb\u79d2\uff09--> <name>fs.jindofsx.rpc.timeout</name> <value>12 * 60 * 60 * 1000</value> </property> <property> <!-- \u6d88\u606f\u961f\u5217\u5927\u5c0f --> <name>fs.jindofsx.rpc.max.unwritten.size</name> <value>256 * 1024 * 1024</value> </property> </configuration>","title":"RPC \u4f18\u5316\u76f8\u5173\u53c2\u6570"},{"location":"backup/jindofsx/configuration/jindosdk_configuration_list/#metrics","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u5ba2\u6237\u7aefmetrics\u5f00\u5173 --> <name>fs.jindofsx.client.metrics.enable</name> <value>false</value> </property> </configuration> \u3010\u6ce8\uff1a4.3.0 \u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u3011","title":"\u7f13\u5b58 Metrics \u76f8\u5173\u53c2\u6570"},{"location":"backup/jindofsx/configuration/jindosdk_configuration_list_ini/","text":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e \u00a7 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 jindosdk.cfg \u4e2d\u3002 # \u9009\u62e9 jindofsx \u4f5c\u4e3a xengine \u5b9e\u73b0 fs.xengine=jindofsx # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable=false # \u5143\u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.meta.cache.enable=false # \u5c0f\u6587\u4ef6\u7f13\u5b58\u4f18\u5316\u5f00\u5173 fs.jindofsx.slice.cache.enable=false # \u5185\u5b58\u7f13\u5b58\u5f00\u5173 fs.jindofsx.ram.cache.enable=false # \u662f\u5426\u76f4\u63a5\u8fde\u63a5\u672c\u5730 storageservice fs.jindofsx.storage.connect.enable=true # \u77ed\u8def\u8bfb\u5f00\u5173\uff0c\u5728\u7f13\u5b58\u7ed3\u70b9\u4e0e\u5ba2\u6237\u7aef\u5728\u540c\u4e00\u53f0\u673a\u5668\u4e0a\u7684\u65f6\u5019\u53ef\u4ee5\u76f4\u63a5\u8bfb\u672c\u5730\u6587\u4ef6\uff0c\u8981\u6c42\u6253\u5f00\u672c\u5730 storageservice \u8fde\u63a5 fs.jindofsx.short.circuit.enable=true RPC \u4f18\u5316\u76f8\u5173\u53c2\u6570 \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 jindosdk.cfg \u4e2d\u3002 # brpc\u7ebf\u7a0b\u6c60\u5927\u5c0f fs.jindofsx.rpc.backup.threads=100 # \u5e76\u53d1\u7ebf\u7a0b\u6570\u91cf fs.jindofsx.rpc.thread.concurrency=10 # \u6d88\u606f\u4f53\u5927\u5c0f fs.jindofsx.rpc.body.size=268435456 # \u8d85\u65f6\u65f6\u95f4\u5927\u5c0f\uff08\u6beb\u79d2\uff09 fs.jindofsx.rpc.timeout=43200000 # \u6d88\u606f\u961f\u5217\u5927\u5c0f fs.jindofsx.rpc.max.unwritten.size=268435456 \u7f13\u5b58 Metrics \u76f8\u5173\u53c2\u6570 \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 jindosdk.cfg \u4e2d\u3002 # \u5ba2\u6237\u7aefmetrics\u5f00\u5173 fs.jindofsx.client.metrics.enable=false \u3010\u6ce8\uff1a4.3.0 \u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u3011","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/jindofsx/configuration/jindosdk_configuration_list_ini/#jindosdk","text":"","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/jindofsx/configuration/jindosdk_configuration_list_ini/#_1","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 jindosdk.cfg \u4e2d\u3002 # \u9009\u62e9 jindofsx \u4f5c\u4e3a xengine \u5b9e\u73b0 fs.xengine=jindofsx # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable=false # \u5143\u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.meta.cache.enable=false # \u5c0f\u6587\u4ef6\u7f13\u5b58\u4f18\u5316\u5f00\u5173 fs.jindofsx.slice.cache.enable=false # \u5185\u5b58\u7f13\u5b58\u5f00\u5173 fs.jindofsx.ram.cache.enable=false # \u662f\u5426\u76f4\u63a5\u8fde\u63a5\u672c\u5730 storageservice fs.jindofsx.storage.connect.enable=true # \u77ed\u8def\u8bfb\u5f00\u5173\uff0c\u5728\u7f13\u5b58\u7ed3\u70b9\u4e0e\u5ba2\u6237\u7aef\u5728\u540c\u4e00\u53f0\u673a\u5668\u4e0a\u7684\u65f6\u5019\u53ef\u4ee5\u76f4\u63a5\u8bfb\u672c\u5730\u6587\u4ef6\uff0c\u8981\u6c42\u6253\u5f00\u672c\u5730 storageservice \u8fde\u63a5 fs.jindofsx.short.circuit.enable=true","title":"\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570"},{"location":"backup/jindofsx/configuration/jindosdk_configuration_list_ini/#rpc","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 jindosdk.cfg \u4e2d\u3002 # brpc\u7ebf\u7a0b\u6c60\u5927\u5c0f fs.jindofsx.rpc.backup.threads=100 # \u5e76\u53d1\u7ebf\u7a0b\u6570\u91cf fs.jindofsx.rpc.thread.concurrency=10 # \u6d88\u606f\u4f53\u5927\u5c0f fs.jindofsx.rpc.body.size=268435456 # \u8d85\u65f6\u65f6\u95f4\u5927\u5c0f\uff08\u6beb\u79d2\uff09 fs.jindofsx.rpc.timeout=43200000 # \u6d88\u606f\u961f\u5217\u5927\u5c0f fs.jindofsx.rpc.max.unwritten.size=268435456","title":"RPC \u4f18\u5316\u76f8\u5173\u53c2\u6570"},{"location":"backup/jindofsx/configuration/jindosdk_configuration_list_ini/#metrics","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 jindosdk.cfg \u4e2d\u3002 # \u5ba2\u6237\u7aefmetrics\u5f00\u5173 fs.jindofsx.client.metrics.enable=false \u3010\u6ce8\uff1a4.3.0 \u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u3011","title":"\u7f13\u5b58 Metrics \u76f8\u5173\u53c2\u6570"},{"location":"backup/jindofsx/deploy/deploy_jindofsx/","text":"\u5feb\u901f\u90e8\u7f72\u4e00\u4e2a\u7b80\u5355\u7684 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u00a7 1. \u4e0b\u8f7d JindoFSx \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 Release \u5305 jindofsx-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u89e3\u538b \u00a7 tar -xzvf jindofsx-x.x.x.tar.gz \u5e76\u5c06\u5b89\u88c5\u76ee\u5f55\u90e8\u7f72\u5230\u6240\u6709\u6240\u9700\u8282\u70b9\u4e0a\u3002 3. \u914d\u7f6e jindofsx.cfg \u00a7 \u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002\u5e76\u5c06\u914d\u7f6e\u6587\u4ef6\u90e8\u7f72\u5230\u6240\u9700\u8282\u70b9\u4e0a\uff08NamespaceService \u548c StorageService \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] logger.jnilogger=false # jni log \u5f00\u5173 metric.report.on = true # metrics \u5f00\u5173 logger.dir = /tmp/jindofsx-log # \u65e5\u5fd7\u76ee\u5f55 [jindofsx-storage]# Storage service \u914d\u7f6e storage.rpc.port=6101 # Storage service \u7aef\u53e3 storage.data-dirs = /mnt/disk1/jindofsx,/mnt/disk2/jindofsx,/mnt/disk3/jindofsx,/mnt/disk4/jindofsx # \u7f13\u5b58\u78c1\u76d8\u76ee\u5f55 storage.data-dirs.capacities = 527371075584,527371075584,527371075584,527371075584 # \u7f13\u5b58\u78c1\u76d8\u5927\u5c0f storage.namespace.rpc.address = emr-header-1:8101 # Namespace service \u5730\u5740\u53ca\u7aef\u53e3 storage.watermark.high.ratio=0.4 # \u7f13\u5b58\u4e0a\u6c34\u4f4d storage.watermark.low.ratio=0.2 # \u7f13\u5b58\u4e0b\u6c34\u4f4d [jindofsx-namespace]# Namespace service \u914d\u7f6e namespace.rpc.port = 8101 # Namespace service \u76d1\u542c\u7aef\u53e3 namespace.meta-dir = /tmp/jindofsx/server # \u5143\u6570\u636e\u5b58\u653e\u76ee\u5f55 \u5fc5\u8981\u914d\u7f6e\u8bf4\u660e\uff1a 1. [jindofsx-storage] \u4e2d\u7684\u53c2\u6570 storage.namespace.rpc.address \u4fee\u6539\u4e3a JindoFSx \u96c6\u7fa4\u7684 NamespaceService \u670d\u52a1\u7684\u6240\u5728\u8282\u70b9\u5730\u5740\uff08hostname \u6216\u8005 ip \u5747\u53ef\uff09\uff0c\u5176\u4e2d 8101 \u4e3a NamespaceService \u670d\u52a1 RPC \u7684\u7aef\u53e3\u53f7\uff0c \u9700\u8981\u4e0e [jindofsx-namespace] \u4e2d\u7684\u53c2\u6570 namespace.rpc.port \u4e2d\u7684\u7aef\u53e3\u53f7\u4e00\u81f4\uff1b 2. [jindofsx-storage] \u4e2d\u9700\u914d\u7f6e\u597d\u6570\u636e\u7f13\u5b58\u76f8\u5173\u53c2\u6570\uff0cstorage.data-dirs \u4e3a\u7f13\u5b58\u76ee\u5f55\uff0c\u53ef\u4ee5\u6307\u5b9a\u591a\u4e2a\uff0c\u4ee5\u9017\u53f7\u9694\u5f00\uff1bstorage.data-dirs.capacities \u4e3a\u6bcf\u5757\u78c1\u76d8\u5927\u5c0f\uff0c\u6570\u91cf\u5e94\u4e0e storage.data-dirs \u4e00\u81f4\uff1bstorage.watermark.high.ratio \u8868\u793a\u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0a\u6c34\u4f4d\u6bd4\u4f8b\uff080\u52301\u4e4b\u95f4\u7684\u5c0f\u6570\uff09\uff0c\u6bcf\u5757\u6570\u636e\u76d8\u7684\u7f13\u5b58\u6570\u636e\u76ee\u5f55\u5360\u7528\u7684\u78c1\u76d8\u7a7a\u95f4\u5230\u8fbe\u4e0a\u6c34\u4f4d\u5373\u4f1a\u89e6\u53d1\u6e05\u7406\uff0c\u5e76\u4e14 StorageService \u4f1a\u786e\u4fdd\u7f13\u5b58\u7a7a\u95f4\u5360\u7528\u4e0d\u8d85\u8fc7\u4e0a\u6c34\u4f4d\uff1bstorage.watermark.low.ratio \u8868\u793a\u4f7f\u7528\u91cf\u7684\u4e0b\u6c34\u4f4d\u6bd4\u4f8b\uff080\u52301\u4e4b\u95f4\u7684\u5c0f\u6570\uff0c\u9700\u5c0f\u4e8e\u4e0a\u6c34\u4f4d\uff09\uff0c\u89e6\u53d1\u6e05\u7406\u540e\u4f1a\u81ea\u52a8\u6e05\u7406\u51b7\u6570\u636e\uff0c\u5c06\u7f13\u5b58\u6570\u636e\u76ee\u5f55\u5360\u7528\u7a7a\u95f4\u6e05\u7406\u5230\u4e0b\u6c34\u4f4d\u3002 3. [jindofsx-common] \u4e2d\u9700\u914d\u7f6e\u65e5\u5fd7\u3001metrics\u3001credentials\u3001endpoint \u76f8\u5173\u4fe1\u606f\uff0c\u52a0\u901f\u5e95\u5c42\u5b58\u50a8\u6240\u9700\u8981\u7684 credentials \u548c endpoint \u914d\u7f6e\u65b9\u6cd5\u89c1\u76f8\u5e94 credential provider \u914d\u7f6e\u6587\u6863 \u3002 4. \u914d\u7f6e\u73af\u5883\u53d8\u91cf \u00a7 \u4ee5 jindofsx-4.6.12 \u5b89\u88c5\u5728 /usr/lib \u4e3a\u4f8b\uff1a export JINDOFSX_CONF_DIR=/usr/lib/jindofsx-4.6.12/conf/ \u5e76\u786e\u8ba4\u90e8\u7f72\u5230\u6240\u6709\u8282\u70b9\u3002 5. \u542f\u52a8 JindoFSx \u7f13\u5b58\u670d\u52a1 \u00a7 \u5728 master \u8282\u70b9\uff08\u542f\u52a8 JindoFSx NamespaceService \u7684\u8282\u70b9\uff09\u7f16\u8f91 sbin/nodes \u6587\u4ef6\uff0c\u914d\u7f6e\u4e0a\u6240\u6709\u9700\u8981\u542f\u52a8 StorageService \u7684\u8282\u70b9\u5217\u8868\uff0c\u4f8b\u5982\uff1a worker-1 worker-2 worker-3 \u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\uff08\u6ce8\u610f\uff1a\u7531\u4e8e\u811a\u672c\u901a\u8fc7 pssh \u547d\u4ee4\u6279\u91cf\u6267\u884c\u8fdc\u7a0b\u8282\u70b9\u7684\u670d\u52a1\u542f\u505c\uff0c\u9700\u8981\u4fdd\u8bc1 master \u8282\u70b9\u5230\u6240\u6709\u5176\u4ed6\u8282\u70b9 ssh \u7684\u514d\u5bc6\u8fde\u901a\uff09 cd jindofsx-x.x.x sh sbin/start-service.sh \u6267\u884c\u5b8c\u6210\u540e\uff0cmaster \u8282\u70b9\u5c06\u4f1a\u542f\u52a8 Namespace Service\uff0cnodes \u914d\u7f6e\u7684\u6240\u6709\u8282\u70b9\u5c06\u4f1a\u542f\u52a8 StorageService\u3002 \u53e6\u5916\uff0c\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u547d\u4ee4\u505c\u6b62\u6240\u6709\u7f13\u5b58\u670d\u52a1\uff1a cd jindofsx-x.x.x sh sbin/stop-service.sh 6. \u7f13\u5b58\u670d\u52a1\u72b6\u6001\u68c0\u67e5 \u00a7 \u7f13\u5b58\u670d\u52a1\u7684\u8fdb\u7a0b\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u547d\u4ee4\u67e5\u770b\uff0c\u65e5\u5fd7\u6587\u4ef6\u53ef\u5728 logger.dir \u914d\u7f6e\u7684\u76ee\u5f55\u4e0b\u67e5\u770b ps -aux | grep jindofsx-namespaceservice ps -aux | grep jindofsx-storageservice \u5982\u9700\u90e8\u7f72\u9ad8\u53ef\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf, \u8bf7\u53c2\u8003 \u90e8\u7f72\u9ad8\u53ef\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf","title":"\u5feb\u901f\u90e8\u7f72\u4e00\u4e2a\u7b80\u5355\u7684 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf"},{"location":"backup/jindofsx/deploy/deploy_jindofsx/#jindofsx","text":"","title":"\u5feb\u901f\u90e8\u7f72\u4e00\u4e2a\u7b80\u5355\u7684 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf"},{"location":"backup/jindofsx/deploy/deploy_jindofsx/#1-jindofsx","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 Release \u5305 jindofsx-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoFSx \u5305"},{"location":"backup/jindofsx/deploy/deploy_jindofsx/#2","text":"tar -xzvf jindofsx-x.x.x.tar.gz \u5e76\u5c06\u5b89\u88c5\u76ee\u5f55\u90e8\u7f72\u5230\u6240\u6709\u6240\u9700\u8282\u70b9\u4e0a\u3002","title":"2. \u89e3\u538b"},{"location":"backup/jindofsx/deploy/deploy_jindofsx/#3-jindofsxcfg","text":"\u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002\u5e76\u5c06\u914d\u7f6e\u6587\u4ef6\u90e8\u7f72\u5230\u6240\u9700\u8282\u70b9\u4e0a\uff08NamespaceService \u548c StorageService \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] logger.jnilogger=false # jni log \u5f00\u5173 metric.report.on = true # metrics \u5f00\u5173 logger.dir = /tmp/jindofsx-log # \u65e5\u5fd7\u76ee\u5f55 [jindofsx-storage]# Storage service \u914d\u7f6e storage.rpc.port=6101 # Storage service \u7aef\u53e3 storage.data-dirs = /mnt/disk1/jindofsx,/mnt/disk2/jindofsx,/mnt/disk3/jindofsx,/mnt/disk4/jindofsx # \u7f13\u5b58\u78c1\u76d8\u76ee\u5f55 storage.data-dirs.capacities = 527371075584,527371075584,527371075584,527371075584 # \u7f13\u5b58\u78c1\u76d8\u5927\u5c0f storage.namespace.rpc.address = emr-header-1:8101 # Namespace service \u5730\u5740\u53ca\u7aef\u53e3 storage.watermark.high.ratio=0.4 # \u7f13\u5b58\u4e0a\u6c34\u4f4d storage.watermark.low.ratio=0.2 # \u7f13\u5b58\u4e0b\u6c34\u4f4d [jindofsx-namespace]# Namespace service \u914d\u7f6e namespace.rpc.port = 8101 # Namespace service \u76d1\u542c\u7aef\u53e3 namespace.meta-dir = /tmp/jindofsx/server # \u5143\u6570\u636e\u5b58\u653e\u76ee\u5f55 \u5fc5\u8981\u914d\u7f6e\u8bf4\u660e\uff1a 1. [jindofsx-storage] \u4e2d\u7684\u53c2\u6570 storage.namespace.rpc.address \u4fee\u6539\u4e3a JindoFSx \u96c6\u7fa4\u7684 NamespaceService \u670d\u52a1\u7684\u6240\u5728\u8282\u70b9\u5730\u5740\uff08hostname \u6216\u8005 ip \u5747\u53ef\uff09\uff0c\u5176\u4e2d 8101 \u4e3a NamespaceService \u670d\u52a1 RPC \u7684\u7aef\u53e3\u53f7\uff0c \u9700\u8981\u4e0e [jindofsx-namespace] \u4e2d\u7684\u53c2\u6570 namespace.rpc.port \u4e2d\u7684\u7aef\u53e3\u53f7\u4e00\u81f4\uff1b 2. [jindofsx-storage] \u4e2d\u9700\u914d\u7f6e\u597d\u6570\u636e\u7f13\u5b58\u76f8\u5173\u53c2\u6570\uff0cstorage.data-dirs \u4e3a\u7f13\u5b58\u76ee\u5f55\uff0c\u53ef\u4ee5\u6307\u5b9a\u591a\u4e2a\uff0c\u4ee5\u9017\u53f7\u9694\u5f00\uff1bstorage.data-dirs.capacities \u4e3a\u6bcf\u5757\u78c1\u76d8\u5927\u5c0f\uff0c\u6570\u91cf\u5e94\u4e0e storage.data-dirs \u4e00\u81f4\uff1bstorage.watermark.high.ratio \u8868\u793a\u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0a\u6c34\u4f4d\u6bd4\u4f8b\uff080\u52301\u4e4b\u95f4\u7684\u5c0f\u6570\uff09\uff0c\u6bcf\u5757\u6570\u636e\u76d8\u7684\u7f13\u5b58\u6570\u636e\u76ee\u5f55\u5360\u7528\u7684\u78c1\u76d8\u7a7a\u95f4\u5230\u8fbe\u4e0a\u6c34\u4f4d\u5373\u4f1a\u89e6\u53d1\u6e05\u7406\uff0c\u5e76\u4e14 StorageService \u4f1a\u786e\u4fdd\u7f13\u5b58\u7a7a\u95f4\u5360\u7528\u4e0d\u8d85\u8fc7\u4e0a\u6c34\u4f4d\uff1bstorage.watermark.low.ratio \u8868\u793a\u4f7f\u7528\u91cf\u7684\u4e0b\u6c34\u4f4d\u6bd4\u4f8b\uff080\u52301\u4e4b\u95f4\u7684\u5c0f\u6570\uff0c\u9700\u5c0f\u4e8e\u4e0a\u6c34\u4f4d\uff09\uff0c\u89e6\u53d1\u6e05\u7406\u540e\u4f1a\u81ea\u52a8\u6e05\u7406\u51b7\u6570\u636e\uff0c\u5c06\u7f13\u5b58\u6570\u636e\u76ee\u5f55\u5360\u7528\u7a7a\u95f4\u6e05\u7406\u5230\u4e0b\u6c34\u4f4d\u3002 3. [jindofsx-common] \u4e2d\u9700\u914d\u7f6e\u65e5\u5fd7\u3001metrics\u3001credentials\u3001endpoint \u76f8\u5173\u4fe1\u606f\uff0c\u52a0\u901f\u5e95\u5c42\u5b58\u50a8\u6240\u9700\u8981\u7684 credentials \u548c endpoint \u914d\u7f6e\u65b9\u6cd5\u89c1\u76f8\u5e94 credential provider \u914d\u7f6e\u6587\u6863 \u3002","title":"3. \u914d\u7f6e jindofsx.cfg"},{"location":"backup/jindofsx/deploy/deploy_jindofsx/#4","text":"\u4ee5 jindofsx-4.6.12 \u5b89\u88c5\u5728 /usr/lib \u4e3a\u4f8b\uff1a export JINDOFSX_CONF_DIR=/usr/lib/jindofsx-4.6.12/conf/ \u5e76\u786e\u8ba4\u90e8\u7f72\u5230\u6240\u6709\u8282\u70b9\u3002","title":"4. \u914d\u7f6e\u73af\u5883\u53d8\u91cf"},{"location":"backup/jindofsx/deploy/deploy_jindofsx/#5-jindofsx","text":"\u5728 master \u8282\u70b9\uff08\u542f\u52a8 JindoFSx NamespaceService \u7684\u8282\u70b9\uff09\u7f16\u8f91 sbin/nodes \u6587\u4ef6\uff0c\u914d\u7f6e\u4e0a\u6240\u6709\u9700\u8981\u542f\u52a8 StorageService \u7684\u8282\u70b9\u5217\u8868\uff0c\u4f8b\u5982\uff1a worker-1 worker-2 worker-3 \u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\uff08\u6ce8\u610f\uff1a\u7531\u4e8e\u811a\u672c\u901a\u8fc7 pssh \u547d\u4ee4\u6279\u91cf\u6267\u884c\u8fdc\u7a0b\u8282\u70b9\u7684\u670d\u52a1\u542f\u505c\uff0c\u9700\u8981\u4fdd\u8bc1 master \u8282\u70b9\u5230\u6240\u6709\u5176\u4ed6\u8282\u70b9 ssh \u7684\u514d\u5bc6\u8fde\u901a\uff09 cd jindofsx-x.x.x sh sbin/start-service.sh \u6267\u884c\u5b8c\u6210\u540e\uff0cmaster \u8282\u70b9\u5c06\u4f1a\u542f\u52a8 Namespace Service\uff0cnodes \u914d\u7f6e\u7684\u6240\u6709\u8282\u70b9\u5c06\u4f1a\u542f\u52a8 StorageService\u3002 \u53e6\u5916\uff0c\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u547d\u4ee4\u505c\u6b62\u6240\u6709\u7f13\u5b58\u670d\u52a1\uff1a cd jindofsx-x.x.x sh sbin/stop-service.sh","title":"5. \u542f\u52a8 JindoFSx \u7f13\u5b58\u670d\u52a1"},{"location":"backup/jindofsx/deploy/deploy_jindofsx/#6","text":"\u7f13\u5b58\u670d\u52a1\u7684\u8fdb\u7a0b\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u547d\u4ee4\u67e5\u770b\uff0c\u65e5\u5fd7\u6587\u4ef6\u53ef\u5728 logger.dir \u914d\u7f6e\u7684\u76ee\u5f55\u4e0b\u67e5\u770b ps -aux | grep jindofsx-namespaceservice ps -aux | grep jindofsx-storageservice \u5982\u9700\u90e8\u7f72\u9ad8\u53ef\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf, \u8bf7\u53c2\u8003 \u90e8\u7f72\u9ad8\u53ef\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf","title":"6. \u7f13\u5b58\u670d\u52a1\u72b6\u6001\u68c0\u67e5"},{"location":"backup/jindofsx/deploy/deploy_jindofsx_nsonly/","text":"\u5feb\u901f\u90e8\u7f72\u4e00\u4e2a\u7b80\u5355\u7684 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf(\u4ec5 Namespace) \u00a7 \u5f53\u65e0\u9700\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u65f6\uff0c\u53ef\u4ee5\u642d\u5efa\u4e00\u4e2a\u4ec5\u5305\u542b Namespace \u670d\u52a1\u7684 JindoFSx \u96c6\u7fa4\uff0c\u800c\u4e0d\u9700\u8981\u90e8\u7f72 Storage \u8fdb\u7a0b\u3002 1. \u4e0b\u8f7d JindoFSx \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 Release \u5305 jindofsx-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u89e3\u538b \u00a7 tar -xzvf jindofsx-x.x.x.tar.gz \u5e76\u5c06\u5b89\u88c5\u76ee\u5f55\u90e8\u7f72\u5230 master \u8282\u70b9\u4e0a\u3002 3. \u914d\u7f6e jindofsx.cfg \u00a7 \u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002\u5e76\u5c06\u914d\u7f6e\u6587\u4ef6\u90e8\u7f72\u5230 NamespaceService \u6240\u5728\u8282\u70b9\u3002 [jindofsx-common] logger.jnilogger=false # jni log \u5f00\u5173 metric.report.on = true # metrics \u5f00\u5173 logger.dir = /tmp/jindofsx-log # \u65e5\u5fd7\u76ee\u5f55 [jindofsx-namespace]# Namespace service \u914d\u7f6e namespace.rpc.port = 8101 # Namespace service \u76d1\u542c\u7aef\u53e3 namespace.meta-dir = /tmp/jindofsx/server # \u5143\u6570\u636e\u5b58\u653e\u76ee\u5f55 \u5fc5\u8981\u914d\u7f6e\u8bf4\u660e\uff1a 1. [jindofsx-common] \u4e2d\u9700\u914d\u7f6e\u65e5\u5fd7\u3001metrics\u3001credentials\u3001endpoint \u76f8\u5173\u4fe1\u606f\uff0c\u52a0\u901f\u5e95\u5c42\u5b58\u50a8\u6240\u9700\u8981\u7684 credentials \u548c endpoint \u914d\u7f6e\u65b9\u6cd5\u89c1\u76f8\u5e94 credential provider \u914d\u7f6e\u6587\u6863 \u3002 4. \u914d\u7f6e\u73af\u5883\u53d8\u91cf \u00a7 \u4ee5 jindofsx-4.6.12 \u5b89\u88c5\u5728 /usr/lib \u4e3a\u4f8b\uff1a export JINDOFSX_CONF_DIR=/usr/lib/jindofsx-4.6.12/conf/ \u5e76\u786e\u8ba4\u90e8\u7f72\u5230 Namespace \u670d\u52a1\u6240\u5728\u8282\u70b9\u3002 5. \u542f\u52a8 JindoFSx \u7f13\u5b58\u670d\u52a1 \u00a7 \u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\uff1a cd jindofsx-x.x.x sh sbin/start-service.sh \u6267\u884c\u5b8c\u6210\u540e\uff0cmaster \u8282\u70b9\u5c06\u4f1a\u542f\u52a8 Namespace Service\u3002 \u53e6\u5916\uff0c\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u547d\u4ee4\u505c\u6b62\u6240\u6709\u7f13\u5b58\u670d\u52a1\uff1a cd jindofsx-x.x.x sh sbin/stop-service.sh \u5982\u9700\u90e8\u7f72\u4ec5 Namespace \u7684\u9ad8\u53ef\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf, \u8bf7\u53c2\u8003 \u90e8\u7f72\u9ad8\u53ef\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 6. \u5ba2\u6237\u7aef\u914d\u7f6e \u00a7 \u914d\u7f6e\u5728\u6240\u6709\u8282\u70b9\u7684 Hadoop \u914d\u7f6e\u6587\u4ef6\u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Client\u7aef\u8bbf\u95ee\u7684Namespace\u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>hostname:8101</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u6ce8\uff1a fs.jindofsx.namespace.rpc.address \u4e2d\u7684 hostname \u4e3a Namespace \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\u5730\u5740\u3002 \u53ef\u4ee5\u7528\u5982\u4e0b\u547d\u4ee4\u9a8c\u8bc1 JindoFSx \u670d\u52a1\u53ef\u7528\uff1a jindo admin -reportMetrics","title":"\u5feb\u901f\u90e8\u7f72\u4e00\u4e2a\u7b80\u5355\u7684 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf(\u4ec5 Namespace)"},{"location":"backup/jindofsx/deploy/deploy_jindofsx_nsonly/#jindofsx-namespace","text":"\u5f53\u65e0\u9700\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u65f6\uff0c\u53ef\u4ee5\u642d\u5efa\u4e00\u4e2a\u4ec5\u5305\u542b Namespace \u670d\u52a1\u7684 JindoFSx \u96c6\u7fa4\uff0c\u800c\u4e0d\u9700\u8981\u90e8\u7f72 Storage \u8fdb\u7a0b\u3002","title":"\u5feb\u901f\u90e8\u7f72\u4e00\u4e2a\u7b80\u5355\u7684 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf(\u4ec5 Namespace)"},{"location":"backup/jindofsx/deploy/deploy_jindofsx_nsonly/#1-jindofsx","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 Release \u5305 jindofsx-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoFSx \u5305"},{"location":"backup/jindofsx/deploy/deploy_jindofsx_nsonly/#2","text":"tar -xzvf jindofsx-x.x.x.tar.gz \u5e76\u5c06\u5b89\u88c5\u76ee\u5f55\u90e8\u7f72\u5230 master \u8282\u70b9\u4e0a\u3002","title":"2. \u89e3\u538b"},{"location":"backup/jindofsx/deploy/deploy_jindofsx_nsonly/#3-jindofsxcfg","text":"\u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002\u5e76\u5c06\u914d\u7f6e\u6587\u4ef6\u90e8\u7f72\u5230 NamespaceService \u6240\u5728\u8282\u70b9\u3002 [jindofsx-common] logger.jnilogger=false # jni log \u5f00\u5173 metric.report.on = true # metrics \u5f00\u5173 logger.dir = /tmp/jindofsx-log # \u65e5\u5fd7\u76ee\u5f55 [jindofsx-namespace]# Namespace service \u914d\u7f6e namespace.rpc.port = 8101 # Namespace service \u76d1\u542c\u7aef\u53e3 namespace.meta-dir = /tmp/jindofsx/server # \u5143\u6570\u636e\u5b58\u653e\u76ee\u5f55 \u5fc5\u8981\u914d\u7f6e\u8bf4\u660e\uff1a 1. [jindofsx-common] \u4e2d\u9700\u914d\u7f6e\u65e5\u5fd7\u3001metrics\u3001credentials\u3001endpoint \u76f8\u5173\u4fe1\u606f\uff0c\u52a0\u901f\u5e95\u5c42\u5b58\u50a8\u6240\u9700\u8981\u7684 credentials \u548c endpoint \u914d\u7f6e\u65b9\u6cd5\u89c1\u76f8\u5e94 credential provider \u914d\u7f6e\u6587\u6863 \u3002","title":"3. \u914d\u7f6e jindofsx.cfg"},{"location":"backup/jindofsx/deploy/deploy_jindofsx_nsonly/#4","text":"\u4ee5 jindofsx-4.6.12 \u5b89\u88c5\u5728 /usr/lib \u4e3a\u4f8b\uff1a export JINDOFSX_CONF_DIR=/usr/lib/jindofsx-4.6.12/conf/ \u5e76\u786e\u8ba4\u90e8\u7f72\u5230 Namespace \u670d\u52a1\u6240\u5728\u8282\u70b9\u3002","title":"4. \u914d\u7f6e\u73af\u5883\u53d8\u91cf"},{"location":"backup/jindofsx/deploy/deploy_jindofsx_nsonly/#5-jindofsx","text":"\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\uff1a cd jindofsx-x.x.x sh sbin/start-service.sh \u6267\u884c\u5b8c\u6210\u540e\uff0cmaster \u8282\u70b9\u5c06\u4f1a\u542f\u52a8 Namespace Service\u3002 \u53e6\u5916\uff0c\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u547d\u4ee4\u505c\u6b62\u6240\u6709\u7f13\u5b58\u670d\u52a1\uff1a cd jindofsx-x.x.x sh sbin/stop-service.sh \u5982\u9700\u90e8\u7f72\u4ec5 Namespace \u7684\u9ad8\u53ef\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf, \u8bf7\u53c2\u8003 \u90e8\u7f72\u9ad8\u53ef\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf","title":"5. \u542f\u52a8 JindoFSx \u7f13\u5b58\u670d\u52a1"},{"location":"backup/jindofsx/deploy/deploy_jindofsx_nsonly/#6","text":"\u914d\u7f6e\u5728\u6240\u6709\u8282\u70b9\u7684 Hadoop \u914d\u7f6e\u6587\u4ef6\u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Client\u7aef\u8bbf\u95ee\u7684Namespace\u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>hostname:8101</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u6ce8\uff1a fs.jindofsx.namespace.rpc.address \u4e2d\u7684 hostname \u4e3a Namespace \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\u5730\u5740\u3002 \u53ef\u4ee5\u7528\u5982\u4e0b\u547d\u4ee4\u9a8c\u8bc1 JindoFSx \u670d\u52a1\u53ef\u7528\uff1a jindo admin -reportMetrics","title":"6. \u5ba2\u6237\u7aef\u914d\u7f6e"},{"location":"backup/jindofsx/deploy/deploy_jindosdk/","text":"\u90e8\u7f72 JindoSDK \u00a7 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 Release \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u89e3\u538b \u00a7 tar -xzvf jindosdk-x.x.x.tar.gz \u5e76\u5c06 JindoSDK \u5b89\u88c5\u5305\u90e8\u7f72\u5230\u6240\u6709\u8282\u70b9\u3002 3. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ \u914d\u7f6e core-site.xml \u914d\u7f6e\u5728\u6240\u6709\u8282\u70b9\u7684 Hadoop \u914d\u7f6e\u6587\u4ef6\u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Client\u7aef\u8bbf\u95ee\u7684Namespace\u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u5e76\u5c06\u73af\u5883\u53d8\u91cf\u914d\u7f6e\u66f4\u65b0\u5230\u6240\u6709\u8282\u70b9\u3002","title":"\u90e8\u7f72 JindoSDK"},{"location":"backup/jindofsx/deploy/deploy_jindosdk/#jindosdk","text":"","title":"\u90e8\u7f72 JindoSDK"},{"location":"backup/jindofsx/deploy/deploy_jindosdk/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 Release \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindofsx/deploy/deploy_jindosdk/#2","text":"tar -xzvf jindosdk-x.x.x.tar.gz \u5e76\u5c06 JindoSDK \u5b89\u88c5\u5305\u90e8\u7f72\u5230\u6240\u6709\u8282\u70b9\u3002","title":"2. \u89e3\u538b"},{"location":"backup/jindofsx/deploy/deploy_jindosdk/#3-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ \u914d\u7f6e core-site.xml \u914d\u7f6e\u5728\u6240\u6709\u8282\u70b9\u7684 Hadoop \u914d\u7f6e\u6587\u4ef6\u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Client\u7aef\u8bbf\u95ee\u7684Namespace\u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u5e76\u5c06\u73af\u5883\u53d8\u91cf\u914d\u7f6e\u66f4\u65b0\u5230\u6240\u6709\u8282\u70b9\u3002","title":"3. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofsx/deploy/deploy_raft_ns/","text":"\u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u00a7 \u670d\u52a1\u7aef\u914d\u7f6e\u9ad8\u53ef\u7528 JindoFSx Namespace \u00a7 JindoFSx Namespace \u9ad8\u53ef\u7528\u90e8\u7f72\u9700\u8981 3 \u4e2a\u8282\u70b9\u5206\u522b\u542f\u52a8 NamespaceService \u670d\u52a1\u7ec4\u6210 1 \u4e2a Raft \u5b9e\u4f8b\u3002 1. \u6682\u505c JindoFSx \u670d\u52a1 \u00a7 cd jindofsx-x.x.x sh sbin/stop-service.sh 2. \u914d\u7f6e\u672c\u5730 raft \u540e\u7aef \u00a7 \u53d8\u66f4\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg \u6587\u4ef6\u4e2d [jindofsx-namespace] section \u4e0b\u7684\u914d\u7f6e\u9879\u3002 [jindofsx-namespace]# Namespace \u914d\u7f6e namespace.backend.type = raft #\u8bbe\u7f6enamespace\u540e\u7aef\u5b58\u50a8\u7c7b\u578b\uff0c\u652f\u6301 rocksdb \u548c raft, \u9ed8\u8ba4\u4e3a rocksdb\u3002 namespace.backend.raft.initial-conf = master-1-1:8103:0,master-1-2:8103:0,master-1-3:8103:0 #\u90e8\u7f72raft\u5b9e\u4f8b\u76843\u4e2aNamespace\u5730\u5740 3. \u914d\u7f6e Namespace \u5730\u5740 \u00a7 \u56e0\u4e3a 3 \u4e2a Master \u8282\u70b9\u7ec4\u6210 1 \u4e2a Raft \u5b9e\u4f8b\uff0c\u9700\u66f4\u65b0 jindofsx-x.x.x/sbin/headers \u6587\u4ef6\uff0c\u6dfb\u52a0 3 \u4e2a Namespace \u6240\u5728\u8282\u70b9\u5217\u8868\uff0c\u4f8b\u5982\uff1a master-1-1 master-1-2 master-1-3 4. \u540c\u6b65 jindofsx-x.x.x \u6587\u4ef6\u5939 \u00a7 \u5b8c\u6210\u4e0a\u8ff0\u914d\u7f6e\u540e\uff0c\u5c06 jindofsx-x.x.x \u540c\u6b65\u81f3\u53e6\u4e24\u53f0 Namespace \u8282\u70b9\u7684\u540c\u4e00\u76ee\u5f55\u3002 5. \u542f\u52a8 JindoFSx \u670d\u52a1 \u00a7 cd jindofsx-x.x.x sh sbin/start-service.sh \u5ba2\u6237\u7aef\u914d\u7f6e \u00a7 \u914d\u7f6e\u5728\u6240\u6709\u8282\u70b9\u7684 Hadoop \u914d\u7f6e\u6587\u4ef6\u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Client\u7aef\u8bbf\u95eeraft\u5b9e\u4f8b\u76843\u4e2aNamespace\u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>master-1-1:8101,master-1-2:8101,master-1-3:8101</value> </property> </configuration> \u6ce8\uff1a fs.jindofsx.namespace.rpc.address \u4e2d\u7684 hostname \u4e3a Namespace \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\u5730\u5740\u5217\u8868\uff0c \u4e0e namespace.backend.raft.initial-conf \u4e2d\u7684 hostname \u4e00\u81f4\u3002","title":"\u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528"},{"location":"backup/jindofsx/deploy/deploy_raft_ns/#jindofsx-namespace","text":"","title":"\u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528"},{"location":"backup/jindofsx/deploy/deploy_raft_ns/#jindofsx-namespace_1","text":"JindoFSx Namespace \u9ad8\u53ef\u7528\u90e8\u7f72\u9700\u8981 3 \u4e2a\u8282\u70b9\u5206\u522b\u542f\u52a8 NamespaceService \u670d\u52a1\u7ec4\u6210 1 \u4e2a Raft \u5b9e\u4f8b\u3002","title":"\u670d\u52a1\u7aef\u914d\u7f6e\u9ad8\u53ef\u7528 JindoFSx Namespace"},{"location":"backup/jindofsx/deploy/deploy_raft_ns/#1-jindofsx","text":"cd jindofsx-x.x.x sh sbin/stop-service.sh","title":"1. \u6682\u505c JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/deploy/deploy_raft_ns/#2-raft","text":"\u53d8\u66f4\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg \u6587\u4ef6\u4e2d [jindofsx-namespace] section \u4e0b\u7684\u914d\u7f6e\u9879\u3002 [jindofsx-namespace]# Namespace \u914d\u7f6e namespace.backend.type = raft #\u8bbe\u7f6enamespace\u540e\u7aef\u5b58\u50a8\u7c7b\u578b\uff0c\u652f\u6301 rocksdb \u548c raft, \u9ed8\u8ba4\u4e3a rocksdb\u3002 namespace.backend.raft.initial-conf = master-1-1:8103:0,master-1-2:8103:0,master-1-3:8103:0 #\u90e8\u7f72raft\u5b9e\u4f8b\u76843\u4e2aNamespace\u5730\u5740","title":"2. \u914d\u7f6e\u672c\u5730 raft \u540e\u7aef"},{"location":"backup/jindofsx/deploy/deploy_raft_ns/#3-namespace","text":"\u56e0\u4e3a 3 \u4e2a Master \u8282\u70b9\u7ec4\u6210 1 \u4e2a Raft \u5b9e\u4f8b\uff0c\u9700\u66f4\u65b0 jindofsx-x.x.x/sbin/headers \u6587\u4ef6\uff0c\u6dfb\u52a0 3 \u4e2a Namespace \u6240\u5728\u8282\u70b9\u5217\u8868\uff0c\u4f8b\u5982\uff1a master-1-1 master-1-2 master-1-3","title":"3. \u914d\u7f6e Namespace \u5730\u5740"},{"location":"backup/jindofsx/deploy/deploy_raft_ns/#4-jindofsx-xxx","text":"\u5b8c\u6210\u4e0a\u8ff0\u914d\u7f6e\u540e\uff0c\u5c06 jindofsx-x.x.x \u540c\u6b65\u81f3\u53e6\u4e24\u53f0 Namespace \u8282\u70b9\u7684\u540c\u4e00\u76ee\u5f55\u3002","title":"4. \u540c\u6b65 jindofsx-x.x.x \u6587\u4ef6\u5939"},{"location":"backup/jindofsx/deploy/deploy_raft_ns/#5-jindofsx","text":"cd jindofsx-x.x.x sh sbin/start-service.sh","title":"5. \u542f\u52a8 JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/deploy/deploy_raft_ns/#_1","text":"\u914d\u7f6e\u5728\u6240\u6709\u8282\u70b9\u7684 Hadoop \u914d\u7f6e\u6587\u4ef6\u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Client\u7aef\u8bbf\u95eeraft\u5b9e\u4f8b\u76843\u4e2aNamespace\u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>master-1-1:8101,master-1-2:8101,master-1-3:8101</value> </property> </configuration> \u6ce8\uff1a fs.jindofsx.namespace.rpc.address \u4e2d\u7684 hostname \u4e3a Namespace \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\u5730\u5740\u5217\u8868\uff0c \u4e0e namespace.backend.raft.initial-conf \u4e2d\u7684 hostname \u4e00\u81f4\u3002","title":"\u5ba2\u6237\u7aef\u914d\u7f6e"},{"location":"backup/jindofsx/deploy/deploy_raft_ns_emr/","text":"\u5728 EMR \u96c6\u7fa4\u4e2d\u914d\u7f6e\u9ad8\u53ef\u7528 JindoFSx Namespace \u00a7 \u670d\u52a1\u7aef\u914d\u7f6e\u9ad8\u53ef\u7528 JindoFSx Namespace \u00a7 JindoFSx Namespace \u9ad8\u53ef\u7528\u90e8\u7f72\u9700\u8981 3 \u4e2a\u8282\u70b9\u5206\u522b\u542f\u52a8 NamespaceService \u670d\u52a1\u7ec4\u6210 1 \u4e2a Raft \u5b9e\u4f8b\u3002 2. \u914d\u7f6e\u672c\u5730 raft \u540e\u7aef \u00a7 2.1 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u00a7 \u53c2\u6570 \u503c \u8bf4\u660e namespace.backend.type raft \u8bbe\u7f6enamespace\u540e\u7aef\u5b58\u50a8\u7c7b\u578b\uff0c\u652f\u6301 rocksdb \u548c raft, \u9ed8\u8ba4\u4e3a rocksdb\u3002 namespace.backend.raft.initial-conf master-1-1:8103:0,master-1-2:8103:0,master-1-3:8103:0 \u90e8\u7f72raft\u5b9e\u4f8b\u76843\u4e2aNamespace\u5730\u5740\u3002 2.2 \u91cd\u542f JindoFSx Namespace \u670d\u52a1\u3002 \u00a7 \u5ba2\u6237\u7aef\u914d\u7f6e \u00a7 \u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \uff0c \u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 <configuration> <property> <!-- Client\u7aef\u8bbf\u95eeraft\u5b9e\u4f8b\u76843\u4e2aNamespace\u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>master-1-1:8101,master-1-2:8101,master-1-3:8101</value> </property> </configuration> \u6ce8\uff1a fs.jindofsx.namespace.rpc.address \u4e2d\u7684 hostname \u4e3a Namespace \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\u5730\u5740\u5217\u8868\uff0c \u4e0e namespace.backend.raft.initial-conf \u4e2d\u7684 hostname \u4e00\u81f4\u3002","title":"\u5728 EMR \u96c6\u7fa4\u4e2d\u914d\u7f6e\u9ad8\u53ef\u7528 JindoFSx Namespace"},{"location":"backup/jindofsx/deploy/deploy_raft_ns_emr/#emr-jindofsx-namespace","text":"","title":"\u5728 EMR \u96c6\u7fa4\u4e2d\u914d\u7f6e\u9ad8\u53ef\u7528 JindoFSx Namespace"},{"location":"backup/jindofsx/deploy/deploy_raft_ns_emr/#jindofsx-namespace","text":"JindoFSx Namespace \u9ad8\u53ef\u7528\u90e8\u7f72\u9700\u8981 3 \u4e2a\u8282\u70b9\u5206\u522b\u542f\u52a8 NamespaceService \u670d\u52a1\u7ec4\u6210 1 \u4e2a Raft \u5b9e\u4f8b\u3002","title":"\u670d\u52a1\u7aef\u914d\u7f6e\u9ad8\u53ef\u7528 JindoFSx Namespace"},{"location":"backup/jindofsx/deploy/deploy_raft_ns_emr/#2-raft","text":"","title":"2. \u914d\u7f6e\u672c\u5730 raft \u540e\u7aef"},{"location":"backup/jindofsx/deploy/deploy_raft_ns_emr/#21-jindodata-namespace","text":"\u53c2\u6570 \u503c \u8bf4\u660e namespace.backend.type raft \u8bbe\u7f6enamespace\u540e\u7aef\u5b58\u50a8\u7c7b\u578b\uff0c\u652f\u6301 rocksdb \u548c raft, \u9ed8\u8ba4\u4e3a rocksdb\u3002 namespace.backend.raft.initial-conf master-1-1:8103:0,master-1-2:8103:0,master-1-3:8103:0 \u90e8\u7f72raft\u5b9e\u4f8b\u76843\u4e2aNamespace\u5730\u5740\u3002","title":"2.1 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9\u914d\u7f6e &gt; namespace &gt; \u65b0\u589e\u914d\u7f6e\u9879\uff0c\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002"},{"location":"backup/jindofsx/deploy/deploy_raft_ns_emr/#22-jindofsx-namespace","text":"","title":"2.2 \u91cd\u542f JindoFSx Namespace \u670d\u52a1\u3002"},{"location":"backup/jindofsx/deploy/deploy_raft_ns_emr/#_1","text":"\u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \uff0c \u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 <configuration> <property> <!-- Client\u7aef\u8bbf\u95eeraft\u5b9e\u4f8b\u76843\u4e2aNamespace\u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>master-1-1:8101,master-1-2:8101,master-1-3:8101</value> </property> </configuration> \u6ce8\uff1a fs.jindofsx.namespace.rpc.address \u4e2d\u7684 hostname \u4e3a Namespace \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\u5730\u5740\u5217\u8868\uff0c \u4e0e namespace.backend.raft.initial-conf \u4e2d\u7684 hostname \u4e00\u81f4\u3002","title":"\u5ba2\u6237\u7aef\u914d\u7f6e"},{"location":"backup/jindofsx/hadoop/hadoop_with_dls/","text":"Hadoop \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09+ JindoFSx \u900f\u660e\u52a0\u901f \u00a7 JindoSDK \u4e3a JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7cfb\u7edf\u63d0\u4f9b\u4e86 Apache Hadoop \u652f\u6301\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1 OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e Access Key \u5c06 OSS-HDFS \u670d\u52a1 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 JindoFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u6570\u636e\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u00a7 \u7528 Hadoop Shell \u8bbf\u95ee JindoFS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<Bucket>.<Endpoint>/ ls \u64cd\u4f5c hadoop fs -ls oss://<Bucket>.<Endpoint>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<Bucket>.<Endpoint>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<Bucket>.<Endpoint>/<path> \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"Hadoop \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09+ JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hadoop/hadoop_with_dls/#hadoop-oss-hdfs-jindofs-jindofsx","text":"JindoSDK \u4e3a JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7cfb\u7edf\u63d0\u4f9b\u4e86 Apache Hadoop \u652f\u6301\u3002","title":"Hadoop \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09+ JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hadoop/hadoop_with_dls/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/hadoop/hadoop_with_dls/#jindosdk","text":"\u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1 OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e Access Key \u5c06 OSS-HDFS \u670d\u52a1 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 JindoFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u6570\u636e\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/hadoop/hadoop_with_dls/#jindosdk-oss-hdfs","text":"\u7528 Hadoop Shell \u8bbf\u95ee JindoFS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<Bucket>.<Endpoint>/ ls \u64cd\u4f5c hadoop fs -ls oss://<Bucket>.<Endpoint>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<Bucket>.<Endpoint>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<Bucket>.<Endpoint>/<path>","title":"\u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS-HDFS \u670d\u52a1"},{"location":"backup/jindofsx/hadoop/hadoop_with_dls/#_2","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/hadoop/hadoop_with_jindo/","text":"Hadoop \u8bbf\u95ee JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e \u00a7 JindoSDK \u4e3a JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7cfb\u7edf\u63d0\u4f9b\u4e86 Apache Hadoop \u652f\u6301\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 AccessKey \u5c06 OSS \u6216 OSS-HDFS \u670d\u52a1 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint \u5c06 OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u6570\u636e\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55 \u00a7 \u6302\u8f7d\u547d\u4ee4 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/ \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1 \u00a7 \u7528 Hadoop Shell \u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> jindo://emr-header-1:8101/jindooss/ ls \u64cd\u4f5c hadoop fs -ls jindo://emr-header-1:8101/jindooss/ mkdir \u64cd\u4f5c hadoop fs -mkdir jindo://emr-header-1:8101/jindooss/<path> rm \u64cd\u4f5c hadoop fs -rm jindo://emr-header-1:8101/jindooss/<path> \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"Hadoop \u8bbf\u95ee JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/hadoop/hadoop_with_jindo/#hadoop-jindofsx","text":"JindoSDK \u4e3a JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7cfb\u7edf\u63d0\u4f9b\u4e86 Apache Hadoop \u652f\u6301\u3002","title":"Hadoop \u8bbf\u95ee JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/hadoop/hadoop_with_jindo/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/hadoop/hadoop_with_jindo/#jindosdk","text":"\u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 AccessKey \u5c06 OSS \u6216 OSS-HDFS \u670d\u52a1 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint \u5c06 OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u6570\u636e\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/hadoop/hadoop_with_jindo/#oss-oss-hdfs","text":"\u6302\u8f7d\u547d\u4ee4 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/","title":"\u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55"},{"location":"backup/jindofsx/hadoop/hadoop_with_jindo/#jindosdk-oss-oss-hdfs","text":"\u7528 Hadoop Shell \u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> jindo://emr-header-1:8101/jindooss/ ls \u64cd\u4f5c hadoop fs -ls jindo://emr-header-1:8101/jindooss/ mkdir \u64cd\u4f5c hadoop fs -mkdir jindo://emr-header-1:8101/jindooss/<path> rm \u64cd\u4f5c hadoop fs -rm jindo://emr-header-1:8101/jindooss/<path>","title":"\u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1"},{"location":"backup/jindofsx/hadoop/hadoop_with_jindo/#_2","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/hadoop/hadoop_with_oss/","text":"Hadoop \u8bbf\u95ee\u963f\u91cc\u4e91 OSS + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 JindoSDK \u4e3a JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u4e86 Apache Hadoop \u652f\u6301\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1 OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e Access Key \u5c06 OSS \u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u00a7 \u7528 Hadoop Shell \u8bbf\u95ee JindoFS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<Bucket>/ ls \u64cd\u4f5c hadoop fs -ls oss://<Bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<Bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<Bucket>/<path> \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"Hadoop \u8bbf\u95ee\u963f\u91cc\u4e91 OSS + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hadoop/hadoop_with_oss/#hadoop-oss-jindofsx","text":"JindoSDK \u4e3a JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u4e86 Apache Hadoop \u652f\u6301\u3002","title":"Hadoop \u8bbf\u95ee\u963f\u91cc\u4e91 OSS + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hadoop/hadoop_with_oss/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/hadoop/hadoop_with_oss/#jindosdk","text":"\u914d\u7f6e OSS \u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1 OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e Access Key \u5c06 OSS \u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/hadoop/hadoop_with_oss/#jindosdk-oss","text":"\u7528 Hadoop Shell \u8bbf\u95ee JindoFS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<Bucket>/ ls \u64cd\u4f5c hadoop fs -ls oss://<Bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<Bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<Bucket>/<path>","title":"\u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/jindofsx/hadoop/hadoop_with_oss/#_2","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_dls/","text":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002 OSS-HDFS \u670d\u52a1\u662f\u963f\u91cc\u4e91\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9 HDFS \u63a5\u53e3, JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u6216 OSS \u670d\u52a1\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\u5b9e\u73b0\u5b58\u7b97\u5206\u79bb\uff0c\u76f8\u5bf9\u4e8e\u672c\u5730HDFS\u5b58\u50a8\uff0c\u4f7f\u7528\u66f4\u52a0\u7075\u6d3b\uff0c\u51cf\u5c11\u8fd0\u7ef4\u6210\u672c\u3002 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u00a7 \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84 \u00a7 \u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u7684\u503c\u4e3a OSS \u5730\u5740\uff0c\u4fee\u6539\u53c2\u6570 hbase.wal.dir \u7684\u503c\u4e3a\u672c\u5730\u7684 HDFS \u5730\u5740\uff0c\u901a\u8fc7\u672c\u5730 HDFS \u96c6\u7fa4\u5b58\u50a8 WAL \u6587\u4ef6\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 OSS\u3002\u53c2\u6570\u503c\u4e3a oss://bucket.endpoint/hbase-root-dir hbase.wal.dir \u6307\u5b9a HBase \u7684 WAL \u5b58\u50a8\u76ee\u5f55\u5230\u672c\u5730 HDFS \u96c6\u7fa4\u3002 \u53c2\u6570\u503c\u4e3a\uff1a HA\u96c6\u7fa4\uff1a hdfs://${nameservice}/hbase \u975eHA\u96c6\u7fa4\uff1a hdfs://${namenode_rpc_address}:${namenode_rpc_port}/hbase","title":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_dls/#hbase-oss-hdfs-jindofs-jindofsx","text":"HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002 OSS-HDFS \u670d\u52a1\u662f\u963f\u91cc\u4e91\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9 HDFS \u63a5\u53e3, JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u6216 OSS \u670d\u52a1\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\u5b9e\u73b0\u5b58\u7b97\u5206\u79bb\uff0c\u76f8\u5bf9\u4e8e\u672c\u5730HDFS\u5b58\u50a8\uff0c\u4f7f\u7528\u66f4\u52a0\u7075\u6d3b\uff0c\u51cf\u5c11\u8fd0\u7ef4\u6210\u672c\u3002","title":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_dls/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_dls/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_dls/#3-oss-hdfs-access-key","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"3. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_dls/#4-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002","title":"4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_dls/#hbase","text":"\u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u7684\u503c\u4e3a OSS \u5730\u5740\uff0c\u4fee\u6539\u53c2\u6570 hbase.wal.dir \u7684\u503c\u4e3a\u672c\u5730\u7684 HDFS \u5730\u5740\uff0c\u901a\u8fc7\u672c\u5730 HDFS \u96c6\u7fa4\u5b58\u50a8 WAL \u6587\u4ef6\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 OSS\u3002\u53c2\u6570\u503c\u4e3a oss://bucket.endpoint/hbase-root-dir hbase.wal.dir \u6307\u5b9a HBase \u7684 WAL \u5b58\u50a8\u76ee\u5f55\u5230\u672c\u5730 HDFS \u96c6\u7fa4\u3002 \u53c2\u6570\u503c\u4e3a\uff1a HA\u96c6\u7fa4\uff1a hdfs://${nameservice}/hbase \u975eHA\u96c6\u7fa4\uff1a hdfs://${namenode_rpc_address}:${namenode_rpc_port}/hbase","title":"\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_jindo/","text":"HBase \u4f7f\u7528 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e \u00a7 HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002 OSS-HDFS \u670d\u52a1\u662f\u963f\u91cc\u4e91\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9 HDFS \u63a5\u53e3, JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\u5b9e\u73b0\u5b58\u7b97\u5206\u79bb\uff0c\u76f8\u5bf9\u4e8e\u672c\u5730HDFS\u5b58\u50a8\uff0c\u4f7f\u7528\u66f4\u52a0\u7075\u6d3b\uff0c\u51cf\u5c11\u8fd0\u7ef4\u6210\u672c\u3002 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Access Key \u5c06 OSS Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindodls oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindodls \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/ \u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84 \u00a7 \u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u7684\u503c\u4e3a FSX \u5730\u5740\uff0c\u4fee\u6539\u53c2\u6570 hbase.wal.dir \u7684\u503c\u4e3a\u672c\u5730\u7684 HDFS \u5730\u5740\uff0c\u901a\u8fc7\u672c\u5730 HDFS \u96c6\u7fa4\u5b58\u50a8 WAL \u6587\u4ef6\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 FSX\u3002\u53c2\u6570\u503c\u4e3a jindo://headerhost:8101/jindooss/hbase-root-dir hbase.wal.dir \u6307\u5b9a HBase \u7684 WAL \u5b58\u50a8\u76ee\u5f55\u5230 FSX \u6216\u672c\u5730 HDFS\u3002\u53c2\u6570\u503c\u4e3a\uff1a HA\u96c6\u7fa4\uff1a hdfs://${nameservice}/hbase \u975eHA\u96c6\u7fa4\uff1a hdfs://${namenode_rpc_address}:${namenode_rpc_port}/hbase","title":"HBase \u4f7f\u7528 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_jindo/#hbase-jindofsx","text":"HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002 OSS-HDFS \u670d\u52a1\u662f\u963f\u91cc\u4e91\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9 HDFS \u63a5\u53e3, JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\u5b9e\u73b0\u5b58\u7b97\u5206\u79bb\uff0c\u76f8\u5bf9\u4e8e\u672c\u5730HDFS\u5b58\u50a8\uff0c\u4f7f\u7528\u66f4\u52a0\u7075\u6d3b\uff0c\u51cf\u5c11\u8fd0\u7ef4\u6210\u672c\u3002","title":"HBase \u4f7f\u7528 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_jindo/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_jindo/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_jindo/#3-jindosdk","text":"\u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Access Key \u5c06 OSS Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindodls oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindodls \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_jindo/#hbase","text":"\u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u7684\u503c\u4e3a FSX \u5730\u5740\uff0c\u4fee\u6539\u53c2\u6570 hbase.wal.dir \u7684\u503c\u4e3a\u672c\u5730\u7684 HDFS \u5730\u5740\uff0c\u901a\u8fc7\u672c\u5730 HDFS \u96c6\u7fa4\u5b58\u50a8 WAL \u6587\u4ef6\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 FSX\u3002\u53c2\u6570\u503c\u4e3a jindo://headerhost:8101/jindooss/hbase-root-dir hbase.wal.dir \u6307\u5b9a HBase \u7684 WAL \u5b58\u50a8\u76ee\u5f55\u5230 FSX \u6216\u672c\u5730 HDFS\u3002\u53c2\u6570\u503c\u4e3a\uff1a HA\u96c6\u7fa4\uff1a hdfs://${nameservice}/hbase \u975eHA\u96c6\u7fa4\uff1a hdfs://${namenode_rpc_address}:${namenode_rpc_port}/hbase","title":"\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_oss/","text":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\uff0c\u4f46\u9700\u901a\u8fc7\u672c\u5730 HDFS \u96c6\u7fa4\u5b58\u50a8 WAL \u6587\u4ef6\u3002 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u5c06 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u3002 \u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84 \u00a7 \u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84\uff0c\u662f\u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u7684\u503c\u4e3a OSS \u5730\u5740\uff0c\u4fee\u6539\u53c2\u6570 hbase.wal.dir \u7684\u503c\u4e3a\u672c\u5730\u7684 HDFS \u5730\u5740\uff0c\u901a\u8fc7\u672c\u5730 HDFS \u96c6\u7fa4\u5b58\u50a8 WAL \u6587\u4ef6\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 OSS\u3002\u53c2\u6570\u503c\u4e3a oss://bucket/hbase-root-dir hbase.wal.dir \u6307\u5b9a HBase \u7684 WAL \u5b58\u50a8\u76ee\u5f55\u5230\u672c\u5730 HDFS \u96c6\u7fa4\u3002 \u53c2\u6570\u503c\u4e3a\uff1a HA\u96c6\u7fa4\uff1a hdfs://${nameservice}/hbase \u975eHA\u96c6\u7fa4\uff1a hdfs://${namenode_rpc_address}:${namenode_rpc_port}/hbase","title":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_oss/#hbase-oss-jindofsx","text":"HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\uff0c\u4f46\u9700\u901a\u8fc7\u672c\u5730 HDFS \u96c6\u7fa4\u5b58\u50a8 WAL \u6587\u4ef6\u3002","title":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_oss/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_oss/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_oss/#3-oss-access-key","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u5c06 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u3002","title":"3. \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_oss/#hbase","text":"\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84\uff0c\u662f\u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u7684\u503c\u4e3a OSS \u5730\u5740\uff0c\u4fee\u6539\u53c2\u6570 hbase.wal.dir \u7684\u503c\u4e3a\u672c\u5730\u7684 HDFS \u5730\u5740\uff0c\u901a\u8fc7\u672c\u5730 HDFS \u96c6\u7fa4\u5b58\u50a8 WAL \u6587\u4ef6\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 OSS\u3002\u53c2\u6570\u503c\u4e3a oss://bucket/hbase-root-dir hbase.wal.dir \u6307\u5b9a HBase \u7684 WAL \u5b58\u50a8\u76ee\u5f55\u5230\u672c\u5730 HDFS \u96c6\u7fa4\u3002 \u53c2\u6570\u503c\u4e3a\uff1a HA\u96c6\u7fa4\uff1a hdfs://${nameservice}/hbase \u975eHA\u96c6\u7fa4\uff1a hdfs://${namenode_rpc_address}:${namenode_rpc_port}/hbase","title":"\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs/","text":"Apache HDFS \u900f\u660e\u7f13\u5b58\u52a0\u901f \u00a7 \u5f53\u8ba1\u7b97\u96c6\u7fa4\u4f7f\u7528 Apache HDFS \u5b58\u50a8\uff0c\u5e76\u4e14\u4e0e\u4f7f\u7528\u7684 HDFS \u5b58\u50a8\u96c6\u7fa4\u5206\u79bb\u65f6\uff0c\u8bbf\u95ee HDFS \u96c6\u7fa4\u7684\u7f51\u7edc\u5e26\u5bbd\u53ef\u80fd\u53d7\u5230\u9650\u5236\u3002Apache HDFS \u900f\u660e\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002\u8ba1\u7b97\u96c6\u7fa4\u6216\u670d\u52a1\u53ef\u7ee7\u7eed\u4f7f\u7528\u539f\u6765\u7684\u5730\u5740 (hdfs://) \u8bbf\u95ee\u539f HDFS \u96c6\u7fa4\uff0c\u540c\u65f6\u901a\u8fc7\u672c\u5730\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u670d\u52a1\u7aef\u914d\u7f6e \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 \u914d\u7f6e HDFS \u96c6\u7fa4\u7684\u7f13\u5b58\u52a0\u901f [jindofsx-common] jindofsx.hdfs.user = xxx #Storage Service \u8bbf\u95ee HDFS \u4f7f\u7528\u7684\u7528\u6237\u540d \u914d\u7f6e HA HDFS \u96c6\u7fa4\u7684\u7f13\u5b58\u52a0\u901f\uff08\u53ef\u9009\uff09 [jindofsx-common] jindofsx.hdfs.XXX.dfs.ha.namenodes = nn1,nn2 jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn1 = nn1-host1:nn1-rpc-port jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn2 = nn2-host1:nn2-rpc-port \u8bf4\u660e: XXX \u4e3a HDFS \u96c6\u7fa4\u4e2d\u914d\u7f6e\u7684 dfs.nameservices \u53c2\u6570\u503c\u3002 \u5e76\u5c06\u914d\u7f6e\u6587\u4ef6\u66f4\u65b0\u5230\u6240\u6709\u670d\u52a1\u6240\u5728\u8282\u70b9\u4e0a\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u63d0\u4f9b\u7684 HDFS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.JindoHdfsFileSystem</value> </property> <property> <name>fs.AbstractFileSystem.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.HDFS</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e HA Namenodes (\u53ef\u9009) \u82e5\u4e3a HA HDFS\u96c6\u7fa4\uff0c\u5219\u5c06\u4e0b\u9762\u53c2\u6570\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.jindofsx.hdfs.XXX.dfs.ha.namenodes</name> <value>nn1,nn2</value> </property> <property> <name>fs.jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn1</name> <value>nn1-host1:nn1-rpc-port</value> </property> <property> <name>fs.jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn2</name> <value>nn1-host1:nn1-rpc-port</value> </property> </configuration> \u8bf4\u660e: XXX \u4e3a HDFS \u96c6\u7fa4\u4e2d\u914d\u7f6e\u7684 dfs.nameservices \u53c2\u6570\u503c\u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u914d\u7f6e\u7f13\u5b58\u52a0\u901f \u00a7 \u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u8bbf\u95ee HDFS \u00a7 \u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u901a\u8fc7 hdfs:// \u524d\u7f00\u8bfb\u53d6 HDFS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 hdfs:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002","title":"Apache HDFS \u900f\u660e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs/#apache-hdfs","text":"\u5f53\u8ba1\u7b97\u96c6\u7fa4\u4f7f\u7528 Apache HDFS \u5b58\u50a8\uff0c\u5e76\u4e14\u4e0e\u4f7f\u7528\u7684 HDFS \u5b58\u50a8\u96c6\u7fa4\u5206\u79bb\u65f6\uff0c\u8bbf\u95ee HDFS \u96c6\u7fa4\u7684\u7f51\u7edc\u5e26\u5bbd\u53ef\u80fd\u53d7\u5230\u9650\u5236\u3002Apache HDFS \u900f\u660e\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002\u8ba1\u7b97\u96c6\u7fa4\u6216\u670d\u52a1\u53ef\u7ee7\u7eed\u4f7f\u7528\u539f\u6765\u7684\u5730\u5740 (hdfs://) \u8bbf\u95ee\u539f HDFS \u96c6\u7fa4\uff0c\u540c\u65f6\u901a\u8fc7\u672c\u5730\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002","title":"Apache HDFS \u900f\u660e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs/#_2","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 \u914d\u7f6e HDFS \u96c6\u7fa4\u7684\u7f13\u5b58\u52a0\u901f [jindofsx-common] jindofsx.hdfs.user = xxx #Storage Service \u8bbf\u95ee HDFS \u4f7f\u7528\u7684\u7528\u6237\u540d \u914d\u7f6e HA HDFS \u96c6\u7fa4\u7684\u7f13\u5b58\u52a0\u901f\uff08\u53ef\u9009\uff09 [jindofsx-common] jindofsx.hdfs.XXX.dfs.ha.namenodes = nn1,nn2 jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn1 = nn1-host1:nn1-rpc-port jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn2 = nn2-host1:nn2-rpc-port \u8bf4\u660e: XXX \u4e3a HDFS \u96c6\u7fa4\u4e2d\u914d\u7f6e\u7684 dfs.nameservices \u53c2\u6570\u503c\u3002 \u5e76\u5c06\u914d\u7f6e\u6587\u4ef6\u66f4\u65b0\u5230\u6240\u6709\u670d\u52a1\u6240\u5728\u8282\u70b9\u4e0a\u3002","title":"\u670d\u52a1\u7aef\u914d\u7f6e"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs/#jindofsx","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs/#jindosdk","text":"\u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u63d0\u4f9b\u7684 HDFS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.JindoHdfsFileSystem</value> </property> <property> <name>fs.AbstractFileSystem.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.HDFS</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e HA Namenodes (\u53ef\u9009) \u82e5\u4e3a HA HDFS\u96c6\u7fa4\uff0c\u5219\u5c06\u4e0b\u9762\u53c2\u6570\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.jindofsx.hdfs.XXX.dfs.ha.namenodes</name> <value>nn1,nn2</value> </property> <property> <name>fs.jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn1</name> <value>nn1-host1:nn1-rpc-port</value> </property> <property> <name>fs.jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn2</name> <value>nn1-host1:nn1-rpc-port</value> </property> </configuration> \u8bf4\u660e: XXX \u4e3a HDFS \u96c6\u7fa4\u4e2d\u914d\u7f6e\u7684 dfs.nameservices \u53c2\u6570\u503c\u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs/#_3","text":"\u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e","title":"\u914d\u7f6e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs/#hdfs","text":"\u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u901a\u8fc7 hdfs:// \u524d\u7f00\u8bfb\u53d6 HDFS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 hdfs:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002","title":"\u8bbf\u95ee HDFS"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs_jindo/","text":"Apache HDFS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f \u00a7 \u5f53\u8ba1\u7b97\u96c6\u7fa4\u4f7f\u7528 Apache HDFS \u5b58\u50a8\uff0c\u5e76\u4e14\u4e0e\u4f7f\u7528\u7684 HDFS \u5b58\u50a8\u96c6\u7fa4\u5206\u79bb\u65f6\uff0c\u8bbf\u95ee HDFS \u96c6\u7fa4\u7684\u7f51\u7edc\u5e26\u5bbd\u53ef\u80fd\u53d7\u5230\u9650\u5236\u3002Apache HDFS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002\u8ba1\u7b97\u96c6\u7fa4\u6216\u670d\u52a1\u4f7f\u7528\u7edf\u4e00\u6302\u8f7d\u7684\u65b9\u5f0f (jindo://) \u8bbf\u95ee\u539f HDFS \u96c6\u7fa4\uff0c\u540c\u65f6\u901a\u8fc7\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u670d\u52a1\u7aef\u914d\u7f6e \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 \u914d\u7f6e HDFS \u96c6\u7fa4\u7684\u7f13\u5b58\u52a0\u901f [jindofsx-common] jindofsx.hdfs.user = xxx #Storage Service \u8bbf\u95ee HDFS \u4f7f\u7528\u7684\u7528\u6237\u540d \u914d\u7f6e HA HDFS \u96c6\u7fa4\u7684\u7f13\u5b58\u52a0\u901f\uff08\u53ef\u9009\uff09 [jindofsx-common] jindofsx.hdfs.XXX.dfs.ha.namenodes = nn1,nn2 jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn1 = nn1-host1:nn1-rpc-port jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn2 = nn2-host1:nn2-rpc-port \u8bf4\u660e: XXX \u4e3a HDFS \u96c6\u7fa4\u4e2d\u914d\u7f6e\u7684 dfs.nameservices \u53c2\u6570\u503c\u3002 \u5e76\u5c06\u914d\u7f6e\u6587\u4ef6\u66f4\u65b0\u5230\u6240\u6709\u670d\u52a1\u6240\u5728\u8282\u70b9\u4e0a\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e HA Namenodes (\u53ef\u9009) \u82e5\u4e3a HA HDFS\u96c6\u7fa4\uff0c\u5219\u5c06\u4e0b\u9762\u53c2\u6570\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.jindofsx.hdfs.XXX.dfs.ha.namenodes</name> <value>nn1,nn2</value> </property> <property> <name>fs.jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn1</name> <value>nn1-host1:nn1-rpc-port</value> </property> <property> <name>fs.jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn2</name> <value>nn1-host1:nn1-rpc-port</value> </property> </configuration> \u8bf4\u660e: XXX \u4e3a HDFS \u96c6\u7fa4\u4e2d\u914d\u7f6e\u7684 dfs.nameservices \u53c2\u6570\u503c\u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u6302\u8f7d HDFS \u00a7 \u6302\u8f7d HDFS \u96c6\u7fa4\u76ee\u5f55 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindohdfs hdfs://emr-header-1:9000/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindohdfs \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f hdfs://emr-header-1:9000/ [root@emr-header-1 ~]# hdfs dfs -ls jindo://emr-header-1:8101/ Found 2 items ---------- 1 0 1970-01-01 08:00 jindo://emr-header-1:8101/jindohdfs \u5373\u8bbf\u95ee jindo://emr-header-1:8101/jindohdfs/ \u7b49\u4ef7\u4e8e\u8bbf\u95ee hdfs://emr-header-1:9000/ \u914d\u7f6e\u7f13\u5b58\u52a0\u901f \u00a7 \u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u8bbf\u95ee HDFS \u00a7 \u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u5728\u6570\u636e\u7f13\u5b58\u5f00\u5173\u6253\u5f00\u65f6\u901a\u8fc7 jindo:// \u524d\u7f00\u8bfb\u53d6 HDFS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 jindo:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002","title":"Apache HDFS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs_jindo/#apache-hdfs","text":"\u5f53\u8ba1\u7b97\u96c6\u7fa4\u4f7f\u7528 Apache HDFS \u5b58\u50a8\uff0c\u5e76\u4e14\u4e0e\u4f7f\u7528\u7684 HDFS \u5b58\u50a8\u96c6\u7fa4\u5206\u79bb\u65f6\uff0c\u8bbf\u95ee HDFS \u96c6\u7fa4\u7684\u7f51\u7edc\u5e26\u5bbd\u53ef\u80fd\u53d7\u5230\u9650\u5236\u3002Apache HDFS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002\u8ba1\u7b97\u96c6\u7fa4\u6216\u670d\u52a1\u4f7f\u7528\u7edf\u4e00\u6302\u8f7d\u7684\u65b9\u5f0f (jindo://) \u8bbf\u95ee\u539f HDFS \u96c6\u7fa4\uff0c\u540c\u65f6\u901a\u8fc7\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002","title":"Apache HDFS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs_jindo/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs_jindo/#_2","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 \u914d\u7f6e HDFS \u96c6\u7fa4\u7684\u7f13\u5b58\u52a0\u901f [jindofsx-common] jindofsx.hdfs.user = xxx #Storage Service \u8bbf\u95ee HDFS \u4f7f\u7528\u7684\u7528\u6237\u540d \u914d\u7f6e HA HDFS \u96c6\u7fa4\u7684\u7f13\u5b58\u52a0\u901f\uff08\u53ef\u9009\uff09 [jindofsx-common] jindofsx.hdfs.XXX.dfs.ha.namenodes = nn1,nn2 jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn1 = nn1-host1:nn1-rpc-port jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn2 = nn2-host1:nn2-rpc-port \u8bf4\u660e: XXX \u4e3a HDFS \u96c6\u7fa4\u4e2d\u914d\u7f6e\u7684 dfs.nameservices \u53c2\u6570\u503c\u3002 \u5e76\u5c06\u914d\u7f6e\u6587\u4ef6\u66f4\u65b0\u5230\u6240\u6709\u670d\u52a1\u6240\u5728\u8282\u70b9\u4e0a\u3002","title":"\u670d\u52a1\u7aef\u914d\u7f6e"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs_jindo/#jindofsx","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs_jindo/#jindosdk","text":"\u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e HA Namenodes (\u53ef\u9009) \u82e5\u4e3a HA HDFS\u96c6\u7fa4\uff0c\u5219\u5c06\u4e0b\u9762\u53c2\u6570\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.jindofsx.hdfs.XXX.dfs.ha.namenodes</name> <value>nn1,nn2</value> </property> <property> <name>fs.jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn1</name> <value>nn1-host1:nn1-rpc-port</value> </property> <property> <name>fs.jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn2</name> <value>nn1-host1:nn1-rpc-port</value> </property> </configuration> \u8bf4\u660e: XXX \u4e3a HDFS \u96c6\u7fa4\u4e2d\u914d\u7f6e\u7684 dfs.nameservices \u53c2\u6570\u503c\u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs_jindo/#hdfs","text":"\u6302\u8f7d HDFS \u96c6\u7fa4\u76ee\u5f55 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindohdfs hdfs://emr-header-1:9000/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindohdfs \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f hdfs://emr-header-1:9000/ [root@emr-header-1 ~]# hdfs dfs -ls jindo://emr-header-1:8101/ Found 2 items ---------- 1 0 1970-01-01 08:00 jindo://emr-header-1:8101/jindohdfs \u5373\u8bbf\u95ee jindo://emr-header-1:8101/jindohdfs/ \u7b49\u4ef7\u4e8e\u8bbf\u95ee hdfs://emr-header-1:9000/","title":"\u6302\u8f7d HDFS"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs_jindo/#_3","text":"\u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e","title":"\u914d\u7f6e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs_jindo/#hdfs_1","text":"\u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u5728\u6570\u636e\u7f13\u5b58\u5f00\u5173\u6253\u5f00\u65f6\u901a\u8fc7 jindo:// \u524d\u7f00\u8bfb\u53d6 HDFS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 jindo:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002","title":"\u8bbf\u95ee HDFS"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/","text":"Hive \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002 \u5b89\u88c5\u914d\u7f6e \u00a7 1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/ 2. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b \u5c06 JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key \u5c06\u5df2\u5f00\u542f OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 3. \u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 4. Hive on MR \u00a7 \u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002 5. Hive on Tez \u00a7 \u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002 6. Hive on Spark \u00a7 \u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002 OSS-HDFS \u670d\u52a1\u7528\u4e8e\u8868\u7684\u5b58\u50a8 \u00a7 \u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS-HDFS \u670d\u52a1\u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'oss://bucket_name/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'oss://bucket_name/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS-HDFS \u670d\u52a1\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>oss://bucket_name/path/to/warehouse</value> </property> </configuration> \u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a \u00a7 ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'oss://bucket_name/path/to/us/part210301cn'; \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Hive \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#hive-oss-hdfs-jindofs-jindofsx","text":"Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002","title":"Hive \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#_1","text":"","title":"\u5b89\u88c5\u914d\u7f6e"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#1-hive-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/","title":"1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#2-jindosdk","text":"\u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b \u5c06 JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key \u5c06\u5df2\u5f00\u542f OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"2. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#3-hive","text":"","title":"3. \u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#4-hive-on-mr","text":"\u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002","title":"4. Hive on MR"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#5-hive-on-tez","text":"\u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002","title":"5. Hive on Tez"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#6-hive-on-spark","text":"\u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002","title":"6. Hive on Spark"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#oss-hdfs","text":"\u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS-HDFS \u670d\u52a1\u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'oss://bucket_name/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'oss://bucket_name/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS-HDFS \u670d\u52a1\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>oss://bucket_name/path/to/warehouse</value> </property> </configuration>","title":"OSS-HDFS \u670d\u52a1\u7528\u4e8e\u8868\u7684\u5b58\u50a8"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#oss","text":"ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'oss://bucket_name/path/to/us/part210301cn';","title":"\u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#_2","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/","text":"Hive \u5904\u7406 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e \u00a7 Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS \u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002 \u5b89\u88c5\u914d\u7f6e \u00a7 1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/ 2. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Access Key \u5c06 OSS Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 3. \u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 4. Hive on MR \u00a7 \u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002 5. Hive on Tez \u00a7 \u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002 6. Hive on Spark \u00a7 \u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002 \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55 \u00a7 \u6302\u8f7d\u547d\u4ee4 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/ OSS \u7528\u4e8e\u8868\u7684\u5b58\u50a8 \u00a7 \u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS \u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS \u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'jindo://headerhost:8101/jindooss/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'jindo://headerhost:8101/jindooss/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS \u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>jindo://headerhost:8101/jindooss/path/to/warehouse</value> </property> </configuration> \u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a \u00a7 ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'jindo://headerhost:8101/jindooss/path/to/us/part210301cn'; \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Hive \u5904\u7406 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#hive-jindofsx","text":"Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS \u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002","title":"Hive \u5904\u7406 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#_1","text":"","title":"\u5b89\u88c5\u914d\u7f6e"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#1-hive-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/","title":"1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#2-jindosdk","text":"\u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Access Key \u5c06 OSS Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"2. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#3-hive","text":"","title":"3. \u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#4-hive-on-mr","text":"\u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002","title":"4. Hive on MR"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#5-hive-on-tez","text":"\u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002","title":"5. Hive on Tez"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#6-hive-on-spark","text":"\u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002","title":"6. Hive on Spark"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#oss-oss-hdfs","text":"\u6302\u8f7d\u547d\u4ee4 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/","title":"\u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#oss","text":"\u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS \u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS \u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'jindo://headerhost:8101/jindooss/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'jindo://headerhost:8101/jindooss/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS \u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>jindo://headerhost:8101/jindooss/path/to/warehouse</value> </property> </configuration>","title":"OSS \u7528\u4e8e\u8868\u7684\u5b58\u50a8"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#oss_1","text":"ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'jindo://headerhost:8101/jindooss/path/to/us/part210301cn';","title":"\u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#_2","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/","text":"Hive \u5904\u7406\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS \u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002 \u5b89\u88c5\u914d\u7f6e \u00a7 1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/ 2. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key \u5c06 OSS Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 3. \u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 4. Hive on MR \u00a7 \u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002 5. Hive on Tez \u00a7 \u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002 6. Hive on Spark \u00a7 \u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002 OSS \u7528\u4e8e\u8868\u7684\u5b58\u50a8 \u00a7 \u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS \u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS \u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'oss://bucketname/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'oss://bucketname/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS \u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>oss://bucketname/path/to/warehouse</value> </property> </configuration> \u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a \u00a7 ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'oss://bucketname/path/to/us/part210301cn'; \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Hive \u5904\u7406\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#hive-oss-jindofsx","text":"Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS \u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002","title":"Hive \u5904\u7406\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#_1","text":"","title":"\u5b89\u88c5\u914d\u7f6e"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#1-hive-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/","title":"1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#2-jindosdk","text":"\u914d\u7f6e OSS \u5b9e\u73b0\u7c7b \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key \u5c06 OSS Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"2. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#3-hive","text":"","title":"3. \u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#4-hive-on-mr","text":"\u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002","title":"4. Hive on MR"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#5-hive-on-tez","text":"\u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002","title":"5. Hive on Tez"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#6-hive-on-spark","text":"\u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002","title":"6. Hive on Spark"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#oss","text":"\u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS \u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS \u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'oss://bucketname/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'oss://bucketname/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS \u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>oss://bucketname/path/to/warehouse</value> </property> </configuration>","title":"OSS \u7528\u4e8e\u8868\u7684\u5b58\u50a8"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#oss_1","text":"ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'oss://bucketname/path/to/us/part210301cn';","title":"\u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#_2","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_dls/","text":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Impala \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Impala \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fddImpala\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728\u6240\u6709 Impala \u8282\u70b9\u5b89\u88c5 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06 sdk \u5305\u5b89\u88c5\u5230 Impala \u7684 classpath \u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $IMPALA_HOME/lib/ 3. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e IMPALA \u4f7f\u7528\u7684 core-site.xml \u914d\u7f6e JindoSDK \u8bbf\u95ee OSS \u00a7 \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 4. \u4f7f\u7528 Impala \u8bbf\u95ee OSS\u3002 \u00a7 \u521b\u5efa\u8868 sql CREATE EXTERNAL TABLE customer_demographics ( `cd_demo_sk` INT, `cd_gender` STRING, `cd_marital_status` STRING, `cd_education_status` STRING, `cd_purchase_estimate` INT, `cd_credit_rating` STRING, `cd_dep_count` INT, `cd_dep_employed_count` INT, `cd_dep_college_count` INT) STORED AS PARQUET LOCATION 'oss://bucket.endpoint/dir'; \u67e5\u8be2 OSS \u8868 sql select * from customer_demographics; 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_dls/#impala-jindosdk-oss-hdfs-jindofs-jindofsx","text":"JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Impala \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_dls/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_dls/#1-impala-hadoop","text":"\u786e\u4fddImpala\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Impala \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_dls/#2-impala-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06 sdk \u5305\u5b89\u88c5\u5230 Impala \u7684 classpath \u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $IMPALA_HOME/lib/","title":"2. \u5728\u6240\u6709 Impala \u8282\u70b9\u5b89\u88c5 JindoSDK"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_dls/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_dls/#impala-core-sitexml-jindosdk-oss","text":"\u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"\u914d\u7f6e IMPALA \u4f7f\u7528\u7684 core-site.xml \u914d\u7f6e JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_dls/#4-impala-oss","text":"\u521b\u5efa\u8868 sql CREATE EXTERNAL TABLE customer_demographics ( `cd_demo_sk` INT, `cd_gender` STRING, `cd_marital_status` STRING, `cd_education_status` STRING, `cd_purchase_estimate` INT, `cd_credit_rating` STRING, `cd_dep_count` INT, `cd_dep_employed_count` INT, `cd_dep_college_count` INT) STORED AS PARQUET LOCATION 'oss://bucket.endpoint/dir'; \u67e5\u8be2 OSS \u8868 sql select * from customer_demographics;","title":"4. \u4f7f\u7528 Impala \u8bbf\u95ee OSS\u3002"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_dls/#5","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_jindo/","text":"Impala \u67e5\u8be2 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e \u00a7 JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Impala \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Impala \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fddImpala\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728\u6240\u6709 Impala \u8282\u70b9\u5b89\u88c5 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06 sdk \u5305\u5b89\u88c5\u5230 Impala \u7684 classpath \u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $IMPALA_HOME/lib/ 3. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e IMPALA \u4f7f\u7528\u7684 core-site.xml \u914d\u7f6e JindoSDK \u8bbf\u95ee OSS \u00a7 \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/ 4. \u4f7f\u7528 Impala \u8bbf\u95ee OSS\u3002 \u00a7 \u521b\u5efa\u8868 sql CREATE EXTERNAL TABLE customer_demographics ( `cd_demo_sk` INT, `cd_gender` STRING, `cd_marital_status` STRING, `cd_education_status` STRING, `cd_purchase_estimate` INT, `cd_credit_rating` STRING, `cd_dep_count` INT, `cd_dep_employed_count` INT, `cd_dep_college_count` INT) STORED AS PARQUET LOCATION 'jindo://headerhost:8101/jindooss/dir'; \u67e5\u8be2 OSS \u8868 sql select * from customer_demographics; 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Impala \u67e5\u8be2 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_jindo/#impala-jindofsx","text":"JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Impala \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Impala \u67e5\u8be2 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_jindo/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_jindo/#1-impala-hadoop","text":"\u786e\u4fddImpala\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Impala \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_jindo/#2-impala-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06 sdk \u5305\u5b89\u88c5\u5230 Impala \u7684 classpath \u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $IMPALA_HOME/lib/","title":"2. \u5728\u6240\u6709 Impala \u8282\u70b9\u5b89\u88c5 JindoSDK"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_jindo/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_jindo/#impala-core-sitexml-jindosdk-oss","text":"\u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/","title":"\u914d\u7f6e IMPALA \u4f7f\u7528\u7684 core-site.xml \u914d\u7f6e JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_jindo/#4-impala-oss","text":"\u521b\u5efa\u8868 sql CREATE EXTERNAL TABLE customer_demographics ( `cd_demo_sk` INT, `cd_gender` STRING, `cd_marital_status` STRING, `cd_education_status` STRING, `cd_purchase_estimate` INT, `cd_credit_rating` STRING, `cd_dep_count` INT, `cd_dep_employed_count` INT, `cd_dep_college_count` INT) STORED AS PARQUET LOCATION 'jindo://headerhost:8101/jindooss/dir'; \u67e5\u8be2 OSS \u8868 sql select * from customer_demographics;","title":"4. \u4f7f\u7528 Impala \u8bbf\u95ee OSS\u3002"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_jindo/#5","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_oss/","text":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Impala \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Impala \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fddImpala\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728\u6240\u6709 Impala \u8282\u70b9\u5b89\u88c5 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06 sdk \u5305\u5b89\u88c5\u5230 Impala \u7684 classpath \u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $IMPALA_HOME/lib/ 3. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e IMPALA \u4f7f\u7528\u7684 core-site.xml \u914d\u7f6e JindoSDK \u8bbf\u95ee OSS \u00a7 \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key \u5c06 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 4. \u4f7f\u7528 Impala \u8bbf\u95ee OSS\u3002 \u00a7 \u521b\u5efa\u8868 sql CREATE EXTERNAL TABLE customer_demographics ( `cd_demo_sk` INT, `cd_gender` STRING, `cd_marital_status` STRING, `cd_education_status` STRING, `cd_purchase_estimate` INT, `cd_credit_rating` STRING, `cd_dep_count` INT, `cd_dep_employed_count` INT, `cd_dep_college_count` INT) STORED AS PARQUET LOCATION 'oss://bucket.endpoint/dir'; \u67e5\u8be2 OSS \u8868 sql select * from customer_demographics; 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_oss/#impala-jindosdk-oss-jindofsx","text":"JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Impala \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_oss/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_oss/#1-impala-hadoop","text":"\u786e\u4fddImpala\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Impala \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_oss/#2-impala-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06 sdk \u5305\u5b89\u88c5\u5230 Impala \u7684 classpath \u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $IMPALA_HOME/lib/","title":"2. \u5728\u6240\u6709 Impala \u8282\u70b9\u5b89\u88c5 JindoSDK"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_oss/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_oss/#impala-core-sitexml-jindosdk-oss","text":"\u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key \u5c06 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"\u914d\u7f6e IMPALA \u4f7f\u7528\u7684 core-site.xml \u914d\u7f6e JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_oss/#4-impala-oss","text":"\u521b\u5efa\u8868 sql CREATE EXTERNAL TABLE customer_demographics ( `cd_demo_sk` INT, `cd_gender` STRING, `cd_marital_status` STRING, `cd_education_status` STRING, `cd_purchase_estimate` INT, `cd_credit_rating` STRING, `cd_dep_count` INT, `cd_dep_employed_count` INT, `cd_dep_college_count` INT) STORED AS PARQUET LOCATION 'oss://bucket.endpoint/dir'; \u67e5\u8be2 OSS \u8868 sql select * from customer_demographics;","title":"4. \u4f7f\u7528 Impala \u8bbf\u95ee OSS\u3002"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_oss/#5","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_aliyun_sls_example/","text":"\u4f7f\u7528\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u6536\u96c6\u670d\u52a1 \u00a7 JindoRuntime \u9ed8\u8ba4\u65e5\u5fd7\u6253\u5728\u5bb9\u5668 std \u8f93\u51fa\u91cc\u9762\uff0c\u53ef\u4ee5\u4f7f\u7528\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u6536\u96c6\u670d\u52a1\u5c06\u65e5\u5fd7\u6536\u96c6\u8d77\u6765\u3002 1\u3001ack \u96c6\u7fa4\u7ed1\u5b9a sls \u670d\u52a1 \u00a7 \u60a8\u53ef\u4ee5\u5728\u521b\u5efa\u96c6\u7fa4\u65f6\u7ed1\u5b9a sls \u670d\u52a1 \u4e5f\u53ef\u4ee5\u5728\u521b\u5efa\u96c6\u7fa4\u540e\u7ed1\u5b9a\u963f\u91cc\u4e91 sls \u670d\u52a1 \u9009\u62e9\u96c6\u7fa4\u4fe1\u606f > \u96c6\u7fa4\u8d44\u6e90 > \u65e5\u5fd7\u670d\u52a1 Project 2\u3001\u521b\u5efa JindoRuntime \u96c6\u7fa4 \u00a7 \u53ef\u53c2\u8003\u6587\u7ae0 \u52a0\u901fOSS\u4e0a\u6570\u636e \u521b\u5efa JindoRuntime \u96c6\u7fa4\u3002 3\u3001\u914d\u7f6e sls \u6536\u96c6 \u00a7 \u60a8\u53ef\u4ee5\u9009\u62e9\u5728 ack \u63a7\u5236\u53f0\u64cd\u4f5c\uff0c\u6216\u8005\u76f4\u63a5\u64cd\u4f5c\u96c6\u7fa4 K8S \u8d44\u6e90 3.1 ack \u63a7\u5236\u53f0\u64cd\u4f5c \u00a7 \u672c\u6587\u4ee5 JindoRuntime \u540d\u5b57\u4e3a test \u8fdb\u884c\u4e3e\u4f8b \u60a8\u53ef\u4ee5\u767b\u9646 ack \u63a7\u5236\u53f0 > \u5de5\u4f5c\u8d1f\u8f7d * \u6709\u72b6\u6001\uff08master/worker\uff09 \u5b88\u62a4\u8fdb\u7a0b\u96c6\uff08fuse\uff09 \u70b9\u51fb \u7f16\u8f91 \u6dfb\u52a0\u65e5\u5fd7\u6536\u96c6 \u5176\u4e2d\u65e5\u5fd7\u5e93\u540d\u79f0\u53ef\u81ea\u5b9a\u4e49\u586b\u5199\uff0c\u8be5\u540d\u79f0\u5373\u4e3a sls \u4e0a\u65e5\u5fd7\u5e93\u7684\u540d\u5b57\uff0c\u5bb9\u5668\u5185\u65e5\u5fd7\u8def\u5f84\u4e3a stdout \u70b9\u51fb\u53f3\u4fa7\u66f4\u65b0 3.2\u3001\u76f4\u63a5\u64cd\u4f5c K8S \u8d44\u6e90 \u00a7 \u4ee5\u6536\u96c6 fuse \u65e5\u5fd7\u4e3a\u4f8b\uff0c\u6267\u884c kubectl edit daemonset test-jindofs-fuse \u4fee\u6539 spec.env \u589e\u52a0 - name: aliyun_logs_jindo-fuse value: stdout name \u683c\u5f0f\u4e3a\uff1a aliyun_logs_ \uff0c\u524d\u7f00\u4e3a aliyun_logs_ value \u4e3a\u56fa\u5b9a\u503c stdout \u5982\u679c\u60f3\u4fee\u6539 master/worker\uff0c\u53ef\u6267\u884c kubectl edit statefulset test-jindofs-master kubectl edit statefulset test-jindofs-worker \u8fdb\u884c\u5bf9\u5e94\u7684\u7f16\u8f91 4\u3001\u767b\u9646\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u670d\u52a1\u67e5\u770b\u76f8\u5173\u65e5\u5fd7\u60c5\u51b5 \u00a7 \u4ee5 jindofuse \u65e5\u5fd7\u6536\u96c6\u4e3a\u4f8b\uff0c\u65e5\u5fd7\u5e93\u540d\u79f0\u4e3a jindo-fuse ,\u5728sls\u4e0a\u627e\u5230\u5bf9\u5e94\u7684\u65e5\u5fd7\u5e93","title":"\u4f7f\u7528\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u6536\u96c6\u670d\u52a1"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_aliyun_sls_example/#sls","text":"JindoRuntime \u9ed8\u8ba4\u65e5\u5fd7\u6253\u5728\u5bb9\u5668 std \u8f93\u51fa\u91cc\u9762\uff0c\u53ef\u4ee5\u4f7f\u7528\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u6536\u96c6\u670d\u52a1\u5c06\u65e5\u5fd7\u6536\u96c6\u8d77\u6765\u3002","title":"\u4f7f\u7528\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u6536\u96c6\u670d\u52a1"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_aliyun_sls_example/#1ack-sls","text":"\u60a8\u53ef\u4ee5\u5728\u521b\u5efa\u96c6\u7fa4\u65f6\u7ed1\u5b9a sls \u670d\u52a1 \u4e5f\u53ef\u4ee5\u5728\u521b\u5efa\u96c6\u7fa4\u540e\u7ed1\u5b9a\u963f\u91cc\u4e91 sls \u670d\u52a1 \u9009\u62e9\u96c6\u7fa4\u4fe1\u606f > \u96c6\u7fa4\u8d44\u6e90 > \u65e5\u5fd7\u670d\u52a1 Project","title":"1\u3001ack \u96c6\u7fa4\u7ed1\u5b9a sls \u670d\u52a1"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_aliyun_sls_example/#2-jindoruntime","text":"\u53ef\u53c2\u8003\u6587\u7ae0 \u52a0\u901fOSS\u4e0a\u6570\u636e \u521b\u5efa JindoRuntime \u96c6\u7fa4\u3002","title":"2\u3001\u521b\u5efa JindoRuntime \u96c6\u7fa4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_aliyun_sls_example/#3-sls","text":"\u60a8\u53ef\u4ee5\u9009\u62e9\u5728 ack \u63a7\u5236\u53f0\u64cd\u4f5c\uff0c\u6216\u8005\u76f4\u63a5\u64cd\u4f5c\u96c6\u7fa4 K8S \u8d44\u6e90","title":"3\u3001\u914d\u7f6e sls \u6536\u96c6"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_aliyun_sls_example/#31-ack","text":"\u672c\u6587\u4ee5 JindoRuntime \u540d\u5b57\u4e3a test \u8fdb\u884c\u4e3e\u4f8b \u60a8\u53ef\u4ee5\u767b\u9646 ack \u63a7\u5236\u53f0 > \u5de5\u4f5c\u8d1f\u8f7d * \u6709\u72b6\u6001\uff08master/worker\uff09 \u5b88\u62a4\u8fdb\u7a0b\u96c6\uff08fuse\uff09 \u70b9\u51fb \u7f16\u8f91 \u6dfb\u52a0\u65e5\u5fd7\u6536\u96c6 \u5176\u4e2d\u65e5\u5fd7\u5e93\u540d\u79f0\u53ef\u81ea\u5b9a\u4e49\u586b\u5199\uff0c\u8be5\u540d\u79f0\u5373\u4e3a sls \u4e0a\u65e5\u5fd7\u5e93\u7684\u540d\u5b57\uff0c\u5bb9\u5668\u5185\u65e5\u5fd7\u8def\u5f84\u4e3a stdout \u70b9\u51fb\u53f3\u4fa7\u66f4\u65b0","title":"3.1 ack \u63a7\u5236\u53f0\u64cd\u4f5c"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_aliyun_sls_example/#32-k8s","text":"\u4ee5\u6536\u96c6 fuse \u65e5\u5fd7\u4e3a\u4f8b\uff0c\u6267\u884c kubectl edit daemonset test-jindofs-fuse \u4fee\u6539 spec.env \u589e\u52a0 - name: aliyun_logs_jindo-fuse value: stdout name \u683c\u5f0f\u4e3a\uff1a aliyun_logs_ \uff0c\u524d\u7f00\u4e3a aliyun_logs_ value \u4e3a\u56fa\u5b9a\u503c stdout \u5982\u679c\u60f3\u4fee\u6539 master/worker\uff0c\u53ef\u6267\u884c kubectl edit statefulset test-jindofs-master kubectl edit statefulset test-jindofs-worker \u8fdb\u884c\u5bf9\u5e94\u7684\u7f16\u8f91","title":"3.2\u3001\u76f4\u63a5\u64cd\u4f5c K8S \u8d44\u6e90"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_aliyun_sls_example/#4-sls","text":"\u4ee5 jindofuse \u65e5\u5fd7\u6536\u96c6\u4e3a\u4f8b\uff0c\u65e5\u5fd7\u5e93\u540d\u79f0\u4e3a jindo-fuse ,\u5728sls\u4e0a\u627e\u5230\u5bf9\u5e94\u7684\u65e5\u5fd7\u5e93","title":"4\u3001\u767b\u9646\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u670d\u52a1\u67e5\u770b\u76f8\u5173\u65e5\u5fd7\u60c5\u51b5"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/","text":"\u6570\u636e\u9884\u52a0\u8f7d \u00a7 \u914d\u7f6e\u5f85\u521b\u5efa\u7684DataLoad\u5bf9\u8c61 \u521b\u5efaDataLoad\u5bf9\u8c61 \u67e5\u770b\u521b\u5efa\u7684DataLoad\u5bf9\u8c61\u72b6\u6001 \u7b49\u5f85\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5b8c\u6210 DataLoad\u8fdb\u9636\u914d\u7f6e \u6307\u5b9a\u4e00\u4e2a\u6216\u591a\u4e2a\u6570\u636e\u96c6\u5b50\u76ee\u5f55\u8fdb\u884c\u52a0\u8f7d \u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf \u539f\u5b50\u6027\u7f13\u5b58 \u5355\u72ec\u8fdb\u884c\u5143\u6570\u636e\u7f13\u5b58\uff0c\u4e0d\u505a\u6570\u636e\u7f13\u5b58 \u7f13\u5b58\u8fdb\u5ea6\u53caLOG\u67e5\u770b \u73af\u5883\u6e05\u7406 \u4e3a\u4e86\u4fdd\u8bc1\u5e94\u7528\u5728\u8bbf\u95ee\u6570\u636e\u65f6\u7684\u6027\u80fd\uff0c\u53ef\u4ee5\u901a\u8fc7 \u6570\u636e\u9884\u52a0\u8f7d \u63d0\u524d\u5c06\u8fdc\u7a0b\u5b58\u50a8\u7cfb\u7edf\u4e2d\u7684\u6570\u636e\u62c9\u53d6\u5230\u9760\u8fd1\u8ba1\u7b97\u7ed3\u70b9\u7684\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d\uff0c\u4f7f\u5f97\u6d88\u8d39\u8be5\u6570\u636e\u96c6\u7684\u5e94\u7528\u80fd\u591f\u5728\u9996\u6b21\u8fd0\u884c\u65f6\u5373\u53ef\u4eab\u53d7\u5230\u7f13\u5b58\u5e26\u6765\u7684\u52a0\u901f\u6548\u679c\u3002 \u4e3a\u6b64\uff0c\u6211\u4eec\u63d0\u4f9b\u4e86DataLoad CRD, \u8be5CRD\u8ba9\u4f60\u53ef\u901a\u8fc7\u7b80\u5355\u7684\u914d\u7f6e\u5c31\u80fd\u5b8c\u6210\u6574\u4e2a\u6570\u636e\u9884\u52a0\u8f7d\u8fc7\u7a0b\uff0c\u5e76\u5bf9\u6570\u636e\u9884\u52a0\u8f7d\u7684\u8bb8\u591a\u884c\u4e3a\u8fdb\u884c\u81ea\u5b9a\u4e49\u63a7\u5236\u3002 \u5047\u8bbe\u6211\u4eec\u5df2\u7ecf\u6709\u5982\u4e0b\u4e00\u4e2aname\u4e3a spark \u7684dataset\uff1a apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: spark spec: mounts: - mountPoint: oss://test-bucket/spark name: sparkV1 accessModes: - ReadWriteMany \u63a5\u4e0b\u6765\u5bf9\u8fd9\u4e2adataset\u8fdb\u884c\u9884\u70ed\uff0c\u8bf7\u6ce8\u610f\u4e0b\u6587\u4e2d\u7684 /sparkV1 \u4e3a mounts[].name \u7684\u503c\uff0c\u5982\u679c\u60a8\u7684 mounts[].path \u4e3a / , \u90a3\u4e48\u4e0b\u6587\u4e2d\u7684 /sparkV1 \u53ef\u4ee5\u76f4\u63a5\u6539\u4e3a / \u914d\u7f6e\u5f85\u521b\u5efa\u7684DataLoad\u5bf9\u8c61 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default target: - path: /sparkV1/ spec.dataset \u6307\u660e\u4e86\u9700\u8981\u8fdb\u884c\u6570\u636e\u9884\u52a0\u8f7d\u7684\u76ee\u6807\u6570\u636e\u96c6\uff0c\u5728\u8be5\u4f8b\u5b50\u4e2d\uff0c\u6211\u4eec\u7684\u6570\u636e\u9884\u52a0\u8f7d\u76ee\u6807\u4e3a default \u547d\u540d\u7a7a\u95f4\u4e0b\u540d\u4e3a spark \u7684\u6570\u636e\u96c6\uff0c\u5982\u679c\u8be5\u914d\u7f6e\u4e0e\u4f60\u6240\u5728\u7684\u5b9e\u9645\u73af\u5883\u4e0d\u7b26\uff0c\u8bf7\u6839\u636e\u4f60\u7684\u5b9e\u9645\u73af\u5883\u5bf9\u5176\u8fdb\u884c\u8c03\u6574\u3002 \u521b\u5efaDataLoad\u5bf9\u8c61 \u00a7 kubectl create -f dataload.yaml \u67e5\u770b\u521b\u5efa\u7684DataLoad\u5bf9\u8c61\u72b6\u6001 \u00a7 kubectl get dataload spark-dataload \u4e0a\u8ff0\u547d\u4ee4\u4f1a\u5f97\u5230\u7c7b\u4f3c\u4ee5\u4e0b\u7ed3\u679c\uff1a NAME DATASET PHASE AGE spark-dataload spark Loading 2m13s \u4f60\u4e5f\u53ef\u4ee5\u901a\u8fc7 kubectl describe \u83b7\u53d6\u6709\u5173\u8be5DataLoad\u7684\u66f4\u591a\u8be6\u7ec6\u4fe1\u606f\uff1a kubectl describe dataload spark-dataload \u5f97\u5230\u4ee5\u4e0b\u7ed3\u679c\uff1a Name: spark-dataload Namespace: default Labels: <none> Annotations: <none> API Version: data.fluid.io/v1alpha1 Kind: DataLoad ... Spec: Dataset: Name: spark Namespace: default Status: Conditions: Phase: Loading Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal DataLoadJobStarted 80s DataLoad The DataLoad job spark-dataload-loader-job started \u4e0a\u8ff0\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u6839\u636e\u4f60\u6240\u5728\u7684\u7f51\u7edc\u73af\u5883\u4e0d\u540c\uff0c\u53ef\u80fd\u4f1a\u8017\u8d39\u6570\u5206\u949f \u7b49\u5f85\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5b8c\u6210 \u00a7 kubectl get dataload spark-dataload \u4f60\u4f1a\u770b\u5230\u8be5DataLoad\u7684 Phase \u72b6\u6001\u5df2\u7ecf\u4ece Loading \u53d8\u4e3a Complete \uff0c\u8fd9\u8868\u660e\u6574\u4e2a\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5df2\u7ecf\u5b8c\u6210 NAME DATASET PHASE AGE spark-dataload spark Complete 5m17s \u6b64\u65f6\u518d\u6b21\u67e5\u770bDataset\u5bf9\u8c61\u7684\u7f13\u5b58\u72b6\u6001\uff1a kubectl get dataset spark \u53ef\u53d1\u73b0\uff0c\u8fdc\u7a0b\u5b58\u50a8\u7cfb\u7edf\u4e2d\u7684\u5168\u90e8\u6570\u636e\u5747\u5df2\u6210\u529f\u7f13\u5b58\u5230\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE spark 1.92GiB 1.92GiB 4.00GiB 100.0% Bound 7m41s DataLoad\u8fdb\u9636\u914d\u7f6e \u00a7 \u6307\u5b9a\u4e00\u4e2a\u6216\u591a\u4e2a\u6570\u636e\u96c6\u5b50\u76ee\u5f55\u8fdb\u884c\u52a0\u8f7d \u00a7 \u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6\u53ef\u4ee5\u52a0\u8f7d\u6307\u5b9a\u7684\u5b50\u76ee\u5f55(\u6216\u6587\u4ef6)\uff0c\u800c\u4e0d\u662f\u6574\u4e2a\u6570\u636e\u96c6\uff0c\u4f8b\u5982\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 - path: /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \u4e0a\u8ff0DataLoad\u4ec5\u4f1a\u52a0\u8f7d /sparkV1/spark-2.4.7 \u76ee\u5f55\u4e0b\u7684\u5168\u90e8\u6587\u4ef6\uff0c\u4ee5\u53ca /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \u6587\u4ef6 \u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf \u00a7 \u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6,\u4f60\u4e5f\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u63a7\u5236\u52a0\u8f7d\u7684\u6570\u636e\u526f\u672c\u6570\u91cf\uff0c\u4f8b\u5982\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 replicas: 1 - path: /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz replicas: 2 \u4e0a\u8ff0DataLoad\u5728\u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6\uff0c\u5bf9\u4e8e /sparkV1/spark-2.4.7 \u76ee\u5f55\u4e0b\u7684\u6587\u4ef6\u4ec5\u4f1a\u5728\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d\u4fdd\u7559 1\u4efd \u6570\u636e\u7f13\u5b58\u526f\u672c\uff0c\u800c\u5bf9\u4e8e\u6587\u4ef6 /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \uff0c\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u5c06\u4f1a\u4fdd\u7559 2\u4efd \u7f13\u5b58\u526f\u672c\u3002 \u539f\u5b50\u6027\u7f13\u5b58 \u00a7 \u5982\u60a8\u60f3\u4fdd\u6301\u7f13\u5b58\u4e00\u81f4\u6027\u548c\u539f\u5b50\u6027\uff0c\u90a3\u4e48\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b yaml \u6765\u5b8c\u6210\u8fd9\u4e2a\u529f\u80fd apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default options: atomicCache: \"true\" \u5355\u72ec\u8fdb\u884c\u5143\u6570\u636e\u7f13\u5b58\uff0c\u4e0d\u505a\u6570\u636e\u7f13\u5b58 \u00a7 \u5728\u8fdb\u884c dataload \u9884\u70ed\u65f6\uff0c\u9ed8\u8ba4\u662f\u4f1a\u505a\u6570\u636e\u7f13\u5b58\uff0c\u539f\u6570\u636e\u7f13\u5b58\u662f\u53ef\u9009\uff0c\u4f46\u5982\u679c\u60a8\u53ea\u60f3\u5143\u6570\u636e\u7f13\u5b58\u4e0d\u60f3\u505a\u6570\u636e\u7f13\u5b58\u7684\u8bdd\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b yaml \u6253\u5f00\u53ea\u505a\u5143\u6570\u636e\u7f13\u5b58\u7684\u5f00\u5173 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default options: loadMetadataOnly: \"true\" \u901a\u8fc7\u901a\u914d\u7b26\u8fdb\u884c\u7f13\u5b58\u8fc7\u6ee4 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 replicas: 1 options: filter: \"*.tgz\" cacheThread: \"2\" dryrun: \"false\" \u4f7f\u7528\u5217\u8868\u6587\u4ef6\u8fdb\u884c\u7f13\u5b58 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true options: accessKeyId: \"xxx\" accessKeySecret: \"xxx\" endpoint: \"oss-cn-shanghai-internal.aliyuncs.com\" url: \"oss://test-bucket/filelist\" dryrun: \"false\" \u7f13\u5b58\u8fdb\u5ea6\u53caLOG\u67e5\u770b \u00a7 $ kubectl get dataload spark-dataload NAME DATASET PHASE AGE spark-dataload spark Loading 2m13s \u67e5\u770b\u5f53\u524d\u7f13\u5b58\u7684\u76f8\u5173\u4fe1\u606f kubectl logs spark-dataload \u73af\u5883\u6e05\u7406 \u00a7 $ kubectl delete dataload spark-dataload","title":"\u6570\u636e\u9884\u52a0\u8f7d"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#_1","text":"\u914d\u7f6e\u5f85\u521b\u5efa\u7684DataLoad\u5bf9\u8c61 \u521b\u5efaDataLoad\u5bf9\u8c61 \u67e5\u770b\u521b\u5efa\u7684DataLoad\u5bf9\u8c61\u72b6\u6001 \u7b49\u5f85\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5b8c\u6210 DataLoad\u8fdb\u9636\u914d\u7f6e \u6307\u5b9a\u4e00\u4e2a\u6216\u591a\u4e2a\u6570\u636e\u96c6\u5b50\u76ee\u5f55\u8fdb\u884c\u52a0\u8f7d \u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf \u539f\u5b50\u6027\u7f13\u5b58 \u5355\u72ec\u8fdb\u884c\u5143\u6570\u636e\u7f13\u5b58\uff0c\u4e0d\u505a\u6570\u636e\u7f13\u5b58 \u7f13\u5b58\u8fdb\u5ea6\u53caLOG\u67e5\u770b \u73af\u5883\u6e05\u7406 \u4e3a\u4e86\u4fdd\u8bc1\u5e94\u7528\u5728\u8bbf\u95ee\u6570\u636e\u65f6\u7684\u6027\u80fd\uff0c\u53ef\u4ee5\u901a\u8fc7 \u6570\u636e\u9884\u52a0\u8f7d \u63d0\u524d\u5c06\u8fdc\u7a0b\u5b58\u50a8\u7cfb\u7edf\u4e2d\u7684\u6570\u636e\u62c9\u53d6\u5230\u9760\u8fd1\u8ba1\u7b97\u7ed3\u70b9\u7684\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d\uff0c\u4f7f\u5f97\u6d88\u8d39\u8be5\u6570\u636e\u96c6\u7684\u5e94\u7528\u80fd\u591f\u5728\u9996\u6b21\u8fd0\u884c\u65f6\u5373\u53ef\u4eab\u53d7\u5230\u7f13\u5b58\u5e26\u6765\u7684\u52a0\u901f\u6548\u679c\u3002 \u4e3a\u6b64\uff0c\u6211\u4eec\u63d0\u4f9b\u4e86DataLoad CRD, \u8be5CRD\u8ba9\u4f60\u53ef\u901a\u8fc7\u7b80\u5355\u7684\u914d\u7f6e\u5c31\u80fd\u5b8c\u6210\u6574\u4e2a\u6570\u636e\u9884\u52a0\u8f7d\u8fc7\u7a0b\uff0c\u5e76\u5bf9\u6570\u636e\u9884\u52a0\u8f7d\u7684\u8bb8\u591a\u884c\u4e3a\u8fdb\u884c\u81ea\u5b9a\u4e49\u63a7\u5236\u3002 \u5047\u8bbe\u6211\u4eec\u5df2\u7ecf\u6709\u5982\u4e0b\u4e00\u4e2aname\u4e3a spark \u7684dataset\uff1a apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: spark spec: mounts: - mountPoint: oss://test-bucket/spark name: sparkV1 accessModes: - ReadWriteMany \u63a5\u4e0b\u6765\u5bf9\u8fd9\u4e2adataset\u8fdb\u884c\u9884\u70ed\uff0c\u8bf7\u6ce8\u610f\u4e0b\u6587\u4e2d\u7684 /sparkV1 \u4e3a mounts[].name \u7684\u503c\uff0c\u5982\u679c\u60a8\u7684 mounts[].path \u4e3a / , \u90a3\u4e48\u4e0b\u6587\u4e2d\u7684 /sparkV1 \u53ef\u4ee5\u76f4\u63a5\u6539\u4e3a /","title":"\u6570\u636e\u9884\u52a0\u8f7d"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#dataload","text":"apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default target: - path: /sparkV1/ spec.dataset \u6307\u660e\u4e86\u9700\u8981\u8fdb\u884c\u6570\u636e\u9884\u52a0\u8f7d\u7684\u76ee\u6807\u6570\u636e\u96c6\uff0c\u5728\u8be5\u4f8b\u5b50\u4e2d\uff0c\u6211\u4eec\u7684\u6570\u636e\u9884\u52a0\u8f7d\u76ee\u6807\u4e3a default \u547d\u540d\u7a7a\u95f4\u4e0b\u540d\u4e3a spark \u7684\u6570\u636e\u96c6\uff0c\u5982\u679c\u8be5\u914d\u7f6e\u4e0e\u4f60\u6240\u5728\u7684\u5b9e\u9645\u73af\u5883\u4e0d\u7b26\uff0c\u8bf7\u6839\u636e\u4f60\u7684\u5b9e\u9645\u73af\u5883\u5bf9\u5176\u8fdb\u884c\u8c03\u6574\u3002","title":"\u914d\u7f6e\u5f85\u521b\u5efa\u7684DataLoad\u5bf9\u8c61"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#dataload_1","text":"kubectl create -f dataload.yaml","title":"\u521b\u5efaDataLoad\u5bf9\u8c61"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#dataload_2","text":"kubectl get dataload spark-dataload \u4e0a\u8ff0\u547d\u4ee4\u4f1a\u5f97\u5230\u7c7b\u4f3c\u4ee5\u4e0b\u7ed3\u679c\uff1a NAME DATASET PHASE AGE spark-dataload spark Loading 2m13s \u4f60\u4e5f\u53ef\u4ee5\u901a\u8fc7 kubectl describe \u83b7\u53d6\u6709\u5173\u8be5DataLoad\u7684\u66f4\u591a\u8be6\u7ec6\u4fe1\u606f\uff1a kubectl describe dataload spark-dataload \u5f97\u5230\u4ee5\u4e0b\u7ed3\u679c\uff1a Name: spark-dataload Namespace: default Labels: <none> Annotations: <none> API Version: data.fluid.io/v1alpha1 Kind: DataLoad ... Spec: Dataset: Name: spark Namespace: default Status: Conditions: Phase: Loading Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal DataLoadJobStarted 80s DataLoad The DataLoad job spark-dataload-loader-job started \u4e0a\u8ff0\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u6839\u636e\u4f60\u6240\u5728\u7684\u7f51\u7edc\u73af\u5883\u4e0d\u540c\uff0c\u53ef\u80fd\u4f1a\u8017\u8d39\u6570\u5206\u949f","title":"\u67e5\u770b\u521b\u5efa\u7684DataLoad\u5bf9\u8c61\u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#_2","text":"kubectl get dataload spark-dataload \u4f60\u4f1a\u770b\u5230\u8be5DataLoad\u7684 Phase \u72b6\u6001\u5df2\u7ecf\u4ece Loading \u53d8\u4e3a Complete \uff0c\u8fd9\u8868\u660e\u6574\u4e2a\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5df2\u7ecf\u5b8c\u6210 NAME DATASET PHASE AGE spark-dataload spark Complete 5m17s \u6b64\u65f6\u518d\u6b21\u67e5\u770bDataset\u5bf9\u8c61\u7684\u7f13\u5b58\u72b6\u6001\uff1a kubectl get dataset spark \u53ef\u53d1\u73b0\uff0c\u8fdc\u7a0b\u5b58\u50a8\u7cfb\u7edf\u4e2d\u7684\u5168\u90e8\u6570\u636e\u5747\u5df2\u6210\u529f\u7f13\u5b58\u5230\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE spark 1.92GiB 1.92GiB 4.00GiB 100.0% Bound 7m41s","title":"\u7b49\u5f85\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5b8c\u6210"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#dataload_3","text":"","title":"DataLoad\u8fdb\u9636\u914d\u7f6e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#_3","text":"\u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6\u53ef\u4ee5\u52a0\u8f7d\u6307\u5b9a\u7684\u5b50\u76ee\u5f55(\u6216\u6587\u4ef6)\uff0c\u800c\u4e0d\u662f\u6574\u4e2a\u6570\u636e\u96c6\uff0c\u4f8b\u5982\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 - path: /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \u4e0a\u8ff0DataLoad\u4ec5\u4f1a\u52a0\u8f7d /sparkV1/spark-2.4.7 \u76ee\u5f55\u4e0b\u7684\u5168\u90e8\u6587\u4ef6\uff0c\u4ee5\u53ca /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \u6587\u4ef6","title":"\u6307\u5b9a\u4e00\u4e2a\u6216\u591a\u4e2a\u6570\u636e\u96c6\u5b50\u76ee\u5f55\u8fdb\u884c\u52a0\u8f7d"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#_4","text":"\u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6,\u4f60\u4e5f\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u63a7\u5236\u52a0\u8f7d\u7684\u6570\u636e\u526f\u672c\u6570\u91cf\uff0c\u4f8b\u5982\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 replicas: 1 - path: /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz replicas: 2 \u4e0a\u8ff0DataLoad\u5728\u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6\uff0c\u5bf9\u4e8e /sparkV1/spark-2.4.7 \u76ee\u5f55\u4e0b\u7684\u6587\u4ef6\u4ec5\u4f1a\u5728\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d\u4fdd\u7559 1\u4efd \u6570\u636e\u7f13\u5b58\u526f\u672c\uff0c\u800c\u5bf9\u4e8e\u6587\u4ef6 /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \uff0c\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u5c06\u4f1a\u4fdd\u7559 2\u4efd \u7f13\u5b58\u526f\u672c\u3002","title":"\u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#_5","text":"\u5982\u60a8\u60f3\u4fdd\u6301\u7f13\u5b58\u4e00\u81f4\u6027\u548c\u539f\u5b50\u6027\uff0c\u90a3\u4e48\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b yaml \u6765\u5b8c\u6210\u8fd9\u4e2a\u529f\u80fd apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default options: atomicCache: \"true\"","title":"\u539f\u5b50\u6027\u7f13\u5b58"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#_6","text":"\u5728\u8fdb\u884c dataload \u9884\u70ed\u65f6\uff0c\u9ed8\u8ba4\u662f\u4f1a\u505a\u6570\u636e\u7f13\u5b58\uff0c\u539f\u6570\u636e\u7f13\u5b58\u662f\u53ef\u9009\uff0c\u4f46\u5982\u679c\u60a8\u53ea\u60f3\u5143\u6570\u636e\u7f13\u5b58\u4e0d\u60f3\u505a\u6570\u636e\u7f13\u5b58\u7684\u8bdd\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b yaml \u6253\u5f00\u53ea\u505a\u5143\u6570\u636e\u7f13\u5b58\u7684\u5f00\u5173 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default options: loadMetadataOnly: \"true\"","title":"\u5355\u72ec\u8fdb\u884c\u5143\u6570\u636e\u7f13\u5b58\uff0c\u4e0d\u505a\u6570\u636e\u7f13\u5b58"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#_7","text":"apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 replicas: 1 options: filter: \"*.tgz\" cacheThread: \"2\" dryrun: \"false\"","title":"\u901a\u8fc7\u901a\u914d\u7b26\u8fdb\u884c\u7f13\u5b58\u8fc7\u6ee4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#_8","text":"apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true options: accessKeyId: \"xxx\" accessKeySecret: \"xxx\" endpoint: \"oss-cn-shanghai-internal.aliyuncs.com\" url: \"oss://test-bucket/filelist\" dryrun: \"false\"","title":"\u4f7f\u7528\u5217\u8868\u6587\u4ef6\u8fdb\u884c\u7f13\u5b58"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#log","text":"$ kubectl get dataload spark-dataload NAME DATASET PHASE AGE spark-dataload spark Loading 2m13s \u67e5\u770b\u5f53\u524d\u7f13\u5b58\u7684\u76f8\u5173\u4fe1\u606f kubectl logs spark-dataload","title":"\u7f13\u5b58\u8fdb\u5ea6\u53caLOG\u67e5\u770b"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#_9","text":"$ kubectl delete dataload spark-dataload","title":"\u73af\u5883\u6e05\u7406"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_download/","text":"\u4e0b\u8f7d 1.0.0-20230602 0.8.0-20221105 0.8.0-20220729 0.8.0-20220630 0.8.0-20220625 0.8.0-20220606 0.8.0-20220526 \u4e0b\u8f7d \u00a7 1.0.0-20230602 \u00a7 \u66f4\u65b0\u65e5\u671f\uff1a2022-11-05 Fluid 1.0.0 JindoRuntime\u76f8\u5173\u529f\u80fd\u5347\u7ea7\uff1a * JindoFSx \u5347\u7ea7\u5230 4.6.12 0.8.0-20221105 \u00a7 \u66f4\u65b0\u65e5\u671f\uff1a2022-11-05 Fluid 0.8.0 JindoRuntime\u76f8\u5173\u529f\u80fd\u5347\u7ea7\uff1a * JindoFSx \u5347\u7ea7\u5230 4.5.2 0.8.0-20220729 \u00a7 \u66f4\u65b0\u65e5\u671f\uff1a2022-07-29 Fluid 0.8.0 commitId:e7d4c18 JindoRuntime\u76f8\u5173\u529f\u80fd\u5347\u7ea7\uff1a * JindoFSx \u5347\u7ea7\u5230 4.5.1 0.8.0-20220630 \u00a7 \u66f4\u65b0\u65e5\u671f\uff1a2022-06-30 Fluid 0.8.0 commitId:32837a3 0.8.0-20220625 \u00a7 \u66f4\u65b0\u65e5\u671f\uff1a2022-06-25 Fluid 0.8.0 commitId:510b30b 0.8.0-20220606 \u00a7 \u66f4\u65b0\u65e5\u671f\uff1a2022-05-26 Fluid 0.8.0 commitId:e305ff1 0.8.0-20220526 \u00a7 \u66f4\u65b0\u65e5\u671f\uff1a2022-05-26 Fluid 0.8.0 commitId:e562a90 JindoRuntime\u76f8\u5173\u529f\u80fd\u5347\u7ea7\uff1a * JindoFSx \u5347\u7ea7\u5230 4.6.12","title":"Jindo fluid download"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_download/#_1","text":"","title":"\u4e0b\u8f7d"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_download/#100-20230602","text":"\u66f4\u65b0\u65e5\u671f\uff1a2022-11-05 Fluid 1.0.0 JindoRuntime\u76f8\u5173\u529f\u80fd\u5347\u7ea7\uff1a * JindoFSx \u5347\u7ea7\u5230 4.6.12","title":"1.0.0-20230602"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_download/#080-20221105","text":"\u66f4\u65b0\u65e5\u671f\uff1a2022-11-05 Fluid 0.8.0 JindoRuntime\u76f8\u5173\u529f\u80fd\u5347\u7ea7\uff1a * JindoFSx \u5347\u7ea7\u5230 4.5.2","title":"0.8.0-20221105"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_download/#080-20220729","text":"\u66f4\u65b0\u65e5\u671f\uff1a2022-07-29 Fluid 0.8.0 commitId:e7d4c18 JindoRuntime\u76f8\u5173\u529f\u80fd\u5347\u7ea7\uff1a * JindoFSx \u5347\u7ea7\u5230 4.5.1","title":"0.8.0-20220729"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_download/#080-20220630","text":"\u66f4\u65b0\u65e5\u671f\uff1a2022-06-30 Fluid 0.8.0 commitId:32837a3","title":"0.8.0-20220630"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_download/#080-20220625","text":"\u66f4\u65b0\u65e5\u671f\uff1a2022-06-25 Fluid 0.8.0 commitId:510b30b","title":"0.8.0-20220625"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_download/#080-20220606","text":"\u66f4\u65b0\u65e5\u671f\uff1a2022-05-26 Fluid 0.8.0 commitId:e305ff1","title":"0.8.0-20220606"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_download/#080-20220526","text":"\u66f4\u65b0\u65e5\u671f\uff1a2022-05-26 Fluid 0.8.0 commitId:e562a90 JindoRuntime\u76f8\u5173\u529f\u80fd\u5347\u7ea7\uff1a * JindoFSx \u5347\u7ea7\u5230 4.6.12","title":"0.8.0-20220526"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/","text":"\u52a0\u901fHDFS\u4e0a\u6570\u636e \u00a7 JindoFSx \u63d0\u4f9b\u4e86\u8bbf\u95ee HDFS \u7684\u80fd\u529b\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e HDFS \u4f5c\u4e3a JindoFSx \u7684\u540e\u7aef\u5b58\u50a8\uff0c\u4f7f Fluid \u901a\u8fc7 JindoFSx \u6765\u8bbf\u95ee HDFS \u4e0a\u7684\u6570\u636e\uff0c\u540c\u65f6 JindoFSx \u4e5f\u63d0\u4f9b\u4e86\u5bf9 HDFS \u4e0a\u7684\u6570\u636e\u4ee5\u53ca\u5143\u6570\u636e\u7684\u7f13\u5b58\u52a0\u901f\u529f\u80fd\u3002 \u672c\u6587\u6863\u5c55\u793a\u4e86\u5982\u4f55\u5728 Fluid \u4e2d\u4ee5\u58f0\u660e\u5f0f\u7684\u65b9\u5f0f\u5b8c\u6210 JindoFSx \u90e8\u7f72\uff0c\u5bf9\u63a5 HDFS \u6570\u636e\u6e90\u3002 \u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 \u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz \u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 1\u3001\u521b\u5efa dataset \u548c JindoRuntime \u00a7 1.1\u3001HA \u96c6\u7fa4\u4ece HDFS \u914d\u7f6e\u6587\u4ef6\u4e2d\u76f8\u5173\u4fe1\u606f\uff0c\u975e HA \u96c6\u7fa4\u8bf7\u7565\u8fc7\u6b64\u6b65\u9aa4 \u00a7 \u4ece HDFS \u96c6\u7fa4\u4e0a\u627e\u5230 hdfs-site.xml \u914d\u7f6e\u6587\u4ef6 * \u963f\u91cc\u4e91EMR\u96c6\u7fa4\u5728 /etc/ecm/hadoop-conf \u8def\u5f84\u4e0b * CDH \u96c6\u7fa4\u53ef\u53c2\u8003 What-is-the-Path-of-hdfs-site-xml-core-xml \u9700\u8981\u5c06\u5176\u4e2d rpc \u7684\u8bbf\u95ee\u5730\u5740\u4e2d\u57df\u540d\u7684\u90e8\u5206\u53c2\u6570\uff0c\u4fee\u6539\u6210\u53ef\u8bbf\u95ee\u7684\u5185\u7f51 IP \u5730\u5740\uff0c\u4f8b\u5982\uff1a <property> <name>dfs.namenode.rpc-address.emr-cluster.nn1</name> <value>emr-header-1.cluster-xxx:8020</value> </property> <property> <name>dfs.namenode.rpc-address.emr-cluster.nn2</name> <value>emr-header-2.cluster-xxx:8020</value> </property> \u4fee\u6539\u4e3a <property> <name>dfs.namenode.rpc-address.emr-cluster.nn1</name> <value>192.168.0.1:8020</value> </property> <property> <name>dfs.namenode.rpc-address.emr-cluster.nn2</name> <value>192.168.0.2:8020</value> </property> \u4ece\u914d\u7f6e\u6587\u4ef6\u91cc\u6211\u4eec\u53ef\u4ee5\u62ff\u5230\u5982\u4e0b\u51e0\u4e2a\u4fe1\u606f \u96c6\u7fa4namespaceService\u540d\u79f0\uff1aemr-cluster \u4e24\u4e2anamenode\u540d\u5b57\uff1ann1 \u548c nn2 \u4e24\u4e2anamenode\u7684\u8bbf\u95ee\u5730\u5740\uff1a192.168.0.1:8020 \u548c 192.168.0.2:8020 1.2\u3001\u521b\u5efa Dataset \u8d44\u6e90\u5bf9\u8c61 \u00a7 \u9996\u5148\u521b\u5efa\u4e00\u4e2a\u540d\u4e3a dataset.yaml \u7684\u6587\u4ef6\uff0c\u9488\u5bf9 HDFS \u96c6\u7fa4\u5206\u4e3a HA \u96c6\u7fa4\u548c\u975e HA \u96c6\u7fa4\u4e24\u79cd\u4e0d\u540c\u7684\u60c5\u51b5\u3002 HA \u96c6\u7fa4 \u00a7 \u5c061.1\u4e2d\u62ff\u5230\u7684\u51e0\u4e2a\u4fe1\u606f\u653e\u5230 mounts.options \u91cc\u9762\uff0cuser \u4ee3\u8868\u7528\u6237\u6743\u9650 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hdfs spec: mounts: - mountPoint: hdfs://emr-cluster/ name: hdfs options: fs.hdfs.emr-cluster.dfs.ha.namenodes = nn1,nn2 fs.hdfs.emr-cluster.dfs.namenode.rpc-address.nn1 = 192.168.0.1:8020 fs.hdfs.emr-cluster.dfs.namenode.rpc-address.nn2 = 192.168.0.2:8020 fs.hdfs.user = hadoop path: / accessModes: - ReadOnlyMany \u5176\u4e2d mountPoint \u4e3a\u9700\u8981\u8bbf\u95ee\u7684 HDFS \u8def\u5f84\uff08\u963f\u91cc\u4e91EMR\u96c6\u7fa4\u9ed8\u8ba4\u540d\u5b57\u4e3a emr-cluster\uff0c\u5982\u4e0b\u6240\u793a\uff09 - accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 <property> <name>fs.defaultFS</name> <value>hdfs://emr-cluster</value> </property> \u975e HA \u96c6\u7fa4 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hdfs spec: mounts: - mountPoint: hdfs://<namenodeIP>:<port>/ name: hdfs path: / accessModes: - ReadOnlyMany \u5176\u4e2d mountPoint \u4e3a HDFS \u96c6\u7fa4\u7684 namenode \u7684 IP \u5730\u5740\u548c\u7aef\u53e3\u7684\u7ec4\u5408 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 \u00a7 1.3\u3001\u521b\u5efa Dataset \u00a7 $ kubectl create -f dataset.yaml 1.4 \u521b\u5efa JindoRuntime \u8d44\u6e90\u5bf9\u8c61 \u00a7 \u521b\u5efa\u4e00\u4e2a JindoRuntime \u5bf9\u5e94\u7684 JindoFSx \u96c6\u7fa4 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hdfs spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /mnt/disk1/ quota: 290Gi high: \"0.9\" low: \"0.2\" \u591a\u5757\u76d8\u591a\u4e2aquota\u8bf7\u53c2\u8003 path: /mnt/disk1/,/mnt/disk2/,/mnt/disk3/ quotaList: 290G,290G,290G \u5176\u4e2d path \u548c quota \u7684\u6570\u91cf\u5e94\u8be5\u76f8\u540c\u3002 1.5\u3001\u521b\u5efa JindoRuntime \u00a7 \u6267\u884c create \u547d\u4ee4 $ kubectl create -f runtime.yaml \u67e5\u770b dataset \u72b6\u6001 \u00a7 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hdfs 180.01GiB 0.00B 261.00GiB 0.0% Bound 62m \u68c0\u67e5 PV\uff0cPVC \u521b\u5efa\u60c5\u51b5 \u00a7 $ kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hdfs 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hdfs Bound hadoop 100Gi RWX 58m \u68c0\u67e5 jindoruntime \u90e8\u7f72\u60c5\u51b5 \u00a7 $ kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hdfs Ready Ready 62m","title":"\u52a0\u901fHDFS\u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#hdfs","text":"JindoFSx \u63d0\u4f9b\u4e86\u8bbf\u95ee HDFS \u7684\u80fd\u529b\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e HDFS \u4f5c\u4e3a JindoFSx \u7684\u540e\u7aef\u5b58\u50a8\uff0c\u4f7f Fluid \u901a\u8fc7 JindoFSx \u6765\u8bbf\u95ee HDFS \u4e0a\u7684\u6570\u636e\uff0c\u540c\u65f6 JindoFSx \u4e5f\u63d0\u4f9b\u4e86\u5bf9 HDFS \u4e0a\u7684\u6570\u636e\u4ee5\u53ca\u5143\u6570\u636e\u7684\u7f13\u5b58\u52a0\u901f\u529f\u80fd\u3002 \u672c\u6587\u6863\u5c55\u793a\u4e86\u5982\u4f55\u5728 Fluid \u4e2d\u4ee5\u58f0\u660e\u5f0f\u7684\u65b9\u5f0f\u5b8c\u6210 JindoFSx \u90e8\u7f72\uff0c\u5bf9\u63a5 HDFS \u6570\u636e\u6e90\u3002","title":"\u52a0\u901fHDFS\u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#_1","text":"kubectl create ns fluid-system","title":"\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#fluid","text":"","title":"\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#fluid_1","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#1-dataset-jindoruntime","text":"","title":"1\u3001\u521b\u5efa dataset \u548c JindoRuntime"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#11ha-hdfs-ha","text":"\u4ece HDFS \u96c6\u7fa4\u4e0a\u627e\u5230 hdfs-site.xml \u914d\u7f6e\u6587\u4ef6 * \u963f\u91cc\u4e91EMR\u96c6\u7fa4\u5728 /etc/ecm/hadoop-conf \u8def\u5f84\u4e0b * CDH \u96c6\u7fa4\u53ef\u53c2\u8003 What-is-the-Path-of-hdfs-site-xml-core-xml \u9700\u8981\u5c06\u5176\u4e2d rpc \u7684\u8bbf\u95ee\u5730\u5740\u4e2d\u57df\u540d\u7684\u90e8\u5206\u53c2\u6570\uff0c\u4fee\u6539\u6210\u53ef\u8bbf\u95ee\u7684\u5185\u7f51 IP \u5730\u5740\uff0c\u4f8b\u5982\uff1a <property> <name>dfs.namenode.rpc-address.emr-cluster.nn1</name> <value>emr-header-1.cluster-xxx:8020</value> </property> <property> <name>dfs.namenode.rpc-address.emr-cluster.nn2</name> <value>emr-header-2.cluster-xxx:8020</value> </property> \u4fee\u6539\u4e3a <property> <name>dfs.namenode.rpc-address.emr-cluster.nn1</name> <value>192.168.0.1:8020</value> </property> <property> <name>dfs.namenode.rpc-address.emr-cluster.nn2</name> <value>192.168.0.2:8020</value> </property> \u4ece\u914d\u7f6e\u6587\u4ef6\u91cc\u6211\u4eec\u53ef\u4ee5\u62ff\u5230\u5982\u4e0b\u51e0\u4e2a\u4fe1\u606f \u96c6\u7fa4namespaceService\u540d\u79f0\uff1aemr-cluster \u4e24\u4e2anamenode\u540d\u5b57\uff1ann1 \u548c nn2 \u4e24\u4e2anamenode\u7684\u8bbf\u95ee\u5730\u5740\uff1a192.168.0.1:8020 \u548c 192.168.0.2:8020","title":"1.1\u3001HA \u96c6\u7fa4\u4ece HDFS \u914d\u7f6e\u6587\u4ef6\u4e2d\u76f8\u5173\u4fe1\u606f\uff0c\u975e HA \u96c6\u7fa4\u8bf7\u7565\u8fc7\u6b64\u6b65\u9aa4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#12-dataset","text":"\u9996\u5148\u521b\u5efa\u4e00\u4e2a\u540d\u4e3a dataset.yaml \u7684\u6587\u4ef6\uff0c\u9488\u5bf9 HDFS \u96c6\u7fa4\u5206\u4e3a HA \u96c6\u7fa4\u548c\u975e HA \u96c6\u7fa4\u4e24\u79cd\u4e0d\u540c\u7684\u60c5\u51b5\u3002","title":"1.2\u3001\u521b\u5efa Dataset \u8d44\u6e90\u5bf9\u8c61"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#ha","text":"\u5c061.1\u4e2d\u62ff\u5230\u7684\u51e0\u4e2a\u4fe1\u606f\u653e\u5230 mounts.options \u91cc\u9762\uff0cuser \u4ee3\u8868\u7528\u6237\u6743\u9650 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hdfs spec: mounts: - mountPoint: hdfs://emr-cluster/ name: hdfs options: fs.hdfs.emr-cluster.dfs.ha.namenodes = nn1,nn2 fs.hdfs.emr-cluster.dfs.namenode.rpc-address.nn1 = 192.168.0.1:8020 fs.hdfs.emr-cluster.dfs.namenode.rpc-address.nn2 = 192.168.0.2:8020 fs.hdfs.user = hadoop path: / accessModes: - ReadOnlyMany \u5176\u4e2d mountPoint \u4e3a\u9700\u8981\u8bbf\u95ee\u7684 HDFS \u8def\u5f84\uff08\u963f\u91cc\u4e91EMR\u96c6\u7fa4\u9ed8\u8ba4\u540d\u5b57\u4e3a emr-cluster\uff0c\u5982\u4e0b\u6240\u793a\uff09 - accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 <property> <name>fs.defaultFS</name> <value>hdfs://emr-cluster</value> </property>","title":"HA \u96c6\u7fa4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#ha_1","text":"apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hdfs spec: mounts: - mountPoint: hdfs://<namenodeIP>:<port>/ name: hdfs path: / accessModes: - ReadOnlyMany \u5176\u4e2d mountPoint \u4e3a HDFS \u96c6\u7fa4\u7684 namenode \u7684 IP \u5730\u5740\u548c\u7aef\u53e3\u7684\u7ec4\u5408 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199","title":"\u975e HA \u96c6\u7fa4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#_2","text":"","title":""},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#13-dataset","text":"$ kubectl create -f dataset.yaml","title":"1.3\u3001\u521b\u5efa Dataset"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#14-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a JindoRuntime \u5bf9\u5e94\u7684 JindoFSx \u96c6\u7fa4 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hdfs spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /mnt/disk1/ quota: 290Gi high: \"0.9\" low: \"0.2\" \u591a\u5757\u76d8\u591a\u4e2aquota\u8bf7\u53c2\u8003 path: /mnt/disk1/,/mnt/disk2/,/mnt/disk3/ quotaList: 290G,290G,290G \u5176\u4e2d path \u548c quota \u7684\u6570\u91cf\u5e94\u8be5\u76f8\u540c\u3002","title":"1.4 \u521b\u5efa JindoRuntime \u8d44\u6e90\u5bf9\u8c61"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#15-jindoruntime","text":"\u6267\u884c create \u547d\u4ee4 $ kubectl create -f runtime.yaml","title":"1.5\u3001\u521b\u5efa JindoRuntime"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#dataset","text":"$ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hdfs 180.01GiB 0.00B 261.00GiB 0.0% Bound 62m","title":"\u67e5\u770b dataset \u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#pvpvc","text":"$ kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hdfs 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hdfs Bound hadoop 100Gi RWX 58m","title":"\u68c0\u67e5 PV\uff0cPVC \u521b\u5efa\u60c5\u51b5"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#jindoruntime","text":"$ kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hdfs Ready Ready 62m","title":"\u68c0\u67e5 jindoruntime \u90e8\u7f72\u60c5\u51b5"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_monitor_metric/","text":"\u76d1\u63a7\u548c\u53ef\u89c2\u6d4b\u6027\u529f\u80fd \u00a7 \u672c\u6587\u4ecb\u7ecd\u5728\u542f\u52a8 Fluid JindoRuntime \u7684\u524d\u63d0\u4e0b\uff0c\u901a\u8fc7 Prometheus + Grafana \u6536\u96c6\u7f13\u5b58\u7cfb\u7edf\u76f8\u5173\u7684 metrics \u6307\u6807\uff0c\u8fdb\u884c\u53ef\u89c6\u5316\u5c55\u793a\u7684\u4f7f\u7528\u65b9\u6cd5\u3002 \u4f7f\u7528 In-Cluster \u6a21\u5f0f\u90e8\u7f72 prometheus \u548c grafana \u76d1\u63a7 Fluid \u5e94\u7528 \u00a7 1. \u90e8\u7f72\u6216\u914d\u7f6e Prometheus \u00a7 \u5982\u679c\u96c6\u7fa4\u5185\u65e0 prometheus\uff0c\u8bf7\u70b9\u51fb\u4e0b\u8f7d prometheus-jindofs.yaml \u6587\u4ef6 $ kubectl apply -f prometheus-jindofs.yaml \u5982\u96c6\u7fa4\u5185\u6709 prometheus,\u53ef\u5c06\u4ee5\u4e0b\u914d\u7f6e\u5199\u5230 prometheus \u914d\u7f6e\u6587\u4ef6\u4e2d: scrape_configs: - job_name: 'jindo runtime' metrics_path: /jindometrics/prometheus kubernetes_sd_configs: - role: endpoints relabel_configs: - source_labels: [__meta_kubernetes_service_label_role] regex: jindofs-master action: keep - source_labels: [__meta_kubernetes_endpoint_port_name] regex: rpc action: keep - source_labels: [__meta_kubernetes_namespace] target_label: namespace replacement: $1 action: replace - source_labels: [__meta_kubernetes_service_label_release] target_label: fluid_runtime replacement: $1 action: replace - source_labels: [__meta_kubernetes_endpoint_address_target_name] target_label: pod replacement: $1 action: replace 2. \u90e8\u7f72 grafana \u00a7 \u5982\u679c\u96c6\u7fa4\u5185\u65e0 grafana\uff0c\u8bf7\u70b9\u51fb\u4e0b\u8f7d grafana.yaml \u6587\u4ef6 $ kubectl apply -f grafana.yaml 3. \u914d\u7f6e grafana \u00a7 \u767b\u5f55 grafana \u627e\u5230 grafana \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\u548c\u6539\u8282\u70b9\u5bf9\u5916\u53ef\u8bbf\u95ee grafana-node-ip \uff0c\u5982\u679c\u60f3\u8981\u5728\u516c\u7f51\u8bbf\u95ee\uff0c\u53ef\u4ee5\u7ed1\u5b9a\u516c\u7f51IP\uff0c $ kubectl get pod -n kube-system -o wide | grep grafana monitoring-grafana-7dfcf6d5cd-5k8qw 1/1 Running 0 108m 10.54.1.11 cn-shanghai.192.168.1.1 <none> <none> \u627e\u5230 grafana \u670d\u52a1\u5bf9\u5e94\u7684 NodePort $ kubectl describe svc monitoring-grafana -n kube-system Name: monitoring-grafana Namespace: kube-system Labels: <none> Annotations: <none> Selector: app=grafana Type: NodePort IP Families: <none> IP: 172.16.83.188 IPs: 172.16.83.188 Port: <unset> 80/TCP TargetPort: 3000/TCP NodePort: <unset> 30396/TCP Endpoints: 10.54.1.75:3000 Session Affinity: None External Traffic Policy: Cluster Events: <none> \u53ef\u4ee5\u770b\u5230\u5f53\u524d\u66b4\u9732\u7684 NodePort \u4e3a 30396 \u8bbf\u95ee http://$grafana-node-ip:NodePort \uff0c\u9ed8\u8ba4\u8d26\u53f7\u5bc6\u7801 admin:admin \u67e5\u770b prometheus svc \u7aef\u53e3 \u540c\u6837\u5148\u627e\u5230 prometheus \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\uff0c\u627e\u5230\u8be5\u8282\u70b9\u7684 IP $ kubectl get pod -n kube-system -o wide | grep grafana prometheus-deployment-d6d8554b4-wfl98 1/1 Running 0 17d 10.54.1.204 cn-shanghai.192.168.1.1 <none> <none> kubectl get no cn-shanghai.192.168.31.107 -o wide NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP cn-shanghai.192.168.1.1 Ready <none> 5h51m v1.20.4-aliyun.1 192.168.1.1 101.132.66.11 \u901a\u8fc7\u67e5\u770b\u8282\u70b9\u7684 INTERNAL-IP \uff0c\u53ef\u4e86\u89e3\u8be5\u8282\u70b9\u7684\u5185\u90e8\u8bbf\u95ee IP \u4e3a 192.168.1.1 $ kubectl get svc -n kube-system | grep prometheus-svc prometheus-svc NodePort 10.100.0.144 <none> 9090:31225/TCP 22h $ kubectl describe svc prometheus-svc -n kube-system Name: prometheus-svc Namespace: kube-system Labels: kubernetes.io/name=Prometheus name=prometheus-svc Annotations: kubectl.kubernetes.io/last-applied-configuration: {\"apiVersion\":\"v1\",\"kind\":\"Service\",\"metadata\":{\"annotations\":{},\"labels\":{\"kubernetes.io/name\":\"Prometheus\",\"name\":\"prometheus-svc\"},\"nam... Selector: app=prometheus Type: NodePort IP: 10.100.0.144 Port: prometheus 9090/TCP TargetPort: 9090/TCP NodePort: prometheus 31225/TCP Endpoints: 10.99.224.138:9090 Session Affinity: None External Traffic Policy: Cluster Events: <none> \u53ef\u4ee5\u770b\u5230\u5f53\u524d prometheus svc \u66b4\u9732\u7684\u7aef\u53e3\u662f 31225 \uff0c\u540e\u7eed\u4f7f\u7528 http://IP:NodePort \u7684\u5f62\u5f0f\u6765\u53d1\u73b0 prometheus \u670d\u52a1 \u914d\u7f6e prometheus data source \u4f7f\u7528 http://IP:NodePort \u7684\u5f62\u5f0f\u6765\u53d1\u73b0 prometheus \u670d\u52a1 \u5bfc\u5165\u5b8c\u6210\u540e\u70b9\u51fbSave & Test \u663e\u793a Data source is working \u5373\u53ef \u5bfc\u5165\u6a21\u677f\u6587\u4ef6 grafana \u9009\u62e9\u5bfc\u5165\u6a21\u677f Json \u6587\u4ef6\uff0c\u70b9\u51fb\u6b64\u5904\u4e0b\u8f7d fluid-prometheus-grafana-monitor-jindofs.json \u67e5\u770b\u76d1\u63a7 \u5728 DashBoards \u4e2d\u627e\u5230\u5bfc\u5165\u7684\u6a21\u7248\u89c6\u56fe \u6ce8\uff1afluid_runtime \u5bf9\u5e94Fluid runtime name; namespace \u5bf9\u5e94 Fluid runtime namespace JindoRuntime Prometheus \u6307\u6807\u9884\u89c8(\u66f4\u65b0\u4e2d) \u00a7 \u4e0b\u9762\u5bf9 JindoRuntime \u5f53\u524d\u6536\u96c6\u7684\u6307\u6807\u8fdb\u884c\u9884\u89c8\u548c\u63cf\u8ff0\uff0c\u60a8\u53ef\u4ee5\u57fa\u4e8e\u5df2\u6536\u96c6\u6307\u6807\u5728 Grafana \u4e2d\u8fdb\u884c\u7ec4\u5408\u548c\u56fe\u8868\u5c55\u793a\uff0c\u5982\u60a8\u9700\u8981\u989d\u5916\u7684\u57fa\u672c metrics \u5c55\u793a\uff0c\u8bf7\u5f00 issue \u8054\u7cfb\u6211\u4eec # HELP jindofsx_ns_backend_read_bytes_time_total_window # TYPE jindofsx_ns_backend_read_bytes_time_total_window gauge jindofsx_ns_backend_read_bytes_time_total_window 20367553 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_read_bytes_total # TYPE jindofsx_ns_backend_read_bytes_total gauge jindofsx_ns_backend_read_bytes_total 166703347021 #\u603b\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_bytes_total_window # TYPE jindofsx_ns_backend_read_bytes_total_window gauge jindofsx_ns_backend_read_bytes_total_window 1174671374 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_time_total # TYPE jindofsx_ns_backend_read_time_total gauge jindofsx_ns_backend_read_time_total 8717632094 #\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_readop_num_total # TYPE jindofsx_ns_backend_readop_num_total gauge jindofsx_ns_backend_readop_num_total 181951 #\u5e95\u5c42\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_local_read_bytes_time_total_window # TYPE jindofsx_ns_local_read_bytes_time_total_window gauge jindofsx_ns_local_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_bytes_total # TYPE jindofsx_ns_local_read_bytes_total gauge jindofsx_ns_local_read_bytes_total 0 #\u603b\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_local_read_bytes_total_window # TYPE jindofsx_ns_local_read_bytes_total_window gauge jindofsx_ns_local_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_time_total # TYPE jindofsx_ns_local_read_time_total gauge jindofsx_ns_local_read_time_total 0 #\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_readop_num_total # TYPE jindofsx_ns_local_readop_num_total gauge jindofsx_ns_local_readop_num_total 0 #\u77ed\u8def\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_remote_read_bytes_time_total_window # TYPE jindofsx_ns_remote_read_bytes_time_total_window gauge jindofsx_ns_remote_read_bytes_time_total_window 73714663 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_read_bytes_total # TYPE jindofsx_ns_remote_read_bytes_total gauge jindofsx_ns_remote_read_bytes_total 112108045498 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_bytes_total_window # TYPE jindofsx_ns_remote_read_bytes_total_window gauge jindofsx_ns_remote_read_bytes_total_window 13380884043 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_time_total # TYPE jindofsx_ns_remote_read_time_total gauge jindofsx_ns_remote_read_time_total 765272359 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u65f6\u95f4\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_readop_num_total # TYPE jindofsx_ns_remote_readop_num_total gauge jindofsx_ns_remote_readop_num_total 107314 #\u8fdc\u7aef\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_slicelet_read_bytes_time_total_window # TYPE jindofsx_ns_slicelet_read_bytes_time_total_window gauge jindofsx_ns_slicelet_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_read_bytes_total # TYPE jindofsx_ns_slicelet_read_bytes_total gauge jindofsx_ns_slicelet_read_bytes_total 0 #\u603b\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_bytes_total_window # TYPE jindofsx_ns_slicelet_read_bytes_total_window gauge jindofsx_ns_slicelet_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_time_total # TYPE jindofsx_ns_slicelet_read_time_total gauge jindofsx_ns_slicelet_read_time_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_readop_num_total # TYPE jindofsx_ns_slicelet_readop_num_total gauge jindofsx_ns_slicelet_readop_num_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_total_disk_cap # TYPE jindofsx_ns_total_disk_cap gauge jindofsx_ns_total_disk_cap 840739848192 #\u78c1\u76d8\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_mem_cap # TYPE jindofsx_ns_total_mem_cap gauge jindofsx_ns_total_mem_cap 0 #\u5185\u5b58\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_stsnodes_num # TYPE jindofsx_ns_total_stsnodes_num gauge jindofsx_ns_total_stsnodes_num 3 #alive worker \u8282\u70b9\u7684\u6570\u91cf # HELP jindofsx_ns_total_used_disk_cap # TYPE jindofsx_ns_total_used_disk_cap gauge jindofsx_ns_total_used_disk_cap 153423446016 #\u5df2\u7528\u78c1\u76d8\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_used_mem_cap # TYPE jindofsx_ns_total_used_mem_cap gauge jindofsx_ns_total_used_mem_cap 0 #\u5df2\u7528\u5185\u5b58\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte","title":"\u76d1\u63a7\u548c\u53ef\u89c2\u6d4b\u6027\u529f\u80fd"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_monitor_metric/#_1","text":"\u672c\u6587\u4ecb\u7ecd\u5728\u542f\u52a8 Fluid JindoRuntime \u7684\u524d\u63d0\u4e0b\uff0c\u901a\u8fc7 Prometheus + Grafana \u6536\u96c6\u7f13\u5b58\u7cfb\u7edf\u76f8\u5173\u7684 metrics \u6307\u6807\uff0c\u8fdb\u884c\u53ef\u89c6\u5316\u5c55\u793a\u7684\u4f7f\u7528\u65b9\u6cd5\u3002","title":"\u76d1\u63a7\u548c\u53ef\u89c2\u6d4b\u6027\u529f\u80fd"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_monitor_metric/#in-cluster-prometheus-grafana-fluid","text":"","title":"\u4f7f\u7528 In-Cluster \u6a21\u5f0f\u90e8\u7f72 prometheus \u548c grafana \u76d1\u63a7 Fluid \u5e94\u7528"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_monitor_metric/#1-prometheus","text":"\u5982\u679c\u96c6\u7fa4\u5185\u65e0 prometheus\uff0c\u8bf7\u70b9\u51fb\u4e0b\u8f7d prometheus-jindofs.yaml \u6587\u4ef6 $ kubectl apply -f prometheus-jindofs.yaml \u5982\u96c6\u7fa4\u5185\u6709 prometheus,\u53ef\u5c06\u4ee5\u4e0b\u914d\u7f6e\u5199\u5230 prometheus \u914d\u7f6e\u6587\u4ef6\u4e2d: scrape_configs: - job_name: 'jindo runtime' metrics_path: /jindometrics/prometheus kubernetes_sd_configs: - role: endpoints relabel_configs: - source_labels: [__meta_kubernetes_service_label_role] regex: jindofs-master action: keep - source_labels: [__meta_kubernetes_endpoint_port_name] regex: rpc action: keep - source_labels: [__meta_kubernetes_namespace] target_label: namespace replacement: $1 action: replace - source_labels: [__meta_kubernetes_service_label_release] target_label: fluid_runtime replacement: $1 action: replace - source_labels: [__meta_kubernetes_endpoint_address_target_name] target_label: pod replacement: $1 action: replace","title":"1. \u90e8\u7f72\u6216\u914d\u7f6e Prometheus"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_monitor_metric/#2-grafana","text":"\u5982\u679c\u96c6\u7fa4\u5185\u65e0 grafana\uff0c\u8bf7\u70b9\u51fb\u4e0b\u8f7d grafana.yaml \u6587\u4ef6 $ kubectl apply -f grafana.yaml","title":"2. \u90e8\u7f72 grafana"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_monitor_metric/#3-grafana","text":"\u767b\u5f55 grafana \u627e\u5230 grafana \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\u548c\u6539\u8282\u70b9\u5bf9\u5916\u53ef\u8bbf\u95ee grafana-node-ip \uff0c\u5982\u679c\u60f3\u8981\u5728\u516c\u7f51\u8bbf\u95ee\uff0c\u53ef\u4ee5\u7ed1\u5b9a\u516c\u7f51IP\uff0c $ kubectl get pod -n kube-system -o wide | grep grafana monitoring-grafana-7dfcf6d5cd-5k8qw 1/1 Running 0 108m 10.54.1.11 cn-shanghai.192.168.1.1 <none> <none> \u627e\u5230 grafana \u670d\u52a1\u5bf9\u5e94\u7684 NodePort $ kubectl describe svc monitoring-grafana -n kube-system Name: monitoring-grafana Namespace: kube-system Labels: <none> Annotations: <none> Selector: app=grafana Type: NodePort IP Families: <none> IP: 172.16.83.188 IPs: 172.16.83.188 Port: <unset> 80/TCP TargetPort: 3000/TCP NodePort: <unset> 30396/TCP Endpoints: 10.54.1.75:3000 Session Affinity: None External Traffic Policy: Cluster Events: <none> \u53ef\u4ee5\u770b\u5230\u5f53\u524d\u66b4\u9732\u7684 NodePort \u4e3a 30396 \u8bbf\u95ee http://$grafana-node-ip:NodePort \uff0c\u9ed8\u8ba4\u8d26\u53f7\u5bc6\u7801 admin:admin \u67e5\u770b prometheus svc \u7aef\u53e3 \u540c\u6837\u5148\u627e\u5230 prometheus \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\uff0c\u627e\u5230\u8be5\u8282\u70b9\u7684 IP $ kubectl get pod -n kube-system -o wide | grep grafana prometheus-deployment-d6d8554b4-wfl98 1/1 Running 0 17d 10.54.1.204 cn-shanghai.192.168.1.1 <none> <none> kubectl get no cn-shanghai.192.168.31.107 -o wide NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP cn-shanghai.192.168.1.1 Ready <none> 5h51m v1.20.4-aliyun.1 192.168.1.1 101.132.66.11 \u901a\u8fc7\u67e5\u770b\u8282\u70b9\u7684 INTERNAL-IP \uff0c\u53ef\u4e86\u89e3\u8be5\u8282\u70b9\u7684\u5185\u90e8\u8bbf\u95ee IP \u4e3a 192.168.1.1 $ kubectl get svc -n kube-system | grep prometheus-svc prometheus-svc NodePort 10.100.0.144 <none> 9090:31225/TCP 22h $ kubectl describe svc prometheus-svc -n kube-system Name: prometheus-svc Namespace: kube-system Labels: kubernetes.io/name=Prometheus name=prometheus-svc Annotations: kubectl.kubernetes.io/last-applied-configuration: {\"apiVersion\":\"v1\",\"kind\":\"Service\",\"metadata\":{\"annotations\":{},\"labels\":{\"kubernetes.io/name\":\"Prometheus\",\"name\":\"prometheus-svc\"},\"nam... Selector: app=prometheus Type: NodePort IP: 10.100.0.144 Port: prometheus 9090/TCP TargetPort: 9090/TCP NodePort: prometheus 31225/TCP Endpoints: 10.99.224.138:9090 Session Affinity: None External Traffic Policy: Cluster Events: <none> \u53ef\u4ee5\u770b\u5230\u5f53\u524d prometheus svc \u66b4\u9732\u7684\u7aef\u53e3\u662f 31225 \uff0c\u540e\u7eed\u4f7f\u7528 http://IP:NodePort \u7684\u5f62\u5f0f\u6765\u53d1\u73b0 prometheus \u670d\u52a1 \u914d\u7f6e prometheus data source \u4f7f\u7528 http://IP:NodePort \u7684\u5f62\u5f0f\u6765\u53d1\u73b0 prometheus \u670d\u52a1 \u5bfc\u5165\u5b8c\u6210\u540e\u70b9\u51fbSave & Test \u663e\u793a Data source is working \u5373\u53ef \u5bfc\u5165\u6a21\u677f\u6587\u4ef6 grafana \u9009\u62e9\u5bfc\u5165\u6a21\u677f Json \u6587\u4ef6\uff0c\u70b9\u51fb\u6b64\u5904\u4e0b\u8f7d fluid-prometheus-grafana-monitor-jindofs.json \u67e5\u770b\u76d1\u63a7 \u5728 DashBoards \u4e2d\u627e\u5230\u5bfc\u5165\u7684\u6a21\u7248\u89c6\u56fe \u6ce8\uff1afluid_runtime \u5bf9\u5e94Fluid runtime name; namespace \u5bf9\u5e94 Fluid runtime namespace","title":"3. \u914d\u7f6e grafana"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_monitor_metric/#jindoruntime-prometheus","text":"\u4e0b\u9762\u5bf9 JindoRuntime \u5f53\u524d\u6536\u96c6\u7684\u6307\u6807\u8fdb\u884c\u9884\u89c8\u548c\u63cf\u8ff0\uff0c\u60a8\u53ef\u4ee5\u57fa\u4e8e\u5df2\u6536\u96c6\u6307\u6807\u5728 Grafana \u4e2d\u8fdb\u884c\u7ec4\u5408\u548c\u56fe\u8868\u5c55\u793a\uff0c\u5982\u60a8\u9700\u8981\u989d\u5916\u7684\u57fa\u672c metrics \u5c55\u793a\uff0c\u8bf7\u5f00 issue \u8054\u7cfb\u6211\u4eec # HELP jindofsx_ns_backend_read_bytes_time_total_window # TYPE jindofsx_ns_backend_read_bytes_time_total_window gauge jindofsx_ns_backend_read_bytes_time_total_window 20367553 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_read_bytes_total # TYPE jindofsx_ns_backend_read_bytes_total gauge jindofsx_ns_backend_read_bytes_total 166703347021 #\u603b\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_bytes_total_window # TYPE jindofsx_ns_backend_read_bytes_total_window gauge jindofsx_ns_backend_read_bytes_total_window 1174671374 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_time_total # TYPE jindofsx_ns_backend_read_time_total gauge jindofsx_ns_backend_read_time_total 8717632094 #\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_readop_num_total # TYPE jindofsx_ns_backend_readop_num_total gauge jindofsx_ns_backend_readop_num_total 181951 #\u5e95\u5c42\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_local_read_bytes_time_total_window # TYPE jindofsx_ns_local_read_bytes_time_total_window gauge jindofsx_ns_local_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_bytes_total # TYPE jindofsx_ns_local_read_bytes_total gauge jindofsx_ns_local_read_bytes_total 0 #\u603b\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_local_read_bytes_total_window # TYPE jindofsx_ns_local_read_bytes_total_window gauge jindofsx_ns_local_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_time_total # TYPE jindofsx_ns_local_read_time_total gauge jindofsx_ns_local_read_time_total 0 #\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_readop_num_total # TYPE jindofsx_ns_local_readop_num_total gauge jindofsx_ns_local_readop_num_total 0 #\u77ed\u8def\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_remote_read_bytes_time_total_window # TYPE jindofsx_ns_remote_read_bytes_time_total_window gauge jindofsx_ns_remote_read_bytes_time_total_window 73714663 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_read_bytes_total # TYPE jindofsx_ns_remote_read_bytes_total gauge jindofsx_ns_remote_read_bytes_total 112108045498 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_bytes_total_window # TYPE jindofsx_ns_remote_read_bytes_total_window gauge jindofsx_ns_remote_read_bytes_total_window 13380884043 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_time_total # TYPE jindofsx_ns_remote_read_time_total gauge jindofsx_ns_remote_read_time_total 765272359 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u65f6\u95f4\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_readop_num_total # TYPE jindofsx_ns_remote_readop_num_total gauge jindofsx_ns_remote_readop_num_total 107314 #\u8fdc\u7aef\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_slicelet_read_bytes_time_total_window # TYPE jindofsx_ns_slicelet_read_bytes_time_total_window gauge jindofsx_ns_slicelet_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_read_bytes_total # TYPE jindofsx_ns_slicelet_read_bytes_total gauge jindofsx_ns_slicelet_read_bytes_total 0 #\u603b\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_bytes_total_window # TYPE jindofsx_ns_slicelet_read_bytes_total_window gauge jindofsx_ns_slicelet_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_time_total # TYPE jindofsx_ns_slicelet_read_time_total gauge jindofsx_ns_slicelet_read_time_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_readop_num_total # TYPE jindofsx_ns_slicelet_readop_num_total gauge jindofsx_ns_slicelet_readop_num_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_total_disk_cap # TYPE jindofsx_ns_total_disk_cap gauge jindofsx_ns_total_disk_cap 840739848192 #\u78c1\u76d8\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_mem_cap # TYPE jindofsx_ns_total_mem_cap gauge jindofsx_ns_total_mem_cap 0 #\u5185\u5b58\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_stsnodes_num # TYPE jindofsx_ns_total_stsnodes_num gauge jindofsx_ns_total_stsnodes_num 3 #alive worker \u8282\u70b9\u7684\u6570\u91cf # HELP jindofsx_ns_total_used_disk_cap # TYPE jindofsx_ns_total_used_disk_cap gauge jindofsx_ns_total_used_disk_cap 153423446016 #\u5df2\u7528\u78c1\u76d8\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_used_mem_cap # TYPE jindofsx_ns_total_used_mem_cap gauge jindofsx_ns_total_used_mem_cap 0 #\u5df2\u7528\u5185\u5b58\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte","title":"JindoRuntime Prometheus \u6307\u6807\u9884\u89c8(\u66f4\u65b0\u4e2d)"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_nas_ufs_example/","text":"\u52a0\u901fOSS\u4e0a\u6570\u636e \u00a7 \u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002 1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001 \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907 \u00a7 5.1 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: nas spec: mounts: - mountPoint: local:///mnt/nas name: data path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: nas spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d NAS \u7684\u8def\u5f84\uff0c local:// \u4e3a\u672c\u5730\u8def\u5f84\u524d\u7f00\uff0c /mnt/nas \u8868\u793a\u672c\u5730\u771f\u5b9e\u8def\u5f84\uff0c\u8be5\u8def\u5f84\u9700\u8981\u5728\u5404\u4e2a\u8282\u70b9\u4e0a\u90fd\u5b58\u5728\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JinRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime nas NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE nas Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset nas NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE nas 511MiB 0.00B 180.00GiB 0.0% Bound 1h \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/nas 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/nas Bound hadoop 100Gi RWX 58m 6\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime nas \u5220\u9664 dataset kubectl delete dataset nas","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_nas_ufs_example/#oss","text":"\u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_nas_ufs_example/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_nas_ufs_example/#2-fluid","text":"","title":"2\u3001\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_nas_ufs_example/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_nas_ufs_example/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_nas_ufs_example/#5","text":"","title":"5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_nas_ufs_example/#51-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: nas spec: mounts: - mountPoint: local:///mnt/nas name: data path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: nas spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d NAS \u7684\u8def\u5f84\uff0c local:// \u4e3a\u672c\u5730\u8def\u5f84\u524d\u7f00\uff0c /mnt/nas \u8868\u793a\u672c\u5730\u771f\u5b9e\u8def\u5f84\uff0c\u8be5\u8def\u5f84\u9700\u8981\u5728\u5404\u4e2a\u8282\u70b9\u4e0a\u90fd\u5b58\u5728\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JinRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime nas NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE nas Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset nas NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE nas 511MiB 0.00B 180.00GiB 0.0% Bound 1h \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/nas 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/nas Bound hadoop 100Gi RWX 58m","title":"5.1 \u521b\u5efa dataset \u548c JindoRuntime"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_nas_ufs_example/#6","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime nas \u5220\u9664 dataset kubectl delete dataset nas","title":"6\u3001\u73af\u5883\u6e05\u7406"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/","text":"\u52a0\u901f OSS-HDFS \u4e0a\u6570\u636e \u00a7 \u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS-HDFS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002 1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001 \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907 \u00a7 5.1 \u51c6\u5907 OSS-HDFS \u670d\u52a1 \u00a7 \u60a8\u9700\u8981\u5f00\u901a\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1(Object Storage Service\uff0c\u7b80\u79f0OSS)\uff0c\u6309\u7167\u4f7f\u7528\u6587\u6863\u8fdb\u884cbucket\u521b\u5efa\uff0c\u5e76\u5728\u521b\u5efa\u65f6\u6307\u5b9abucket\u4e3a\u652f\u6301 OSS-HDFS \u6a21\u5f0f\uff0c\u8bbf\u95ee\u94fe\u63a5 https://github.com../../tree/master/docsdocs/user/4.x/4.6.x/4.6.12/jindofs/jindo_dls_quickstart.md 5.2 \u51c6\u5907\u6d4b\u8bd5\u6837\u4f8b\u6570\u636e \u00a7 \u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 Apache \u955c\u50cf\u7ad9\u70b9\u4e0a\u7684 Spark \u76f8\u5173\u8d44\u6e90\u4f5c\u4e3a\u6f14\u793a\u4e2d\u4f7f\u7528\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002\u8fd9\u4e2a\u9009\u62e9\u5e76\u6ca1\u6709\u4efb\u4f55\u7279\u6b8a\u4e4b\u5904\uff0c\u4f60\u53ef\u4ee5\u5c06\u8fd9\u4e2a\u8fdc\u7a0b\u6587\u4ef6\u4fee\u6539\u4e3a\u4efb\u610f\u4f60\u559c\u6b22\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002 \u73b0\u5728\u8fdc\u7a0b\u8d44\u6e90\u6587\u4ef6\u5230\u672c\u5730 mkdir tmp cd tmp wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz \u4e0a\u4f20\u672c\u5730\u6587\u4ef6\u5230 OSS-HDFS \u4e0a \u60a8\u53ef\u4ee5\u4f7f\u7528 JindoSDK \uff0c\u6309\u7167\u4f7f\u7528\u8bf4\u660e\u5c06\u672c\u5730\u4e0b\u8f7d\u7684\u6587\u4ef6\u4e0a\u4f20\u5230\u8fdc\u7a0b OSS-HDFS \u7684bucket\u4e0a\u3002\u5982\u60a8\u7684bucket\u540d\u5b57\u4e3a test-bucket \u4e14\u4f7f\u7528 JindoSDK \u4f5c\u4e3a\u5ba2\u6237\u7aef\u5de5\u5177\uff0c\u53ef\u6267\u884c\u5982\u4e0b\u547d\u4ee4\u4e0a\u4f20\u6587\u4ef6\u5230 OSS-HDFS \u4e0a\u9762\u3002 hadoop fs -put spark-2.4.7-bin-hadoop2.7.tgz oss://test-bucket/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz hadoop fs -put spark-3.1.1-bin-hadoop3.2.tgz oss://test-bucket/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz 5.3 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/dir/ options: fs.oss.accessKeyId: <OSS_HDFS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_HDFS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_HDFS_ENDPOINT> name: hadoop path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aOSS-HDFS bucket\u7684endpoint\u4fe1\u606f\uff0c\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08cn-xxx.oss-dls.aliyuncs.com\uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08oss-cn-xxx.aliyuncs.com\uff09\u4e0d\u540c\uff0c\u5982\u60a8\u7684\u670d\u52a1\u5728 hangzhou\uff0c\u90a3\u4e48\u5bf9\u5e94\u7684 endpoint \u4e3a\uff1acn-hangzhou.oss-dls.aliyuncs.com replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JinRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hadoop Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE HCFS URL TOTAL FILES CACHE HIT RATIO AGE hadoop [Calculating] 0.00B 200.00GiB Bound hadoop-jindofs-master-0.default:18844 [Calculating] 6m28s \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m 6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: hadoop \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u67e5\u770b\u6587\u4ef6\u5927\u5c0f $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 \u8fdb\u884c\u6587\u4ef6\u7684cp\u89c2\u5bdf\u65f6\u95f4\u6d88\u8017\u4e8618s $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m18.386s user 0m0.002s sys 0m0.105s \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 oss \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: hadoop-dataload spec: loadMetadata: true dataset: name: hadoop namespace: default \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload spark-dataload NAME DATASET PHASE AGE hadoop-dataload hadoop Completed 2m13s \u67e5\u770b\u6b64\u65f6 dataset \u7684\u7f13\u5b58\u60c5\u51b5\uff0c\u53d1\u73b0210MB\u7684\u6570\u636e\u5df2\u7ecf\u90fd\u7f13\u5b58\u5230\u4e86\u672c\u5730\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hadoop 210.00MiB 210.00MiB 180.00GiB 100.0% Bound 1h \u4e3a\u4e86\u907f\u514d\u5176\u4ed6\u56e0\u7d20(\u6bd4\u5982page cache)\u5bf9\u7ed3\u679c\u9020\u6210\u5f71\u54cd\uff0c\u6211\u4eec\u5c06\u5220\u9664\u4e4b\u524d\u7684\u5bb9\u5668\uff0c\u65b0\u5efa\u76f8\u540c\u7684\u5e94\u7528\uff0c\u5c1d\u8bd5\u8bbf\u95ee\u540c\u6837\u7684\u6587\u4ef6\u3002\u7531\u4e8e\u6b64\u65f6\u6587\u4ef6\u5df2\u7ecf\u88ab JindoFSx \u7f13\u5b58\uff0c\u53ef\u4ee5\u770b\u5230\u7b2c\u4e8c\u6b21\u8bbf\u95ee\u6240\u9700\u65f6\u95f4\u8fdc\u5c0f\u4e8e\u7b2c\u4e00\u6b21\u3002 kubectl delete -f app.yaml && kubectl create -f app.yaml \u8fdb\u884c\u6587\u4ef6\u7684\u62f7\u8d1d\u89c2\u5bdf\u65f6\u95f4\uff0c\u53d1\u73b0\u6d88\u801748ms\uff0c\u6574\u4e2a\u62f7\u8d1d\u7684\u65f6\u95f4\u7f29\u77ed\u4e86300\u500d $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m0.048s user 0m0.001s sys 0m0.046s 7\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"\u52a0\u901f OSS-HDFS \u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#oss-hdfs","text":"\u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS-HDFS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002","title":"\u52a0\u901f OSS-HDFS \u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#2-fluid","text":"","title":"2\u3001\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#5","text":"","title":"5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#51-oss-hdfs","text":"\u60a8\u9700\u8981\u5f00\u901a\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1(Object Storage Service\uff0c\u7b80\u79f0OSS)\uff0c\u6309\u7167\u4f7f\u7528\u6587\u6863\u8fdb\u884cbucket\u521b\u5efa\uff0c\u5e76\u5728\u521b\u5efa\u65f6\u6307\u5b9abucket\u4e3a\u652f\u6301 OSS-HDFS \u6a21\u5f0f\uff0c\u8bbf\u95ee\u94fe\u63a5 https://github.com../../tree/master/docsdocs/user/4.x/4.6.x/4.6.12/jindofs/jindo_dls_quickstart.md","title":"5.1 \u51c6\u5907 OSS-HDFS \u670d\u52a1"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#52","text":"\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 Apache \u955c\u50cf\u7ad9\u70b9\u4e0a\u7684 Spark \u76f8\u5173\u8d44\u6e90\u4f5c\u4e3a\u6f14\u793a\u4e2d\u4f7f\u7528\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002\u8fd9\u4e2a\u9009\u62e9\u5e76\u6ca1\u6709\u4efb\u4f55\u7279\u6b8a\u4e4b\u5904\uff0c\u4f60\u53ef\u4ee5\u5c06\u8fd9\u4e2a\u8fdc\u7a0b\u6587\u4ef6\u4fee\u6539\u4e3a\u4efb\u610f\u4f60\u559c\u6b22\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002 \u73b0\u5728\u8fdc\u7a0b\u8d44\u6e90\u6587\u4ef6\u5230\u672c\u5730 mkdir tmp cd tmp wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz \u4e0a\u4f20\u672c\u5730\u6587\u4ef6\u5230 OSS-HDFS \u4e0a \u60a8\u53ef\u4ee5\u4f7f\u7528 JindoSDK \uff0c\u6309\u7167\u4f7f\u7528\u8bf4\u660e\u5c06\u672c\u5730\u4e0b\u8f7d\u7684\u6587\u4ef6\u4e0a\u4f20\u5230\u8fdc\u7a0b OSS-HDFS \u7684bucket\u4e0a\u3002\u5982\u60a8\u7684bucket\u540d\u5b57\u4e3a test-bucket \u4e14\u4f7f\u7528 JindoSDK \u4f5c\u4e3a\u5ba2\u6237\u7aef\u5de5\u5177\uff0c\u53ef\u6267\u884c\u5982\u4e0b\u547d\u4ee4\u4e0a\u4f20\u6587\u4ef6\u5230 OSS-HDFS \u4e0a\u9762\u3002 hadoop fs -put spark-2.4.7-bin-hadoop2.7.tgz oss://test-bucket/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz hadoop fs -put spark-3.1.1-bin-hadoop3.2.tgz oss://test-bucket/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz","title":"5.2 \u51c6\u5907\u6d4b\u8bd5\u6837\u4f8b\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#53-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/dir/ options: fs.oss.accessKeyId: <OSS_HDFS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_HDFS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_HDFS_ENDPOINT> name: hadoop path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aOSS-HDFS bucket\u7684endpoint\u4fe1\u606f\uff0c\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08cn-xxx.oss-dls.aliyuncs.com\uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08oss-cn-xxx.aliyuncs.com\uff09\u4e0d\u540c\uff0c\u5982\u60a8\u7684\u670d\u52a1\u5728 hangzhou\uff0c\u90a3\u4e48\u5bf9\u5e94\u7684 endpoint \u4e3a\uff1acn-hangzhou.oss-dls.aliyuncs.com replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JinRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hadoop Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE HCFS URL TOTAL FILES CACHE HIT RATIO AGE hadoop [Calculating] 0.00B 200.00GiB Bound hadoop-jindofs-master-0.default:18844 [Calculating] 6m28s \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m","title":"5.3 \u521b\u5efa dataset \u548c JindoRuntime"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#6","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: hadoop \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u67e5\u770b\u6587\u4ef6\u5927\u5c0f $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 \u8fdb\u884c\u6587\u4ef6\u7684cp\u89c2\u5bdf\u65f6\u95f4\u6d88\u8017\u4e8618s $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m18.386s user 0m0.002s sys 0m0.105s \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 oss \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: hadoop-dataload spec: loadMetadata: true dataset: name: hadoop namespace: default \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload spark-dataload NAME DATASET PHASE AGE hadoop-dataload hadoop Completed 2m13s \u67e5\u770b\u6b64\u65f6 dataset \u7684\u7f13\u5b58\u60c5\u51b5\uff0c\u53d1\u73b0210MB\u7684\u6570\u636e\u5df2\u7ecf\u90fd\u7f13\u5b58\u5230\u4e86\u672c\u5730\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hadoop 210.00MiB 210.00MiB 180.00GiB 100.0% Bound 1h \u4e3a\u4e86\u907f\u514d\u5176\u4ed6\u56e0\u7d20(\u6bd4\u5982page cache)\u5bf9\u7ed3\u679c\u9020\u6210\u5f71\u54cd\uff0c\u6211\u4eec\u5c06\u5220\u9664\u4e4b\u524d\u7684\u5bb9\u5668\uff0c\u65b0\u5efa\u76f8\u540c\u7684\u5e94\u7528\uff0c\u5c1d\u8bd5\u8bbf\u95ee\u540c\u6837\u7684\u6587\u4ef6\u3002\u7531\u4e8e\u6b64\u65f6\u6587\u4ef6\u5df2\u7ecf\u88ab JindoFSx \u7f13\u5b58\uff0c\u53ef\u4ee5\u770b\u5230\u7b2c\u4e8c\u6b21\u8bbf\u95ee\u6240\u9700\u65f6\u95f4\u8fdc\u5c0f\u4e8e\u7b2c\u4e00\u6b21\u3002 kubectl delete -f app.yaml && kubectl create -f app.yaml \u8fdb\u884c\u6587\u4ef6\u7684\u62f7\u8d1d\u89c2\u5bdf\u65f6\u95f4\uff0c\u53d1\u73b0\u6d88\u801748ms\uff0c\u6574\u4e2a\u62f7\u8d1d\u7684\u65f6\u95f4\u7f29\u77ed\u4e86300\u500d $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m0.048s user 0m0.001s sys 0m0.046s","title":"6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#7","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"7\u3001\u73af\u5883\u6e05\u7406"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/","text":"\u52a0\u901fOSS\u4e0a\u6570\u636e \u00a7 \u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002 1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001 \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907 \u00a7 5.1 \u51c6\u5907 OSS \u670d\u52a1 \u00a7 \u60a8\u9700\u8981\u5f00\u901a\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1(Object Storage Service\uff0c\u7b80\u79f0OSS)\uff0c\u6309\u7167\u4f7f\u7528\u6587\u6863\u8fdb\u884cbucket\u521b\u5efa( https://www.aliyun.com/product/oss ) 5.2 \u51c6\u5907\u6d4b\u8bd5\u6837\u4f8b\u6570\u636e \u00a7 \u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 Apache \u955c\u50cf\u7ad9\u70b9\u4e0a\u7684 Spark \u76f8\u5173\u8d44\u6e90\u4f5c\u4e3a\u6f14\u793a\u4e2d\u4f7f\u7528\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002\u8fd9\u4e2a\u9009\u62e9\u5e76\u6ca1\u6709\u4efb\u4f55\u7279\u6b8a\u4e4b\u5904\uff0c\u4f60\u53ef\u4ee5\u5c06\u8fd9\u4e2a\u8fdc\u7a0b\u6587\u4ef6\u4fee\u6539\u4e3a\u4efb\u610f\u4f60\u559c\u6b22\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002 \u73b0\u5728\u8fdc\u7a0b\u8d44\u6e90\u6587\u4ef6\u5230\u672c\u5730 mkdir tmp cd tmp wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz \u4e0a\u4f20\u672c\u5730\u6587\u4ef6\u5230 OSS \u4e0a \u60a8\u53ef\u4ee5\u4f7f\u7528 OSS \u63d0\u4f9b\u7684\u5ba2\u6237\u7aef ossutil \u6216\u8005\u963f\u91cc\u4e91EMR\u56e2\u961f\u63d0\u4f9b\u7684 JindoSDK \uff0c\u6309\u7167\u4f7f\u7528\u8bf4\u660e\u5c06\u672c\u5730\u4e0b\u8f7d\u7684\u6587\u4ef6\u4e0a\u4f20\u5230\u8fdc\u7a0b OSS \u7684bucket\u4e0a\u3002\u5982\u60a8\u7684bucket\u540d\u5b57\u4e3a test-bucket \u4e14\u4f7f\u7528 JindoSDK \u4f5c\u4e3a\u5ba2\u6237\u7aef\u5de5\u5177\uff0c\u53ef\u6267\u884c\u5982\u4e0b\u547d\u4ee4\u4e0a\u4f20\u6587\u4ef6\u5230 OSS \u4e0a\u9762\u3002 hadoop fs -put spark-2.4.7-bin-hadoop2.7.tgz oss://test-bucket/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz hadoop fs -put spark-3.1.1-bin-hadoop3.2.tgz oss://test-bucket/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz 5.3 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/dir/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aoss bucket\u7684endpoint\u4fe1\u606f\uff0c\u516c\u7f51\u6216\u5185\u7f51\u5730\u5740\u7686\u53ef\uff0c\u5982\u60a8\u7684 bucket \u5728\u676d\u5dde Region\uff0c\u90a3\u4e48\u516c\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou.aliyuncs.com\uff0c\u5185\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou-internal.aliyuncs.com(\u5185\u7f51\u5730\u5740\u4f7f\u7528\u6761\u4ef6\u4e3a\u60a8\u7684 K8S \u96c6\u7fa4\u6240\u5728\u533a\u57df\u548coss\u533a\u57df\u76f8\u540c)\u3002 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JindoRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hadoop Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE HCFS URL TOTAL FILES CACHE HIT RATIO AGE hadoop [Calculating] 0.00B 200.00GiB Bound hadoop-jindofs-master-0.default:18844 [Calculating] 6m28s \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m 6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: hadoop \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u67e5\u770b\u6587\u4ef6\u5927\u5c0f $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 \u8fdb\u884c\u6587\u4ef6\u7684cp\u89c2\u5bdf\u65f6\u95f4\u6d88\u8017\u4e8618s $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m18.386s user 0m0.002s sys 0m0.105s \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 oss \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: hadoop-dataload spec: loadMetadata: true dataset: name: hadoop namespace: default \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload spark-dataload NAME DATASET PHASE AGE hadoop-dataload hadoop Completed 2m13s \u67e5\u770b\u6b64\u65f6 dataset \u7684\u7f13\u5b58\u60c5\u51b5\uff0c\u53d1\u73b0210MB\u7684\u6570\u636e\u5df2\u7ecf\u90fd\u7f13\u5b58\u5230\u4e86\u672c\u5730\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hadoop 210.00MiB 210.00MiB 180.00GiB 100.0% Bound 1h \u4e3a\u4e86\u907f\u514d\u5176\u4ed6\u56e0\u7d20(\u6bd4\u5982page cache)\u5bf9\u7ed3\u679c\u9020\u6210\u5f71\u54cd\uff0c\u6211\u4eec\u5c06\u5220\u9664\u4e4b\u524d\u7684\u5bb9\u5668\uff0c\u65b0\u5efa\u76f8\u540c\u7684\u5e94\u7528\uff0c\u5c1d\u8bd5\u8bbf\u95ee\u540c\u6837\u7684\u6587\u4ef6\u3002\u7531\u4e8e\u6b64\u65f6\u6587\u4ef6\u5df2\u7ecf\u88ab JindoFSx \u7f13\u5b58\uff0c\u53ef\u4ee5\u770b\u5230\u7b2c\u4e8c\u6b21\u8bbf\u95ee\u6240\u9700\u65f6\u95f4\u8fdc\u5c0f\u4e8e\u7b2c\u4e00\u6b21\u3002 kubectl delete -f app.yaml && kubectl create -f app.yaml \u8fdb\u884c\u6587\u4ef6\u7684\u62f7\u8d1d\u89c2\u5bdf\u65f6\u95f4\uff0c\u53d1\u73b0\u6d88\u801748ms\uff0c\u6574\u4e2a\u62f7\u8d1d\u7684\u65f6\u95f4\u7f29\u77ed\u4e86300\u500d $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m0.048s user 0m0.001s sys 0m0.046s 7\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#oss","text":"\u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#2-fluid","text":"","title":"2\u3001\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#5","text":"","title":"5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#51-oss","text":"\u60a8\u9700\u8981\u5f00\u901a\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1(Object Storage Service\uff0c\u7b80\u79f0OSS)\uff0c\u6309\u7167\u4f7f\u7528\u6587\u6863\u8fdb\u884cbucket\u521b\u5efa( https://www.aliyun.com/product/oss )","title":"5.1 \u51c6\u5907 OSS \u670d\u52a1"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#52","text":"\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 Apache \u955c\u50cf\u7ad9\u70b9\u4e0a\u7684 Spark \u76f8\u5173\u8d44\u6e90\u4f5c\u4e3a\u6f14\u793a\u4e2d\u4f7f\u7528\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002\u8fd9\u4e2a\u9009\u62e9\u5e76\u6ca1\u6709\u4efb\u4f55\u7279\u6b8a\u4e4b\u5904\uff0c\u4f60\u53ef\u4ee5\u5c06\u8fd9\u4e2a\u8fdc\u7a0b\u6587\u4ef6\u4fee\u6539\u4e3a\u4efb\u610f\u4f60\u559c\u6b22\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002 \u73b0\u5728\u8fdc\u7a0b\u8d44\u6e90\u6587\u4ef6\u5230\u672c\u5730 mkdir tmp cd tmp wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz \u4e0a\u4f20\u672c\u5730\u6587\u4ef6\u5230 OSS \u4e0a \u60a8\u53ef\u4ee5\u4f7f\u7528 OSS \u63d0\u4f9b\u7684\u5ba2\u6237\u7aef ossutil \u6216\u8005\u963f\u91cc\u4e91EMR\u56e2\u961f\u63d0\u4f9b\u7684 JindoSDK \uff0c\u6309\u7167\u4f7f\u7528\u8bf4\u660e\u5c06\u672c\u5730\u4e0b\u8f7d\u7684\u6587\u4ef6\u4e0a\u4f20\u5230\u8fdc\u7a0b OSS \u7684bucket\u4e0a\u3002\u5982\u60a8\u7684bucket\u540d\u5b57\u4e3a test-bucket \u4e14\u4f7f\u7528 JindoSDK \u4f5c\u4e3a\u5ba2\u6237\u7aef\u5de5\u5177\uff0c\u53ef\u6267\u884c\u5982\u4e0b\u547d\u4ee4\u4e0a\u4f20\u6587\u4ef6\u5230 OSS \u4e0a\u9762\u3002 hadoop fs -put spark-2.4.7-bin-hadoop2.7.tgz oss://test-bucket/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz hadoop fs -put spark-3.1.1-bin-hadoop3.2.tgz oss://test-bucket/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz","title":"5.2 \u51c6\u5907\u6d4b\u8bd5\u6837\u4f8b\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#53-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/dir/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aoss bucket\u7684endpoint\u4fe1\u606f\uff0c\u516c\u7f51\u6216\u5185\u7f51\u5730\u5740\u7686\u53ef\uff0c\u5982\u60a8\u7684 bucket \u5728\u676d\u5dde Region\uff0c\u90a3\u4e48\u516c\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou.aliyuncs.com\uff0c\u5185\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou-internal.aliyuncs.com(\u5185\u7f51\u5730\u5740\u4f7f\u7528\u6761\u4ef6\u4e3a\u60a8\u7684 K8S \u96c6\u7fa4\u6240\u5728\u533a\u57df\u548coss\u533a\u57df\u76f8\u540c)\u3002 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JindoRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hadoop Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE HCFS URL TOTAL FILES CACHE HIT RATIO AGE hadoop [Calculating] 0.00B 200.00GiB Bound hadoop-jindofs-master-0.default:18844 [Calculating] 6m28s \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m","title":"5.3 \u521b\u5efa dataset \u548c JindoRuntime"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#6","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: hadoop \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u67e5\u770b\u6587\u4ef6\u5927\u5c0f $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 \u8fdb\u884c\u6587\u4ef6\u7684cp\u89c2\u5bdf\u65f6\u95f4\u6d88\u8017\u4e8618s $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m18.386s user 0m0.002s sys 0m0.105s \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 oss \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: hadoop-dataload spec: loadMetadata: true dataset: name: hadoop namespace: default \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload spark-dataload NAME DATASET PHASE AGE hadoop-dataload hadoop Completed 2m13s \u67e5\u770b\u6b64\u65f6 dataset \u7684\u7f13\u5b58\u60c5\u51b5\uff0c\u53d1\u73b0210MB\u7684\u6570\u636e\u5df2\u7ecf\u90fd\u7f13\u5b58\u5230\u4e86\u672c\u5730\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hadoop 210.00MiB 210.00MiB 180.00GiB 100.0% Bound 1h \u4e3a\u4e86\u907f\u514d\u5176\u4ed6\u56e0\u7d20(\u6bd4\u5982page cache)\u5bf9\u7ed3\u679c\u9020\u6210\u5f71\u54cd\uff0c\u6211\u4eec\u5c06\u5220\u9664\u4e4b\u524d\u7684\u5bb9\u5668\uff0c\u65b0\u5efa\u76f8\u540c\u7684\u5e94\u7528\uff0c\u5c1d\u8bd5\u8bbf\u95ee\u540c\u6837\u7684\u6587\u4ef6\u3002\u7531\u4e8e\u6b64\u65f6\u6587\u4ef6\u5df2\u7ecf\u88ab JindoFSx \u7f13\u5b58\uff0c\u53ef\u4ee5\u770b\u5230\u7b2c\u4e8c\u6b21\u8bbf\u95ee\u6240\u9700\u65f6\u95f4\u8fdc\u5c0f\u4e8e\u7b2c\u4e00\u6b21\u3002 kubectl delete -f app.yaml && kubectl create -f app.yaml \u8fdb\u884c\u6587\u4ef6\u7684\u62f7\u8d1d\u89c2\u5bdf\u65f6\u95f4\uff0c\u53d1\u73b0\u6d88\u801748ms\uff0c\u6574\u4e2a\u62f7\u8d1d\u7684\u65f6\u95f4\u7f29\u77ed\u4e86300\u500d $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m0.048s user 0m0.001s sys 0m0.046s","title":"6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#7","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"7\u3001\u73af\u5883\u6e05\u7406"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_overview/","text":"Fluid \u7ed3\u5408 JindoCache \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4ecb\u7ecd\u548c\u4f7f\u7528 \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 Kubernetes version > 1.14, \u652f\u6301CSI Golang 1.12+ Helm 3 \u5b89\u88c5\u5305\u4e0b\u8f7d \u00a7 \u4e0b\u8f7d\u9875\u9762 \u4f7f\u7528\u6587\u6863 \u00a7 \u52a0\u901fOSS\u4e0a\u6570\u636e \u52a0\u901fHDFS\u4e0a\u6570\u636e \u52a0\u901fOSS-HDFS\u4e0a\u6570\u636e \u52a0\u901fS3\u4e0a\u6570\u636e \u52a0\u901fNAS\u4e0a\u6570\u636e \u52a0\u901fPVC\u4e0a\u6570\u636e \u76f4\u63a5\u6302\u8f7d OSS \u4e0a\u6570\u636e Fluid + JindoCache \u9ad8\u9636\u529f\u80fd\u4f7f\u7528 \u6570\u636e\u9884\u52a0\u8f7d \u76d1\u63a7\u53ca\u53ef\u89c2\u6d4b\u529f\u80fd \u95ee\u9898\u8bca\u65ad\u53ca\u5904\u7406 \u5bf9\u63a5\u963f\u91cc\u4e91SLS\u65e5\u5fd7\u670d\u52a1 \u6700\u4f73\u5b9e\u8df5\u7cfb\u5217 \u00a7 \u4e91\u539f\u751f AI \u5728\u81ea\u52a8\u9a7e\u9a76\u9886\u57df\u7684\u5e94\u7528\u63a2\u7d22\u4e0e\u6beb\u672b\u667a\u884c\u843d\u5730\u5b9e\u8df5 \u5fae\u535a\u6d77\u91cf\u6df1\u5ea6\u5b66\u4e60\u6a21\u578b\u8bad\u7ec3\u6548\u7387\u8dc3\u5347\u7684\u79d8\u5bc6 \u4f5c\u4e1a\u5e2e\u68c0\u7d22\u670d\u52a1\u57fa\u4e8e Fluid \u7684\u8ba1\u7b97\u5b58\u50a8\u5206\u79bb\u5b9e\u8df5","title":"Fluid \u7ed3\u5408 JindoCache \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4ecb\u7ecd\u548c\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_overview/#fluid-jindocache","text":"","title":"Fluid \u7ed3\u5408 JindoCache \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4ecb\u7ecd\u548c\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_overview/#_1","text":"Kubernetes version > 1.14, \u652f\u6301CSI Golang 1.12+ Helm 3","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_overview/#_2","text":"\u4e0b\u8f7d\u9875\u9762","title":"\u5b89\u88c5\u5305\u4e0b\u8f7d"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_overview/#_3","text":"\u52a0\u901fOSS\u4e0a\u6570\u636e \u52a0\u901fHDFS\u4e0a\u6570\u636e \u52a0\u901fOSS-HDFS\u4e0a\u6570\u636e \u52a0\u901fS3\u4e0a\u6570\u636e \u52a0\u901fNAS\u4e0a\u6570\u636e \u52a0\u901fPVC\u4e0a\u6570\u636e \u76f4\u63a5\u6302\u8f7d OSS \u4e0a\u6570\u636e Fluid + JindoCache \u9ad8\u9636\u529f\u80fd\u4f7f\u7528 \u6570\u636e\u9884\u52a0\u8f7d \u76d1\u63a7\u53ca\u53ef\u89c2\u6d4b\u529f\u80fd \u95ee\u9898\u8bca\u65ad\u53ca\u5904\u7406 \u5bf9\u63a5\u963f\u91cc\u4e91SLS\u65e5\u5fd7\u670d\u52a1","title":"\u4f7f\u7528\u6587\u6863"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_overview/#_4","text":"\u4e91\u539f\u751f AI \u5728\u81ea\u52a8\u9a7e\u9a76\u9886\u57df\u7684\u5e94\u7528\u63a2\u7d22\u4e0e\u6beb\u672b\u667a\u884c\u843d\u5730\u5b9e\u8df5 \u5fae\u535a\u6d77\u91cf\u6df1\u5ea6\u5b66\u4e60\u6a21\u578b\u8bad\u7ec3\u6548\u7387\u8dc3\u5347\u7684\u79d8\u5bc6 \u4f5c\u4e1a\u5e2e\u68c0\u7d22\u670d\u52a1\u57fa\u4e8e Fluid \u7684\u8ba1\u7b97\u5b58\u50a8\u5206\u79bb\u5b9e\u8df5","title":"\u6700\u4f73\u5b9e\u8df5\u7cfb\u5217"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/","text":"\u52a0\u901fPVC\u4e0a\u6570\u636e \u00a7 \u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c PVC \u6570\u636e\u5377\u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002 1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001 \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907 \u00a7 5.1 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u4e3a\u5df2\u5b58\u5728\u7684PVC\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 testPVC\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: pvc spec: mounts: - mountPoint: pvc://testPVC name: data path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: pvc spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d pvc \u7684\u8def\u5f84\uff0c pvc:// \u4e3a\u6302\u8f7d\u672c\u5730volume\u524d\u7f00\uff0c testPVC \u8868\u793a\u9700\u8981\u52a0\u901f\u7684PVC\u7684\u540d\u5b57\uff0c\u9700\u8981\u6709\u8bbf\u95ee\u6743\u9650\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JinRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime pvc NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE pvc Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset pvc NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE pvc 511MiB 0.00B 180.00GiB 0.0% Bound 1h \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/pvc 100Gi ROX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/pvc Bound hadoop 100Gi RWX 58m 6\u3001\u6570\u636e\u9884\u70ed \u00a7 \u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf \u00a7 \u53ef\u4ee5\u8fdb\u884c\u6570\u636e\u9884\u70ed\u63d0\u524d\u5c06\u6587\u4ef6\u7f13\u5b58\u5230\u672c\u5730\uff0c\u5047\u8bbe\u9700\u8981\u52a0\u901f\u8be5 pvc \u4e0b\u9762\u7684\u4e24\u4e2a\u5b50\u76ee\u5f55 /data1 \u548c /data1 \u540c\u65f6\u5206\u522b\u6307\u5b9a\u7f13\u5b58\u526f\u672c\u6570\u91cf\uff0c\u53ef\u4ee5\u53c2\u8003\u5982\u4e0bdataload.yaml\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: test spec: dataset: name: pvc namespace: default loadMetadata: true target: - path: /data1/ replicas: 1 - path: /data2/ replicas: 2 kubectl create dataload.yaml \u89c2\u5bdf dataload \u72b6\u6001 kubectl get dataload test \u5177\u4f53 dataload \u7684\u529f\u80fd\u53ef\u53c2\u8003 \u6570\u636e\u9884\u52a0\u8f7d 7\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime \u548c dataset kubectl delete dataset pvc","title":"\u52a0\u901fPVC\u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/#pvc","text":"\u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c PVC \u6570\u636e\u5377\u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002","title":"\u52a0\u901fPVC\u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/#2-fluid","text":"","title":"2\u3001\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/#5","text":"","title":"5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/#51-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u4e3a\u5df2\u5b58\u5728\u7684PVC\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 testPVC\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: pvc spec: mounts: - mountPoint: pvc://testPVC name: data path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: pvc spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d pvc \u7684\u8def\u5f84\uff0c pvc:// \u4e3a\u6302\u8f7d\u672c\u5730volume\u524d\u7f00\uff0c testPVC \u8868\u793a\u9700\u8981\u52a0\u901f\u7684PVC\u7684\u540d\u5b57\uff0c\u9700\u8981\u6709\u8bbf\u95ee\u6743\u9650\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JinRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime pvc NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE pvc Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset pvc NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE pvc 511MiB 0.00B 180.00GiB 0.0% Bound 1h \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/pvc 100Gi ROX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/pvc Bound hadoop 100Gi RWX 58m","title":"5.1 \u521b\u5efa dataset \u548c JindoRuntime"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/#6","text":"","title":"6\u3001\u6570\u636e\u9884\u70ed"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/#_1","text":"\u53ef\u4ee5\u8fdb\u884c\u6570\u636e\u9884\u70ed\u63d0\u524d\u5c06\u6587\u4ef6\u7f13\u5b58\u5230\u672c\u5730\uff0c\u5047\u8bbe\u9700\u8981\u52a0\u901f\u8be5 pvc \u4e0b\u9762\u7684\u4e24\u4e2a\u5b50\u76ee\u5f55 /data1 \u548c /data1 \u540c\u65f6\u5206\u522b\u6307\u5b9a\u7f13\u5b58\u526f\u672c\u6570\u91cf\uff0c\u53ef\u4ee5\u53c2\u8003\u5982\u4e0bdataload.yaml\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: test spec: dataset: name: pvc namespace: default loadMetadata: true target: - path: /data1/ replicas: 1 - path: /data2/ replicas: 2 kubectl create dataload.yaml \u89c2\u5bdf dataload \u72b6\u6001 kubectl get dataload test \u5177\u4f53 dataload \u7684\u529f\u80fd\u53ef\u53c2\u8003 \u6570\u636e\u9884\u52a0\u8f7d","title":"\u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/#7","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime \u548c dataset kubectl delete dataset pvc","title":"7\u3001\u73af\u5883\u6e05\u7406"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_question/","text":"\u95ee\u9898\u8bca\u65ad\u53ca\u5904\u7406 \u00a7 \u5982\u60a8\u7684\u73af\u5883\u51fa\u73b0\u95ee\u9898\uff0c\u4e5f\u53ef\u5f00 ISSUE \u7ed9\u6211\u4eec\uff0c\u4f1a\u7b2c\u4e00\u65f6\u95f4\u5904\u7406 Fluid \u5b89\u88c5\u4f7f\u7528\u76f8\u5173\u95ee\u9898 \u00a7 1. \u4e3a\u4ec0\u4e48\u6211\u4f7f\u7528Helm\u5b89\u88c5fluid\u5931\u8d25\u4e86\uff1f \u00a7 \u56de\u7b54 :\u63a8\u8350\u6309\u7167\u5b89\u88c5\u6587\u6863\u4f9d\u6b21\u786e\u8ba4Fluid\u7ec4\u4ef6\u662f\u5426\u6b63\u5e38\u8fd0\u884c\u3002 Fluid\u5b89\u88c5\u6587\u6863\u662f\u4ee5 Helm 3 \u4e3a\u4f8b\u8fdb\u884c\u90e8\u7f72\u7684\u3002\u5982\u679c\u60a8\u4f7f\u7528 Helm 3 \u4ee5\u4e0b\u7684\u7248\u672c\u90e8\u7f72Fluid\uff0c \u5e76\u4e14\u9047\u5230\u4e86 CRD\u6ca1\u6709\u6b63\u5e38\u542f\u52a8 \u7684\u60c5\u51b5\uff0c\u8fd9\u53ef\u80fd\u662f\u56e0\u4e3a Helm 3 \u53ca\u5176\u4ee5\u4e0a\u7248\u672c\u4f1a\u5728 helm install \u7684\u65f6\u5019\u81ea\u52a8\u5b89\u88c5CRD\uff0c \u800c\u4f4e\u7248\u672c\u7684Helm\u5219\u4e0d\u4f1a\u3002 \u53c2\u89c1 Helm\u5b98\u65b9\u6587\u6863\u8bf4\u660e \u3002 \u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u60a8\u9700\u8981\u624b\u52a8\u5b89\u88c5CRD\uff1a $ kubectl create -f fluid/crds 2. \u4e3a\u4ec0\u4e48\u6211\u65e0\u6cd5\u5220\u9664Runtime\uff1f \u00a7 \u56de\u7b54 :\u8bf7\u68c0\u67e5\u76f8\u5173Pod\u8fd0\u884c\u72b6\u6001\u548cRuntime\u7684Events\u3002 \u53ea\u8981\u6709\u4efb\u4f55\u6d3b\u8dc3Pod\u8fd8\u5728\u4f7f\u7528Fluid\u521b\u5efa\u7684Volume\uff0cFluid\u5c31\u4e0d\u4f1a\u5b8c\u6210\u5220\u9664\u64cd\u4f5c\u3002 \u5982\u4e0b\u7684\u547d\u4ee4\u53ef\u4ee5\u5feb\u901f\u5730\u627e\u51fa\u8fd9\u4e9b\u6d3b\u8dc3Pod\uff0c\u4f7f\u7528\u65f6\u628a <dataset_name> \u548c <dataset_namespace> \u6362\u6210\u81ea\u5df1\u7684\u5373\u53ef\uff1a kubectl describe pvc <dataset_name> -n <dataset_namespace> | \\ awk '/^Mounted/ {flag=1}; /^Events/ {flag=0}; flag' | \\ awk 'NR==1 {print $3}; NR!=1 {print $1}' | \\ xargs -I {} kubectl get po {} | \\ grep -E \"Running|Terminating|Pending\" | \\ cut -d \" \" -f 1 3. \u4e3a\u4ec0\u4e48\u6211\u5728\u521b\u5efa\u4efb\u52a1\u6302\u8f7d Runtime \u521b\u5efa\u7684 PVC \u7684\u65f6\u5019\u51fa\u73b0 driver name fuse.csi.fluid.io not found in the list of registered CSI drivers \u9519\u8bef\uff1f \u00a7 \u56de\u7b54 :\u8bf7\u67e5\u770b\u4efb\u52a1\u88ab\u8c03\u5ea6\u8282\u70b9\u6240\u5728\u7684 kubelet \u914d\u7f6e\u662f\u5426\u4e3a\u9ed8\u8ba4 /var/lib/kubelet \u3002 \u9996\u5148\u901a\u8fc7\u547d\u4ee4\u67e5\u770bFluid\u7684CSI\u7ec4\u4ef6\u662f\u5426\u6b63\u5e38 \u5982\u4e0b\u7684\u547d\u4ee4\u53ef\u4ee5\u5feb\u901f\u5730\u627e\u51faPod\uff0c\u4f7f\u7528\u65f6\u628a <node_name> \u548c <fluid_namespace> \u6362\u6210\u81ea\u5df1\u7684\u5373\u53ef\uff1a kubectl get pod -n <fluid_namespace> | grep <node_name> # <pod_name> \u4e3a\u4e0a\u4e00\u6b65pod\u540d kubectl logs <pod_name> node-driver-registrar -n <fluid_namespace> kubectl logs <pod_name> plugins -n <fluid_namespace> \u5982\u679c\u4e0a\u8ff0\u6b65\u9aa4\u7684Log\u65e0\u9519\u8bef\uff0c\u8bf7\u67e5\u770bcsidriver\u5bf9\u8c61\u662f\u5426\u5b58\u5728\uff1a kubectl get csidriver \u5982\u679ccsidriver\u5bf9\u8c61\u5b58\u5728\uff0c\u8bf7\u67e5\u770b\u67e5\u770bcsi\u6ce8\u518c\u8282\u70b9\u662f\u5426\u5305\u542b <node_name> \uff1a kubectl get csinode | grep <node_name> \u5982\u679c\u4e0a\u8ff0\u547d\u4ee4\u65e0\u8f93\u51fa\uff0c\u67e5\u770b\u4efb\u52a1\u88ab\u8c03\u5ea6\u8282\u70b9\u6240\u5728\u7684 kubelet \u914d\u7f6e\u662f\u5426\u4e3a\u9ed8\u8ba4 /var/lib/kubelet \u3002\u56e0\u4e3aFluid\u7684CSI\u7ec4\u4ef6\u901a\u8fc7\u56fa\u5b9a\u5730\u5740\u7684socket\u6ce8\u518c\u5230kubelet\uff0c\u9ed8\u8ba4\u4e3a --csi-address=/var/lib/kubelet/csi-plugins/fuse.csi.fluid.io/csi.sock --kubelet-registration-path=/var/lib/kubelet/csi-plugins/fuse.csi.fluid.io/csi.sock \u3002 4. \u4e3a\u4ec0\u4e48\u66f4\u65b0\u4e86fluid\u540e\uff0c\u4f7f\u7528 kubectl get \u67e5\u8be2\u66f4\u65b0\u524d\u521b\u5efa\u7684dataset\uff0c\u53d1\u73b0\u76f8\u6bd4\u65b0\u5efa\u7684dataset\u7f3a\u5c11\u4e86\u67d0\u4e9b\u5b57\u6bb5\uff1f \u00a7 \u56de\u7b54 :\u7531\u4e8e\u6211\u4eec\u5728fluid\u7684\u5347\u7ea7\u8fc7\u7a0b\u4e2d\u53ef\u80fd\u66f4\u65b0\u4e86CRD\uff0c\u4f60\u5728\u65e7\u7248\u672c\u521b\u5efa\u7684dataset\uff0c\u4f1a\u5c06CRD\u4e2d\u65b0\u589e\u7684\u5b57\u6bb5\u8bbe\u7f6e\u4e3a\u7a7a \u4f8b\u5982\uff0c\u5982\u679c\u4f60\u4ecev0.4\u6216\u66f4\u65e9\u7248\u672c\u5347\u7ea7\uff0c\u90a3\u65f6\u5019\u7684dataset\u6ca1\u6709FileNum\u5b57\u6bb5 \u66f4\u65b0fluid\u540e\uff0c\u5982\u679c\u4f60\u4f7f\u7528 kubectl get \u547d\u4ee4\uff0c\u65e0\u6cd5\u67e5\u8be2\u5230\u8be5dataset\u7684FileNum \u4f60\u53ef\u4ee5\u91cd\u5efadataset\uff0c\u65b0\u5efa\u7684dataset\u4f1a\u6b63\u5e38\u663e\u793a\u8fd9\u4e9b\u5b57\u6bb5 5. \u4e3a\u4ec0\u4e48\u5728\u5e94\u7528\u7a0b\u5e8f\u4e2d\u4f7f\u7528 PVC \u65f6\u4f1a\u4ea7\u751f\u4e86 Volume Attachment \u8d85\u65f6\u95ee\u9898\uff1f \u00a7 \u56de\u7b54 :Volume Attachment \u8d85\u65f6\u95ee\u9898\u662f Kubelet \u8fdb\u884c\u8bf7\u6c42 CSI Driver \u65f6\u672a\u6536\u5230 CSI Driver \u7684\u54cd\u5e94\u800c\u9020\u6210\u7684\u8d85\u65f6\u3002 \u8be5\u95ee\u9898\u662f\u7531\u4e8e Fluid \u7684 CSI Driver \u6ca1\u6709\u5b89\u88c5\uff0c\u6216\u8005kubelet\u6ca1\u6709\u8bbf\u95ee CSI Driver \u7684\u6743\u9650\u5bfc\u81f4\u7684\u3002 \u7531\u4e8e CSI Driver \u662f\u7531 Kubelet \u8fdb\u884c\u56de\u8c03\uff0c\u4f46\u662f\u5982\u679c Fluid \u6ca1\u6709\u5b89\u88c5 CSI Driver \u6216\u8005 Kubelet \u6ca1\u6709\u6743\u9650\u67e5\u770b CSI Driver\uff0c\u5c31\u4f1a\u5bfc\u81f4 CSI Plugin \u6ca1\u6709\u88ab\u6b63\u786e\u89e6\u53d1\u3002 \u9996\u5148\u9700\u8981\u4f7f\u7528\u547d\u4ee4 kubectl get csidriver \u67e5\u770b\u662f\u5426\u5b89\u88c5\u4e86 CSI Driver\u3002 \u5982\u679c\u6ca1\u6709\u5b89\u88c5\uff0c\u4f7f\u7528\u547d\u4ee4 kubectl apply -f charts/fluid/fluid/templates/csi/driver.yaml \u8fdb\u884c\u5b89\u88c5\uff0c\u7136\u540e\u89c2\u5bdf PVC \u662f\u5426\u6210\u529f\u6302\u8f7d\u5230\u5e94\u7528\u7a0b\u5e8f\u4e2d\u3002 \u5982\u679c\u4ecd\u672a\u80fd\u89e3\u51b3\uff0c\u4f7f\u7528 export KUBECONFIG=/etc/kubernetes/kubelet.conf && kubectl get csidriver \u6765\u67e5\u770b Kubelet \u80fd\u591f\u5177\u6709\u6743\u9650\u770b\u5230 CSI Driver","title":"\u95ee\u9898\u8bca\u65ad\u53ca\u5904\u7406"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_question/#_1","text":"\u5982\u60a8\u7684\u73af\u5883\u51fa\u73b0\u95ee\u9898\uff0c\u4e5f\u53ef\u5f00 ISSUE \u7ed9\u6211\u4eec\uff0c\u4f1a\u7b2c\u4e00\u65f6\u95f4\u5904\u7406","title":"\u95ee\u9898\u8bca\u65ad\u53ca\u5904\u7406"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_question/#fluid","text":"","title":"Fluid \u5b89\u88c5\u4f7f\u7528\u76f8\u5173\u95ee\u9898"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_question/#1-helmfluid","text":"\u56de\u7b54 :\u63a8\u8350\u6309\u7167\u5b89\u88c5\u6587\u6863\u4f9d\u6b21\u786e\u8ba4Fluid\u7ec4\u4ef6\u662f\u5426\u6b63\u5e38\u8fd0\u884c\u3002 Fluid\u5b89\u88c5\u6587\u6863\u662f\u4ee5 Helm 3 \u4e3a\u4f8b\u8fdb\u884c\u90e8\u7f72\u7684\u3002\u5982\u679c\u60a8\u4f7f\u7528 Helm 3 \u4ee5\u4e0b\u7684\u7248\u672c\u90e8\u7f72Fluid\uff0c \u5e76\u4e14\u9047\u5230\u4e86 CRD\u6ca1\u6709\u6b63\u5e38\u542f\u52a8 \u7684\u60c5\u51b5\uff0c\u8fd9\u53ef\u80fd\u662f\u56e0\u4e3a Helm 3 \u53ca\u5176\u4ee5\u4e0a\u7248\u672c\u4f1a\u5728 helm install \u7684\u65f6\u5019\u81ea\u52a8\u5b89\u88c5CRD\uff0c \u800c\u4f4e\u7248\u672c\u7684Helm\u5219\u4e0d\u4f1a\u3002 \u53c2\u89c1 Helm\u5b98\u65b9\u6587\u6863\u8bf4\u660e \u3002 \u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u60a8\u9700\u8981\u624b\u52a8\u5b89\u88c5CRD\uff1a $ kubectl create -f fluid/crds","title":"1. \u4e3a\u4ec0\u4e48\u6211\u4f7f\u7528Helm\u5b89\u88c5fluid\u5931\u8d25\u4e86\uff1f"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_question/#2-runtime","text":"\u56de\u7b54 :\u8bf7\u68c0\u67e5\u76f8\u5173Pod\u8fd0\u884c\u72b6\u6001\u548cRuntime\u7684Events\u3002 \u53ea\u8981\u6709\u4efb\u4f55\u6d3b\u8dc3Pod\u8fd8\u5728\u4f7f\u7528Fluid\u521b\u5efa\u7684Volume\uff0cFluid\u5c31\u4e0d\u4f1a\u5b8c\u6210\u5220\u9664\u64cd\u4f5c\u3002 \u5982\u4e0b\u7684\u547d\u4ee4\u53ef\u4ee5\u5feb\u901f\u5730\u627e\u51fa\u8fd9\u4e9b\u6d3b\u8dc3Pod\uff0c\u4f7f\u7528\u65f6\u628a <dataset_name> \u548c <dataset_namespace> \u6362\u6210\u81ea\u5df1\u7684\u5373\u53ef\uff1a kubectl describe pvc <dataset_name> -n <dataset_namespace> | \\ awk '/^Mounted/ {flag=1}; /^Events/ {flag=0}; flag' | \\ awk 'NR==1 {print $3}; NR!=1 {print $1}' | \\ xargs -I {} kubectl get po {} | \\ grep -E \"Running|Terminating|Pending\" | \\ cut -d \" \" -f 1","title":"2. \u4e3a\u4ec0\u4e48\u6211\u65e0\u6cd5\u5220\u9664Runtime\uff1f"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_question/#3-runtime-pvc-driver-name-fusecsifluidio-not-found-in-the-list-of-registered-csi-drivers","text":"\u56de\u7b54 :\u8bf7\u67e5\u770b\u4efb\u52a1\u88ab\u8c03\u5ea6\u8282\u70b9\u6240\u5728\u7684 kubelet \u914d\u7f6e\u662f\u5426\u4e3a\u9ed8\u8ba4 /var/lib/kubelet \u3002 \u9996\u5148\u901a\u8fc7\u547d\u4ee4\u67e5\u770bFluid\u7684CSI\u7ec4\u4ef6\u662f\u5426\u6b63\u5e38 \u5982\u4e0b\u7684\u547d\u4ee4\u53ef\u4ee5\u5feb\u901f\u5730\u627e\u51faPod\uff0c\u4f7f\u7528\u65f6\u628a <node_name> \u548c <fluid_namespace> \u6362\u6210\u81ea\u5df1\u7684\u5373\u53ef\uff1a kubectl get pod -n <fluid_namespace> | grep <node_name> # <pod_name> \u4e3a\u4e0a\u4e00\u6b65pod\u540d kubectl logs <pod_name> node-driver-registrar -n <fluid_namespace> kubectl logs <pod_name> plugins -n <fluid_namespace> \u5982\u679c\u4e0a\u8ff0\u6b65\u9aa4\u7684Log\u65e0\u9519\u8bef\uff0c\u8bf7\u67e5\u770bcsidriver\u5bf9\u8c61\u662f\u5426\u5b58\u5728\uff1a kubectl get csidriver \u5982\u679ccsidriver\u5bf9\u8c61\u5b58\u5728\uff0c\u8bf7\u67e5\u770b\u67e5\u770bcsi\u6ce8\u518c\u8282\u70b9\u662f\u5426\u5305\u542b <node_name> \uff1a kubectl get csinode | grep <node_name> \u5982\u679c\u4e0a\u8ff0\u547d\u4ee4\u65e0\u8f93\u51fa\uff0c\u67e5\u770b\u4efb\u52a1\u88ab\u8c03\u5ea6\u8282\u70b9\u6240\u5728\u7684 kubelet \u914d\u7f6e\u662f\u5426\u4e3a\u9ed8\u8ba4 /var/lib/kubelet \u3002\u56e0\u4e3aFluid\u7684CSI\u7ec4\u4ef6\u901a\u8fc7\u56fa\u5b9a\u5730\u5740\u7684socket\u6ce8\u518c\u5230kubelet\uff0c\u9ed8\u8ba4\u4e3a --csi-address=/var/lib/kubelet/csi-plugins/fuse.csi.fluid.io/csi.sock --kubelet-registration-path=/var/lib/kubelet/csi-plugins/fuse.csi.fluid.io/csi.sock \u3002","title":"3. \u4e3a\u4ec0\u4e48\u6211\u5728\u521b\u5efa\u4efb\u52a1\u6302\u8f7d Runtime \u521b\u5efa\u7684 PVC \u7684\u65f6\u5019\u51fa\u73b0 driver name fuse.csi.fluid.io not found in the list of registered CSI drivers \u9519\u8bef\uff1f"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_question/#4-fluid-kubectl-get-datasetdataset","text":"\u56de\u7b54 :\u7531\u4e8e\u6211\u4eec\u5728fluid\u7684\u5347\u7ea7\u8fc7\u7a0b\u4e2d\u53ef\u80fd\u66f4\u65b0\u4e86CRD\uff0c\u4f60\u5728\u65e7\u7248\u672c\u521b\u5efa\u7684dataset\uff0c\u4f1a\u5c06CRD\u4e2d\u65b0\u589e\u7684\u5b57\u6bb5\u8bbe\u7f6e\u4e3a\u7a7a \u4f8b\u5982\uff0c\u5982\u679c\u4f60\u4ecev0.4\u6216\u66f4\u65e9\u7248\u672c\u5347\u7ea7\uff0c\u90a3\u65f6\u5019\u7684dataset\u6ca1\u6709FileNum\u5b57\u6bb5 \u66f4\u65b0fluid\u540e\uff0c\u5982\u679c\u4f60\u4f7f\u7528 kubectl get \u547d\u4ee4\uff0c\u65e0\u6cd5\u67e5\u8be2\u5230\u8be5dataset\u7684FileNum \u4f60\u53ef\u4ee5\u91cd\u5efadataset\uff0c\u65b0\u5efa\u7684dataset\u4f1a\u6b63\u5e38\u663e\u793a\u8fd9\u4e9b\u5b57\u6bb5","title":"4. \u4e3a\u4ec0\u4e48\u66f4\u65b0\u4e86fluid\u540e\uff0c\u4f7f\u7528 kubectl get \u67e5\u8be2\u66f4\u65b0\u524d\u521b\u5efa\u7684dataset\uff0c\u53d1\u73b0\u76f8\u6bd4\u65b0\u5efa\u7684dataset\u7f3a\u5c11\u4e86\u67d0\u4e9b\u5b57\u6bb5\uff1f"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_question/#5-pvc-volume-attachment","text":"\u56de\u7b54 :Volume Attachment \u8d85\u65f6\u95ee\u9898\u662f Kubelet \u8fdb\u884c\u8bf7\u6c42 CSI Driver \u65f6\u672a\u6536\u5230 CSI Driver \u7684\u54cd\u5e94\u800c\u9020\u6210\u7684\u8d85\u65f6\u3002 \u8be5\u95ee\u9898\u662f\u7531\u4e8e Fluid \u7684 CSI Driver \u6ca1\u6709\u5b89\u88c5\uff0c\u6216\u8005kubelet\u6ca1\u6709\u8bbf\u95ee CSI Driver \u7684\u6743\u9650\u5bfc\u81f4\u7684\u3002 \u7531\u4e8e CSI Driver \u662f\u7531 Kubelet \u8fdb\u884c\u56de\u8c03\uff0c\u4f46\u662f\u5982\u679c Fluid \u6ca1\u6709\u5b89\u88c5 CSI Driver \u6216\u8005 Kubelet \u6ca1\u6709\u6743\u9650\u67e5\u770b CSI Driver\uff0c\u5c31\u4f1a\u5bfc\u81f4 CSI Plugin \u6ca1\u6709\u88ab\u6b63\u786e\u89e6\u53d1\u3002 \u9996\u5148\u9700\u8981\u4f7f\u7528\u547d\u4ee4 kubectl get csidriver \u67e5\u770b\u662f\u5426\u5b89\u88c5\u4e86 CSI Driver\u3002 \u5982\u679c\u6ca1\u6709\u5b89\u88c5\uff0c\u4f7f\u7528\u547d\u4ee4 kubectl apply -f charts/fluid/fluid/templates/csi/driver.yaml \u8fdb\u884c\u5b89\u88c5\uff0c\u7136\u540e\u89c2\u5bdf PVC \u662f\u5426\u6210\u529f\u6302\u8f7d\u5230\u5e94\u7528\u7a0b\u5e8f\u4e2d\u3002 \u5982\u679c\u4ecd\u672a\u80fd\u89e3\u51b3\uff0c\u4f7f\u7528 export KUBECONFIG=/etc/kubernetes/kubelet.conf && kubectl get csidriver \u6765\u67e5\u770b Kubelet \u80fd\u591f\u5177\u6709\u6743\u9650\u770b\u5230 CSI Driver","title":"5. \u4e3a\u4ec0\u4e48\u5728\u5e94\u7528\u7a0b\u5e8f\u4e2d\u4f7f\u7528 PVC \u65f6\u4f1a\u4ea7\u751f\u4e86 Volume Attachment \u8d85\u65f6\u95ee\u9898\uff1f"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_s3_ufs_example/","text":"\u62e5\u62b1\u4e91\u539f\u751f\uff0cFluid \u7ed3\u5408 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff1a\u52a0\u901f\u5bf9\u8c61\u5b58\u50a8\u4f7f\u7528 S3 \u534f\u8bae \u00a7 JindoFSx \u63d0\u4f9b\u4e86\u8bbf\u95ee S3 \u534f\u8bae\u7684\u80fd\u529b\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e S3 \u4f5c\u4e3a JindoFSx \u7684\u540e\u7aef\u5b58\u50a8\uff0c\u4f7f Fluid \u901a\u8fc7 JindoFSx \u6765\u8bbf\u95ee S3 \u4e0a\u7684\u6570\u636e\uff0c\u540c\u65f6 JindoFSx \u4e5f\u63d0\u4f9b\u4e86\u5bf9 S3 \u4e0a\u7684\u6570\u636e\u4ee5\u53ca\u5143\u6570\u636e\u7684\u7f13\u5b58\u52a0\u901f\u529f\u80fd\u3002 \u672c\u6587\u6863\u5c55\u793a\u4e86\u5982\u4f55\u5728 Fluid \u4e2d\u4ee5\u58f0\u660e\u5f0f\u7684\u65b9\u5f0f\u5b8c\u6210 JindoFSx \u90e8\u7f72\uff0c\u5bf9\u63a5 S3 \u6570\u636e\u6e90\uff08\u5305\u62ec\u63d0\u4f9b S3 \u517c\u5bb9\u7684\u6240\u6709\u5bf9\u8c61\u5b58\u50a8\uff09\u3002 \u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 \u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz \u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: s3 spec: mounts: - mountPoint: s3://<bucket>/<dir> options: fs.s3.accessKeyId: xxx fs.s3.accessKeySecret: xxx fs.s3.endpoint: s3.ap-east-1.amazonaws.com fs.s3.region: ap-east-1 name: s3 path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: s3 spec: replicas: 2 tieredstore: levels: - mediumtype: MEM path: /dev/shm quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d s3 \u7684\u8def\u5f84\uff0c\u652f\u6301\u6807\u51c6 s3 \u534f\u8bae\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u7684 worker \u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c\u5355\u4f4dGi\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b dataset \u7684\u60c5\u51b5 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE s3 210MiB 0.00B 180.00GiB 0.0% Bound 1h \u6267\u884c create \u547d\u4ee4 $ kubectl create -f runtime.yaml \u67e5\u770b dataset \u72b6\u6001 \u00a7 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE s3 180.01GiB 0.00B 261.00GiB 0.0% Bound 62m \u68c0\u67e5 PV\uff0cPVC \u521b\u5efa\u60c5\u51b5 \u00a7 $ kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/s3 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/s3 Bound hadoop 100Gi RWX 58m","title":"\u62e5\u62b1\u4e91\u539f\u751f\uff0cFluid \u7ed3\u5408 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff1a\u52a0\u901f\u5bf9\u8c61\u5b58\u50a8\u4f7f\u7528 S3 \u534f\u8bae"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_s3_ufs_example/#fluid-jindofsx-s3","text":"JindoFSx \u63d0\u4f9b\u4e86\u8bbf\u95ee S3 \u534f\u8bae\u7684\u80fd\u529b\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e S3 \u4f5c\u4e3a JindoFSx \u7684\u540e\u7aef\u5b58\u50a8\uff0c\u4f7f Fluid \u901a\u8fc7 JindoFSx \u6765\u8bbf\u95ee S3 \u4e0a\u7684\u6570\u636e\uff0c\u540c\u65f6 JindoFSx \u4e5f\u63d0\u4f9b\u4e86\u5bf9 S3 \u4e0a\u7684\u6570\u636e\u4ee5\u53ca\u5143\u6570\u636e\u7684\u7f13\u5b58\u52a0\u901f\u529f\u80fd\u3002 \u672c\u6587\u6863\u5c55\u793a\u4e86\u5982\u4f55\u5728 Fluid \u4e2d\u4ee5\u58f0\u660e\u5f0f\u7684\u65b9\u5f0f\u5b8c\u6210 JindoFSx \u90e8\u7f72\uff0c\u5bf9\u63a5 S3 \u6570\u636e\u6e90\uff08\u5305\u62ec\u63d0\u4f9b S3 \u517c\u5bb9\u7684\u6240\u6709\u5bf9\u8c61\u5b58\u50a8\uff09\u3002","title":"\u62e5\u62b1\u4e91\u539f\u751f\uff0cFluid \u7ed3\u5408 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff1a\u52a0\u901f\u5bf9\u8c61\u5b58\u50a8\u4f7f\u7528 S3 \u534f\u8bae"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_s3_ufs_example/#_1","text":"kubectl create ns fluid-system","title":"\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_s3_ufs_example/#fluid","text":"","title":"\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_s3_ufs_example/#helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_s3_ufs_example/#fluid_1","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_s3_ufs_example/#dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: s3 spec: mounts: - mountPoint: s3://<bucket>/<dir> options: fs.s3.accessKeyId: xxx fs.s3.accessKeySecret: xxx fs.s3.endpoint: s3.ap-east-1.amazonaws.com fs.s3.region: ap-east-1 name: s3 path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: s3 spec: replicas: 2 tieredstore: levels: - mediumtype: MEM path: /dev/shm quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d s3 \u7684\u8def\u5f84\uff0c\u652f\u6301\u6807\u51c6 s3 \u534f\u8bae\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u7684 worker \u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c\u5355\u4f4dGi\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b dataset \u7684\u60c5\u51b5 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE s3 210MiB 0.00B 180.00GiB 0.0% Bound 1h \u6267\u884c create \u547d\u4ee4 $ kubectl create -f runtime.yaml","title":"\u521b\u5efa dataset \u548c JindoRuntime"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_s3_ufs_example/#dataset","text":"$ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE s3 180.01GiB 0.00B 261.00GiB 0.0% Bound 62m","title":"\u67e5\u770b dataset \u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_s3_ufs_example/#pvpvc","text":"$ kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/s3 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/s3 Bound hadoop 100Gi RWX 58m","title":"\u68c0\u67e5 PV\uff0cPVC \u521b\u5efa\u60c5\u51b5"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/","text":"Fluid + JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u9ad8\u9636\u529f\u80fd\u4f7f\u7528 \u00a7 \u4f7f\u7528\u5217\u8868 \u6302\u8f7d\u70b9\u5728\u6839\u76ee\u5f55\u4e0b Secret \u52a0\u5bc6 AK \u53c2\u6570 Raft 3 master \u6a21\u5f0f \u4f7f\u7528 Placement \u90e8\u7f72\u591a\u4e2a runtime \u4f7f\u7528 NoseSelector \u90e8\u7f72\u8282\u70b9 \u4f7f\u7528 dataset nodeAffinity \u529f\u80fd Worker \u4e2a\u6570\u6269\u7f29\u5bb9 \u4f7f\u7528 tolerations \u529f\u80fd Resource \u8d44\u6e90 Fuse \u56de\u6536\u7b56\u7565 JindoFSx \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528 JindoFSx Fuse \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528 FuseOnly \u4f7f\u7528\u65b9\u5f0f \u591a\u6302\u8f7d\u70b9 master\u5143\u6570\u636e\u6301\u4e45\u5316 \u6302\u8f7d\u70b9\u5728\u6839\u76ee\u5f55\u4e0b \u00a7 \u9ed8\u8ba4\u4f7f\u7528 JindoRuntime \u4f1a\u5728\u6302\u8f7d\u70b9\u591a\u4e00\u5c42 / \u7684\u76ee\u5f55\uff0c\u5982\u679c\u60f3\u6302\u8f7d\u5728\u6839\u76ee\u5f55\u4e0b\u53ef\u4ee5\u5728 dataset \u91cc\u8fdb\u884c\u53c2\u6570\u6307\u5b9a apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase path: / \u6307\u5b9a spec.mounts.path = /\uff0c\u5219\u4f1a\u5c06\u6587\u4ef6\u6302\u8f7d\u5728\u6839\u76ee\u5f55\u4e0b Secret \u52a0\u5bc6 AK \u53c2\u6570 \u00a7 apiVersion: v1 kind: Secret metadata: name: mysecret stringData: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> \u5728 dataset \u91cc\u4f7f\u7528 secret apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://<OSS_BUCKET>/<OSS_DIRECTORY>/ name: hadoop options: fs.oss.endpoint: <OSS_ENDPOINT> encryptOptions: - name: fs.oss.accessKeyId valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeyId - name: fs.oss.accessKeySecret valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeySecret Raft 3 master \u6a21\u5f0f \u00a7 JindoRuntime \u6a21\u5f0f\u542f\u52a8\u4e00\u4e2a master\uff0c\u53ef\u4ee5\u901a\u8fc7\u4f7f\u7528 master.replicas \u6765\u542f\u52a8 3 \u4e2a master \u8fdb\u884c HA \u8f6c\u6362\u3002 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 3 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 2G high: \"0.8\" low: \"0.7\" master: replicas: 3 master.replicas = 3 : \u542f\u52a8 3 \u4e2a master pod \u4f7f\u7528 Placement \u90e8\u7f72\u591a\u4e2a runtime \u00a7 JindoRuntime \u542f\u52a8\u540e\uff0c\u9ed8\u8ba4 worker \u8282\u70b9\u4e0a\u53ea\u80fd\u542f\u52a8\u4e00\u4e2a runtime \u7684worker\uff0c\u5c5e\u4e8e\u72ec\u5360\u6a21\u5f0f\uff0c\u5982\u679c\u60f3\u8981\u5728\u540c\u4e00\u4e2a\u8282\u70b9\u4e0a\u652f\u6301\u90e8\u7f72\u591a\u4e2a worker\uff0c\u53ef\u4ee5\u4f7f\u7528 shared \u6a21\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase placement: \"Shared\" \u4f7f\u7528 NoseSelector \u90e8\u7f72\u8282\u70b9 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /mnt/disk1 quota: 10Gi high: \"0.9\" low: \"0.8\" master: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1 worker: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1 fuse: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1 \u4f7f\u7528 dataset nodeAffinity \u529f\u80fd \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: hbase-cache operator: In values: - \"true\" Worker \u4e2a\u6570\u6269\u7f29\u5bb9 \u00a7 \u4f7f\u7528 kubectl scale \u5bf9 Runtime \u7684 Worker \u6570\u91cf\u8fdb\u884c\u8c03\u6574 $ kubectl scale jindoruntime <runtime_name> --replicas=<replica_num> \u5176\u4e2d * runtime_name\uff1aruntime \u7684\u540d\u5b57 * replica_num\uff1a\u8868\u793a\u6269\u6240\u5bb9\u7684 Worker \u6570\u91cf \u4f7f\u7528 tolerations \u529f\u80fd \u00a7 \u60a8\u53ef\u4ee5\u5728 dataset \u91cc\u5b9a\u4e49 tolerations \u6307\u5b9a worker \u8282\u70b9\u7684\u8c03\u5ea6 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase tolerations: - key: hbase operator: Equal value: \"true\" \u4e5f\u53ef\u4ee5\u4e3a master/worker/fuse \u5355\u72ec\u6307\u5b9a tolerations \u8c03\u5ea6\u7b56\u7565 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 3 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 2Gi high: \"0.8\" low: \"0.7\" master: tolerations: - key: hbase operator: Equal value: \"true\" worker: tolerations: - key: hbase operator: Equal value: \"true\" fuse: tolerations: - key: hbase operator: Equal value: \"true\" Resource \u8d44\u6e90 \u00a7 \u53ef\u4ee5\u6307\u5b9a master/worker \u7b49\u7684 resource \u8d44\u6e90 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" master: resources: limits: cpu: \"4\" memory: \"8Gi\" requests: cpu: \"2\" memory: \"3Gi\" worker: resources: limits: cpu: \"4\" memory: \"8Gi\" requests: cpu: \"2\" memory: \"3Gi\" Fuse \u56de\u6536\u7b56\u7565 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: cleanPolicy: OnDemand fuse.cleanPolicy: onDemand / OnRuntimeDeleted \u5206\u522b\u8868\u793a\u6309\u9700\u542f\u52a8\uff0c\u4efb\u52a1\u7ed3\u675f fuse \u4e5f\u7ed3\u675f\u548c\u5728 runtime \u9500\u6bc1\u7684\u65f6\u5019\u548c\u5176\u4ed6\u7ec4\u4ef6\u4e00\u8d77\u9500\u6bc1\u6389 JindoFSx \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528 \u00a7 \u4e0b\u9762\u4ecb\u7ecd\u4e00\u4e9b\u548c JindoFSx \u7f13\u5b58\u76f8\u5173\u7684\u53c2\u6570\uff0c\u53ef\u4ee5\u6309\u7167\u5b9e\u9645\u573a\u666f\u8fdb\u884c\u9009\u62e9 Parameter Value Description fs.jindofsx.data.cache.enable true/false \u662f\u5426\u6253\u5f00\u6570\u636e\u7f13\u5b58\uff0c\u9ed8\u8ba4\u4e3a\u6253\u5f00 fs.jindofsx.meta.cache.enable true/false \u662f\u5426\u6253\u5f00\u5143\u6570\u636e\u7f13\u5b58\uff0c\u9ed8\u8ba4\u4e3a\u5173\u95ed fs.jindofsx.slicelet.cache.enable true/false \u662f\u5426\u6253\u5f00\u5c0f\u6587\u4ef6\u7f13\u5b58\u4f18\u5316(\u5199\u78c1\u76d8)\uff0c\u9ed8\u8ba4\u4e3a\u5173\u95ed fs.jindofsx.short.circuit.enable true/false \u662f\u5426\u6253\u5f00\u77ed\u8def\u8bfb\uff0c\u9ed8\u8ba4\u4e3a\u6253\u5f00 \u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: properties: fs.jindofsx.data.cache.enable: \"false\" fs.jindofsx.meta.cache.enable: \"true\" fs.jindofsx.slicelet.cache.enable: \"false\" fs.jindofsx.short.circuit.enable: \"false\" \u4ee5 kv \u65b9\u5f0f\u5199\u5728 spec.fuse.properties \u91cc\uff0c\u6309\u7167\u9700\u8981\u586b\u5199\u5373\u53ef\u3002 JindoFSx Fuse \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528 \u00a7 \u53c2\u6570\u540d\u79f0 \u53c2\u6570\u8bf4\u660e \u4f7f\u7528\u8303\u4f8b auto_unmount fuse\u8fdb\u7a0b\u9000\u51fa\u540e\u81ea\u52a8umount\u6302\u8f7d\u8282\u70b9\u3002 -oauto_unmount ro \u53ea\u8bfb\u6302\u8f7d\uff0c\u542f\u7528\u53c2\u6570\u540e\u4e0d\u5141\u8bb8\u5199\u64cd\u4f5c\u3002 -oro direct_io \u5f00\u542f\u540e\uff0c\u8bfb\u5199\u6587\u4ef6\u53ef\u4ee5\u7ed5\u8fc7page cache\u3002 -odirect_io kernel_cache \u5f00\u542f\u540e\uff0c\u5229\u7528\u5185\u6838\u7f13\u5b58\u4f18\u5316\u8bfb\u6027\u80fd\u3002 -okernel_cache auto_cache \u9ed8\u8ba4\u5f00\u542f\uff0c\u4e0ekernel_cache \u4e8c\u9009\u4e00\uff0c\u4e0ekernel_cache\u4e0d\u540c\u7684\u662f\uff0c\u5982\u679c\u6587\u4ef6\u5927\u5c0f\u6216\u4fee\u6539\u65f6\u95f4\u53d1\u751f\u53d8\u5316\uff0c\u7f13\u5b58\u5c31\u4f1a\u5931\u6548\u3002 -oauto_cache entry_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-oentry_timeout=60 attr_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-oattr_timeout=60 negative_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u5931\u8d25\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-onegative_timeout=30 \u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: args: - -oro - -oattr_timeout=60 - -oentry_timeout=60 - -onegative_timeout=60 \u4ee5\u6570\u7ec4\u65b9\u5f0f\u5199\u5728 spec.fuse.args \u91cc\uff0c\u6309\u7167\u9700\u8981\u586b\u5199\u5373\u53ef \u5982\u5355\u72ec\u914d\u7f6e\u5ba2\u6237\u7aefpread\u53c2\u6570\uff1a apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: args: - -oro - -ometrics_port=0 - -okernel_cache - -oattr_timeout=7200 - -oentry_timeout=7200 - -onegative_timeout=7200 - -opread FuseOnly \u4f7f\u7528\u65b9\u5f0f \u00a7 \u5982\u60a8\u60f3\u4f7f\u7528 FuseOnly \u65b9\u5f0f\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: master: disabled: true worker: disabled: true \u591a\u6302\u8f7d\u70b9 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: spark - mountPoint: oss://test-bucket-2/dir2/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop \u5728\u6302\u8f7d pvc \u548c fuse \u4ee5\u53ca\u505a dataload \u7684\u65f6\u5019\uff0c\u8fd9\u4e24\u4e2a\u6570\u636e\u6e90\u7684\u65b9\u5f0f\u5c06\u4ee5 mounts.name \u4f5c\u4e3a\u533a\u5206\uff0c\u6bd4\u5982 /spark \u8def\u5f84\u4e0b\u8bbf\u95ee oss://test-bucket-1/dir1/ \u4e0b\u7684\u6587\u4ef6\uff0c/hadoop \u8def\u5f84\u4e0b\u8bbf\u95ee oss://test-bucket-2/dir2/ \u4e0b\u7684\u6587\u4ef6\u5185\u5bb9 master\u5143\u6570\u636e\u6301\u4e45\u5316 \u00a7 \u5982\u679c\u60f3\u901a\u8fc7\u7ed9master\u6302\u8f7dvolume\u7684\u65b9\u5f0f\uff0c\u8ba9master\u7684\u5143\u6570\u636e\u6301\u4e45\u5316\u5230\u6307\u5b9a\u7684\u5b58\u50a8\u4e0a\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" volumes: - name: nas persistentVolumeClaim: claimName: nas-test master: volumeMounts: - name: nas subPath: test mountPath: /var/cache/ properties: namespace.meta-dir: \"/var/cache/\" \u8be5\u4f8b\u5b50\u5c06\u5143\u6570\u636e\u76ee\u5f55 namespace.meta-dir \u6307\u5b9a\u5230\u6302\u8f7d\u7684volume\u4e0a\uff0c\u5e76\u8fdb\u884c\u6301\u4e45\u5316\uff0c\u6ce8\u610f\u6302\u8f7d\u7684vomlume\u5fc5\u987b\u5c31\u6709\u8bfb/\u5199/\u5220\u9664\u6743\u9650\uff0c\u4e0d\u53ef\u6307\u5b9a\u4e3a\u53ea\u8bfb","title":"Fluid + JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u9ad8\u9636\u529f\u80fd\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#fluid-jindofsx","text":"\u4f7f\u7528\u5217\u8868 \u6302\u8f7d\u70b9\u5728\u6839\u76ee\u5f55\u4e0b Secret \u52a0\u5bc6 AK \u53c2\u6570 Raft 3 master \u6a21\u5f0f \u4f7f\u7528 Placement \u90e8\u7f72\u591a\u4e2a runtime \u4f7f\u7528 NoseSelector \u90e8\u7f72\u8282\u70b9 \u4f7f\u7528 dataset nodeAffinity \u529f\u80fd Worker \u4e2a\u6570\u6269\u7f29\u5bb9 \u4f7f\u7528 tolerations \u529f\u80fd Resource \u8d44\u6e90 Fuse \u56de\u6536\u7b56\u7565 JindoFSx \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528 JindoFSx Fuse \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528 FuseOnly \u4f7f\u7528\u65b9\u5f0f \u591a\u6302\u8f7d\u70b9 master\u5143\u6570\u636e\u6301\u4e45\u5316","title":"Fluid + JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u9ad8\u9636\u529f\u80fd\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#_1","text":"\u9ed8\u8ba4\u4f7f\u7528 JindoRuntime \u4f1a\u5728\u6302\u8f7d\u70b9\u591a\u4e00\u5c42 / \u7684\u76ee\u5f55\uff0c\u5982\u679c\u60f3\u6302\u8f7d\u5728\u6839\u76ee\u5f55\u4e0b\u53ef\u4ee5\u5728 dataset \u91cc\u8fdb\u884c\u53c2\u6570\u6307\u5b9a apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase path: / \u6307\u5b9a spec.mounts.path = /\uff0c\u5219\u4f1a\u5c06\u6587\u4ef6\u6302\u8f7d\u5728\u6839\u76ee\u5f55\u4e0b","title":"\u6302\u8f7d\u70b9\u5728\u6839\u76ee\u5f55\u4e0b"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#secret-ak","text":"apiVersion: v1 kind: Secret metadata: name: mysecret stringData: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> \u5728 dataset \u91cc\u4f7f\u7528 secret apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://<OSS_BUCKET>/<OSS_DIRECTORY>/ name: hadoop options: fs.oss.endpoint: <OSS_ENDPOINT> encryptOptions: - name: fs.oss.accessKeyId valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeyId - name: fs.oss.accessKeySecret valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeySecret","title":"Secret \u52a0\u5bc6 AK \u53c2\u6570"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#raft-3-master","text":"JindoRuntime \u6a21\u5f0f\u542f\u52a8\u4e00\u4e2a master\uff0c\u53ef\u4ee5\u901a\u8fc7\u4f7f\u7528 master.replicas \u6765\u542f\u52a8 3 \u4e2a master \u8fdb\u884c HA \u8f6c\u6362\u3002 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 3 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 2G high: \"0.8\" low: \"0.7\" master: replicas: 3 master.replicas = 3 : \u542f\u52a8 3 \u4e2a master pod","title":"Raft 3 master \u6a21\u5f0f"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#placement-runtime","text":"JindoRuntime \u542f\u52a8\u540e\uff0c\u9ed8\u8ba4 worker \u8282\u70b9\u4e0a\u53ea\u80fd\u542f\u52a8\u4e00\u4e2a runtime \u7684worker\uff0c\u5c5e\u4e8e\u72ec\u5360\u6a21\u5f0f\uff0c\u5982\u679c\u60f3\u8981\u5728\u540c\u4e00\u4e2a\u8282\u70b9\u4e0a\u652f\u6301\u90e8\u7f72\u591a\u4e2a worker\uff0c\u53ef\u4ee5\u4f7f\u7528 shared \u6a21\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase placement: \"Shared\"","title":"\u4f7f\u7528 Placement \u90e8\u7f72\u591a\u4e2a runtime"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#noseselector","text":"apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /mnt/disk1 quota: 10Gi high: \"0.9\" low: \"0.8\" master: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1 worker: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1 fuse: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1","title":"\u4f7f\u7528 NoseSelector \u90e8\u7f72\u8282\u70b9"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#dataset-nodeaffinity","text":"apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: hbase-cache operator: In values: - \"true\"","title":"\u4f7f\u7528 dataset nodeAffinity \u529f\u80fd"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#worker","text":"\u4f7f\u7528 kubectl scale \u5bf9 Runtime \u7684 Worker \u6570\u91cf\u8fdb\u884c\u8c03\u6574 $ kubectl scale jindoruntime <runtime_name> --replicas=<replica_num> \u5176\u4e2d * runtime_name\uff1aruntime \u7684\u540d\u5b57 * replica_num\uff1a\u8868\u793a\u6269\u6240\u5bb9\u7684 Worker \u6570\u91cf","title":"Worker \u4e2a\u6570\u6269\u7f29\u5bb9"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#tolerations","text":"\u60a8\u53ef\u4ee5\u5728 dataset \u91cc\u5b9a\u4e49 tolerations \u6307\u5b9a worker \u8282\u70b9\u7684\u8c03\u5ea6 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase tolerations: - key: hbase operator: Equal value: \"true\" \u4e5f\u53ef\u4ee5\u4e3a master/worker/fuse \u5355\u72ec\u6307\u5b9a tolerations \u8c03\u5ea6\u7b56\u7565 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 3 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 2Gi high: \"0.8\" low: \"0.7\" master: tolerations: - key: hbase operator: Equal value: \"true\" worker: tolerations: - key: hbase operator: Equal value: \"true\" fuse: tolerations: - key: hbase operator: Equal value: \"true\"","title":"\u4f7f\u7528 tolerations \u529f\u80fd"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#resource","text":"\u53ef\u4ee5\u6307\u5b9a master/worker \u7b49\u7684 resource \u8d44\u6e90 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" master: resources: limits: cpu: \"4\" memory: \"8Gi\" requests: cpu: \"2\" memory: \"3Gi\" worker: resources: limits: cpu: \"4\" memory: \"8Gi\" requests: cpu: \"2\" memory: \"3Gi\"","title":"Resource \u8d44\u6e90"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#fuse","text":"apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: cleanPolicy: OnDemand fuse.cleanPolicy: onDemand / OnRuntimeDeleted \u5206\u522b\u8868\u793a\u6309\u9700\u542f\u52a8\uff0c\u4efb\u52a1\u7ed3\u675f fuse \u4e5f\u7ed3\u675f\u548c\u5728 runtime \u9500\u6bc1\u7684\u65f6\u5019\u548c\u5176\u4ed6\u7ec4\u4ef6\u4e00\u8d77\u9500\u6bc1\u6389","title":"Fuse \u56de\u6536\u7b56\u7565"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#jindofsx","text":"\u4e0b\u9762\u4ecb\u7ecd\u4e00\u4e9b\u548c JindoFSx \u7f13\u5b58\u76f8\u5173\u7684\u53c2\u6570\uff0c\u53ef\u4ee5\u6309\u7167\u5b9e\u9645\u573a\u666f\u8fdb\u884c\u9009\u62e9 Parameter Value Description fs.jindofsx.data.cache.enable true/false \u662f\u5426\u6253\u5f00\u6570\u636e\u7f13\u5b58\uff0c\u9ed8\u8ba4\u4e3a\u6253\u5f00 fs.jindofsx.meta.cache.enable true/false \u662f\u5426\u6253\u5f00\u5143\u6570\u636e\u7f13\u5b58\uff0c\u9ed8\u8ba4\u4e3a\u5173\u95ed fs.jindofsx.slicelet.cache.enable true/false \u662f\u5426\u6253\u5f00\u5c0f\u6587\u4ef6\u7f13\u5b58\u4f18\u5316(\u5199\u78c1\u76d8)\uff0c\u9ed8\u8ba4\u4e3a\u5173\u95ed fs.jindofsx.short.circuit.enable true/false \u662f\u5426\u6253\u5f00\u77ed\u8def\u8bfb\uff0c\u9ed8\u8ba4\u4e3a\u6253\u5f00 \u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: properties: fs.jindofsx.data.cache.enable: \"false\" fs.jindofsx.meta.cache.enable: \"true\" fs.jindofsx.slicelet.cache.enable: \"false\" fs.jindofsx.short.circuit.enable: \"false\" \u4ee5 kv \u65b9\u5f0f\u5199\u5728 spec.fuse.properties \u91cc\uff0c\u6309\u7167\u9700\u8981\u586b\u5199\u5373\u53ef\u3002","title":"JindoFSx \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#jindofsx-fuse","text":"\u53c2\u6570\u540d\u79f0 \u53c2\u6570\u8bf4\u660e \u4f7f\u7528\u8303\u4f8b auto_unmount fuse\u8fdb\u7a0b\u9000\u51fa\u540e\u81ea\u52a8umount\u6302\u8f7d\u8282\u70b9\u3002 -oauto_unmount ro \u53ea\u8bfb\u6302\u8f7d\uff0c\u542f\u7528\u53c2\u6570\u540e\u4e0d\u5141\u8bb8\u5199\u64cd\u4f5c\u3002 -oro direct_io \u5f00\u542f\u540e\uff0c\u8bfb\u5199\u6587\u4ef6\u53ef\u4ee5\u7ed5\u8fc7page cache\u3002 -odirect_io kernel_cache \u5f00\u542f\u540e\uff0c\u5229\u7528\u5185\u6838\u7f13\u5b58\u4f18\u5316\u8bfb\u6027\u80fd\u3002 -okernel_cache auto_cache \u9ed8\u8ba4\u5f00\u542f\uff0c\u4e0ekernel_cache \u4e8c\u9009\u4e00\uff0c\u4e0ekernel_cache\u4e0d\u540c\u7684\u662f\uff0c\u5982\u679c\u6587\u4ef6\u5927\u5c0f\u6216\u4fee\u6539\u65f6\u95f4\u53d1\u751f\u53d8\u5316\uff0c\u7f13\u5b58\u5c31\u4f1a\u5931\u6548\u3002 -oauto_cache entry_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-oentry_timeout=60 attr_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-oattr_timeout=60 negative_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u5931\u8d25\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-onegative_timeout=30 \u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: args: - -oro - -oattr_timeout=60 - -oentry_timeout=60 - -onegative_timeout=60 \u4ee5\u6570\u7ec4\u65b9\u5f0f\u5199\u5728 spec.fuse.args \u91cc\uff0c\u6309\u7167\u9700\u8981\u586b\u5199\u5373\u53ef \u5982\u5355\u72ec\u914d\u7f6e\u5ba2\u6237\u7aefpread\u53c2\u6570\uff1a apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: args: - -oro - -ometrics_port=0 - -okernel_cache - -oattr_timeout=7200 - -oentry_timeout=7200 - -onegative_timeout=7200 - -opread","title":"JindoFSx Fuse \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#fuseonly","text":"\u5982\u60a8\u60f3\u4f7f\u7528 FuseOnly \u65b9\u5f0f\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: master: disabled: true worker: disabled: true","title":"FuseOnly \u4f7f\u7528\u65b9\u5f0f"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#_2","text":"apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: spark - mountPoint: oss://test-bucket-2/dir2/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop \u5728\u6302\u8f7d pvc \u548c fuse \u4ee5\u53ca\u505a dataload \u7684\u65f6\u5019\uff0c\u8fd9\u4e24\u4e2a\u6570\u636e\u6e90\u7684\u65b9\u5f0f\u5c06\u4ee5 mounts.name \u4f5c\u4e3a\u533a\u5206\uff0c\u6bd4\u5982 /spark \u8def\u5f84\u4e0b\u8bbf\u95ee oss://test-bucket-1/dir1/ \u4e0b\u7684\u6587\u4ef6\uff0c/hadoop \u8def\u5f84\u4e0b\u8bbf\u95ee oss://test-bucket-2/dir2/ \u4e0b\u7684\u6587\u4ef6\u5185\u5bb9","title":"\u591a\u6302\u8f7d\u70b9"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#master","text":"\u5982\u679c\u60f3\u901a\u8fc7\u7ed9master\u6302\u8f7dvolume\u7684\u65b9\u5f0f\uff0c\u8ba9master\u7684\u5143\u6570\u636e\u6301\u4e45\u5316\u5230\u6307\u5b9a\u7684\u5b58\u50a8\u4e0a\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" volumes: - name: nas persistentVolumeClaim: claimName: nas-test master: volumeMounts: - name: nas subPath: test mountPath: /var/cache/ properties: namespace.meta-dir: \"/var/cache/\" \u8be5\u4f8b\u5b50\u5c06\u5143\u6570\u636e\u76ee\u5f55 namespace.meta-dir \u6307\u5b9a\u5230\u6302\u8f7d\u7684volume\u4e0a\uff0c\u5e76\u8fdb\u884c\u6301\u4e45\u5316\uff0c\u6ce8\u610f\u6302\u8f7d\u7684vomlume\u5fc5\u987b\u5c31\u6709\u8bfb/\u5199/\u5220\u9664\u6743\u9650\uff0c\u4e0d\u53ef\u6307\u5b9a\u4e3a\u53ea\u8bfb","title":"master\u5143\u6570\u636e\u6301\u4e45\u5316"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_nas_ufs_example/","text":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001\u4e0b\u8f7d\u5b89\u88c5\u5305 \u00a7 \u70b9\u51fb\u4e0b\u8f7d 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-0.7.0.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907 \u00a7 5.1 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: nas spec: mounts: - mountPoint: local:///mnt/nas name: nasData --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: nas spec: replicas: 2 tieredstore: levels: - mediumtype: MEM path: /dev/shm quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d NAS \u7684\u8def\u5f84\uff0c local:// \u4e3a\u672c\u5730\u8def\u5f84\u524d\u7f00\uff0c /mnt/nas \u8868\u793a\u672c\u5730\u771f\u5b9e\u8def\u5f84\uff0c\u8be5\u8def\u5f84\u9700\u8981\u5728\u5bbf\u4e3b\u673a\u4e0a\u53ef\u89c1\u3002 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JindoRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime nas NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE nas Ready Ready 62m \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/nas 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/nas Bound hadoop 100Gi RWX 58m \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 NAS \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: test spec: dataset: name: nas namespace: default target: - path: /nasData replicas: 1 \u5176\u4e2d /nasData \u4e2d nasData \u4e3a dataset \u4e2d\u6302\u8f7d\u70b9\u7684 name\uff0c\u53ef\u4ee5\u4f7f\u7528 /nasData/xxx \u6765\u9884\u70ed\u5176\u4e2d\u7684\u5b50\u76ee\u5f55\u3002 replicas \u8868\u793a\u7f13\u5b58\u7684\u5907\u4efd\u6570\uff0c\u9ed8\u8ba4\u662f1\u3002 \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload test-dataload NAME DATASET PHASE AGE dataload nas Completed 2m13s 6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: nas \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u8bbf\u95ee\u6302\u8f7d\u70b9\u5bf9\u5e94\u7684\u5e95\u5c42\u6587\u4ef6 $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 7\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime nas \u5220\u9664 dataset kubectl delete dataset nas","title":"Jindo fluid nas ufs example"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_nas_ufs_example/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_nas_ufs_example/#2","text":"\u70b9\u51fb\u4e0b\u8f7d","title":"2\u3001\u4e0b\u8f7d\u5b89\u88c5\u5305"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_nas_ufs_example/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-0.7.0.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_nas_ufs_example/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_nas_ufs_example/#5","text":"","title":"5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_nas_ufs_example/#51-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: nas spec: mounts: - mountPoint: local:///mnt/nas name: nasData --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: nas spec: replicas: 2 tieredstore: levels: - mediumtype: MEM path: /dev/shm quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d NAS \u7684\u8def\u5f84\uff0c local:// \u4e3a\u672c\u5730\u8def\u5f84\u524d\u7f00\uff0c /mnt/nas \u8868\u793a\u672c\u5730\u771f\u5b9e\u8def\u5f84\uff0c\u8be5\u8def\u5f84\u9700\u8981\u5728\u5bbf\u4e3b\u673a\u4e0a\u53ef\u89c1\u3002 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JindoRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime nas NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE nas Ready Ready 62m \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/nas 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/nas Bound hadoop 100Gi RWX 58m \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 NAS \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: test spec: dataset: name: nas namespace: default target: - path: /nasData replicas: 1 \u5176\u4e2d /nasData \u4e2d nasData \u4e3a dataset \u4e2d\u6302\u8f7d\u70b9\u7684 name\uff0c\u53ef\u4ee5\u4f7f\u7528 /nasData/xxx \u6765\u9884\u70ed\u5176\u4e2d\u7684\u5b50\u76ee\u5f55\u3002 replicas \u8868\u793a\u7f13\u5b58\u7684\u5907\u4efd\u6570\uff0c\u9ed8\u8ba4\u662f1\u3002 \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload test-dataload NAME DATASET PHASE AGE dataload nas Completed 2m13s","title":"5.1 \u521b\u5efa dataset \u548c JindoRuntime"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_nas_ufs_example/#6","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: nas \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u8bbf\u95ee\u6302\u8f7d\u70b9\u5bf9\u5e94\u7684\u5e95\u5c42\u6587\u4ef6 $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","title":"6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_nas_ufs_example/#7","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime nas \u5220\u9664 dataset kubectl delete dataset nas","title":"7\u3001\u73af\u5883\u6e05\u7406"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_oss_append_fuseonly/","text":"\u52a0\u901fOSS\u4e0a\u6570\u636e \u00a7 \u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002 1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001 \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s 5\u3001\u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop path: / accessModes: - ReadWriteMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: master: disabled: true worker: disabled: true mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aoss bucket\u7684endpoint\u4fe1\u606f\uff0c\u516c\u7f51\u6216\u5185\u7f51\u5730\u5740\u7686\u53ef\uff0c\u5982\u60a8\u7684 bucket \u5728\u676d\u5dde Region\uff0c\u90a3\u4e48\u516c\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou.aliyuncs.com\uff0c\u5185\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou-internal.aliyuncs.com(\u5185\u7f51\u5730\u5740\u4f7f\u7528\u6761\u4ef6\u4e3a\u60a8\u7684 K8S \u96c6\u7fa4\u6240\u5728\u533a\u57df\u548coss\u533a\u57df\u76f8\u540c)\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m 6\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_oss_append_fuseonly/#oss","text":"\u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_oss_append_fuseonly/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_oss_append_fuseonly/#2-fluid","text":"","title":"2\u3001\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_oss_append_fuseonly/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_oss_append_fuseonly/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_oss_append_fuseonly/#5-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop path: / accessModes: - ReadWriteMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: master: disabled: true worker: disabled: true mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aoss bucket\u7684endpoint\u4fe1\u606f\uff0c\u516c\u7f51\u6216\u5185\u7f51\u5730\u5740\u7686\u53ef\uff0c\u5982\u60a8\u7684 bucket \u5728\u676d\u5dde Region\uff0c\u90a3\u4e48\u516c\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou.aliyuncs.com\uff0c\u5185\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou-internal.aliyuncs.com(\u5185\u7f51\u5730\u5740\u4f7f\u7528\u6761\u4ef6\u4e3a\u60a8\u7684 K8S \u96c6\u7fa4\u6240\u5728\u533a\u57df\u548coss\u533a\u57df\u76f8\u540c)\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m","title":"5\u3001\u521b\u5efa dataset \u548c JindoRuntime"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_oss_append_fuseonly/#6","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"6\u3001\u73af\u5883\u6e05\u7406"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_sdk_connect_example/","text":"\u4f7f\u7528 JindoSDK \u8bbf\u95ee Fluid JindoRuntime \u96c6\u7fa4 \u00a7 \u6b65\u9aa4 \u00a7 \u524d\u63d0\uff1a\u542f\u52a8 Fluid JindoRuntime \u96c6\u7fa4 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e core-site \u00a7 \u767b\u9646\u5230 xxx-jindofs-master-0 \u5bb9\u5668\u91cc kubectl exec -ti xxx-jindofs-master-0 bash \u627e\u5230 /hdfs-3.2.1/etc/hadoop/core-site.xml \u6587\u4ef6\uff0c\u5c06\u6587\u4ef6\u5185\u5bb9\u5168\u90e8\u52a0\u5165\u5230 hadoop \u7684 classpath \u4e0b\u7684 core-site \u6587\u4ef6\u4e2d 4. shell\u547d\u4ee4 \u00a7 \u53ef\u4ee5\u6267\u884c hadoop shell \u547d\u4ee4\u6765\u5224\u65ad\u670d\u52a1\u662f\u5426\u8fde\u63a5\u6210\u529f hadoop fs -ls oss://<Bucket>/<Dir> 6. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoFSx \u53c2\u6570\u8bf4\u660e JindoFSx \u5305\u542b\u4e00\u4e9b\u7f13\u5b58\u76f8\u5173\u7684\u53c2\u6570\uff0c\u8bf7\u53c2\u8003\u6587\u6863 JindoFSx \u5ba2\u6237\u7aef\u914d\u7f6e\u5217\u8868 * JindoSDK \u53c2\u6570\u8bf4\u660e JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 7. \u6743\u9650\u7ba1\u7406 \u00a7 \u81ea\u5efa\u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u963f\u91cc\u4e91 E-MapReduce \u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848","title":"\u4f7f\u7528 JindoSDK \u8bbf\u95ee Fluid JindoRuntime \u96c6\u7fa4"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_sdk_connect_example/#jindosdk-fluid-jindoruntime","text":"","title":"\u4f7f\u7528 JindoSDK \u8bbf\u95ee Fluid JindoRuntime \u96c6\u7fa4"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_sdk_connect_example/#_1","text":"\u524d\u63d0\uff1a\u542f\u52a8 Fluid JindoRuntime \u96c6\u7fa4","title":"\u6b65\u9aa4"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_sdk_connect_example/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_sdk_connect_example/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_sdk_connect_example/#3-core-site","text":"\u767b\u9646\u5230 xxx-jindofs-master-0 \u5bb9\u5668\u91cc kubectl exec -ti xxx-jindofs-master-0 bash \u627e\u5230 /hdfs-3.2.1/etc/hadoop/core-site.xml \u6587\u4ef6\uff0c\u5c06\u6587\u4ef6\u5185\u5bb9\u5168\u90e8\u52a0\u5165\u5230 hadoop \u7684 classpath \u4e0b\u7684 core-site \u6587\u4ef6\u4e2d","title":"3. \u914d\u7f6e core-site"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_sdk_connect_example/#4-shell","text":"\u53ef\u4ee5\u6267\u884c hadoop shell \u547d\u4ee4\u6765\u5224\u65ad\u670d\u52a1\u662f\u5426\u8fde\u63a5\u6210\u529f hadoop fs -ls oss://<Bucket>/<Dir>","title":"4. shell\u547d\u4ee4"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_sdk_connect_example/#6","text":"JindoFSx \u53c2\u6570\u8bf4\u660e JindoFSx \u5305\u542b\u4e00\u4e9b\u7f13\u5b58\u76f8\u5173\u7684\u53c2\u6570\uff0c\u8bf7\u53c2\u8003\u6587\u6863 JindoFSx \u5ba2\u6237\u7aef\u914d\u7f6e\u5217\u8868 * JindoSDK \u53c2\u6570\u8bf4\u660e JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"6. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_sdk_connect_example/#7","text":"\u81ea\u5efa\u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u963f\u91cc\u4e91 E-MapReduce \u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848","title":"7. \u6743\u9650\u7ba1\u7406"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_update_version_example/","text":"Fluid JindoRuntime \u955c\u50cf\u5347\u7ea7 \u00a7 1\u3001\u4fee\u6539 jindoruntime-controller \u7684\u4fe1\u606f kubectl edit deployment jindoruntime-controller -n fluid-system 2\u3001\u7f16\u8f91 controller\uff0c\u627e\u5230\u5982\u4e0b\u5185\u5bb9 - name: JINDO_SMARTDATA_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:4.5.2 - name: JINDO_FUSE_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:4.5.2 3\u3001\u4f8b\u5982\u4ece 4.5.1 \u5347\u7ea7\u5230 4.5.2\uff0c\u4fee\u6539\u5982\u4e0b\u955c\u50cf\u5373\u53ef - name: JINDO_SMARTDATA_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:4.5.2 - name: JINDO_FUSE_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:4.5.2 \u5982\u679c\u662f fuseOnly \u6a21\u5f0f\uff0c\u90a3\u4e48\u53ea\u9700\u4fee\u6539 JINDO_FUSE_IMAGE_ENV \u5b57\u6bb5\u5373\u53ef 4\u3001\u4fdd\u5b58\u4fee\u6539\uff0c\u91cd\u65b0\u521b\u5efa\u96c6\u7fa4\u5373\u53ef","title":"Fluid JindoRuntime \u955c\u50cf\u5347\u7ea7"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_update_version_example/#fluid-jindoruntime","text":"1\u3001\u4fee\u6539 jindoruntime-controller \u7684\u4fe1\u606f kubectl edit deployment jindoruntime-controller -n fluid-system 2\u3001\u7f16\u8f91 controller\uff0c\u627e\u5230\u5982\u4e0b\u5185\u5bb9 - name: JINDO_SMARTDATA_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:4.5.2 - name: JINDO_FUSE_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:4.5.2 3\u3001\u4f8b\u5982\u4ece 4.5.1 \u5347\u7ea7\u5230 4.5.2\uff0c\u4fee\u6539\u5982\u4e0b\u955c\u50cf\u5373\u53ef - name: JINDO_SMARTDATA_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:4.5.2 - name: JINDO_FUSE_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:4.5.2 \u5982\u679c\u662f fuseOnly \u6a21\u5f0f\uff0c\u90a3\u4e48\u53ea\u9700\u4fee\u6539 JINDO_FUSE_IMAGE_ENV \u5b57\u6bb5\u5373\u53ef 4\u3001\u4fdd\u5b58\u4fee\u6539\uff0c\u91cd\u65b0\u521b\u5efa\u96c6\u7fa4\u5373\u53ef","title":"Fluid JindoRuntime \u955c\u50cf\u5347\u7ea7"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_util_intro/","text":"jindo-util \u4f7f\u7528\u65b9\u6cd5 \u00a7 \u4e0b\u8f7d jindo-util \u5de5\u5177\uff08\u5f53\u524d\u53ea\u652f\u6301 Linux \u73af\u5883\u4f7f\u7528\uff09 \u00a7 1\u3001\u4e0b\u8f7d jindofsx-4.5.2.tar.gz 2\u3001\u89e3\u538b tar \u5305 tar -zxvf jindofsx-4.5.2.tar.gz 3\u3001\u5728 /jindofsx-4.5.2/bin/ \u76ee\u5f55\u4e0b\u627e\u5230 jindo-util \u4e8c\u8fdb\u5236\u6587\u4ef6 chmod 700 jindo-util mv jindo-util jindo 4\u3001\u67e5\u770b jindo-util \u6240\u6709\u652f\u6301\u7684\u547d\u4ee4\u548c\u4f7f\u7528\u65b9\u6cd5 # jindo fs -help Usage: jindo fs [-appendToFile <localsrc> ... <dst>] [-archive [-i] [-c] <path>] [-cat <src> ...] [-count [-p] [-r] <path> ...] [-cp [-f] <src> ... <dst>] [-checksum <src> ...] [-chgrp [-R] GROUP PATH...] [-chmod [-R] <MODE[,MODE]... | OCTALMODE> PATH...] [-chown [-R] [OWNER][:[GROUP]] PATH...] [-copyFromLocal [-f] <localsrc> ... <dst>] [-copyToLocal [-f] <src> ... <localdst>] [-createSnapshot <snapshotDir> [<snapshotName>]] [-checkStoragePolicy -path <path>] [-deleteSnapshot <snapshotDir> <snapshotName>] [-du [-s] [-p] [-r] <path> ...] [-decommission -hosts <hosts>] [-formatCache] [-get [-f] <src> ... <localdst>] [-getfacl [-R] <path>] [-getfattr [-R] {-n name | -d} <path>] [-getStoragePolicy -path <path>] [-ls [-R] <path>] [-listPolicies] [-load [-meta] [-data] [-s] [-m] [-R] [-replica value] <path>] [-mkdir <path>] [-mv <src> ... <dst>] [-moveFromLocal <localsrc> ... <dst>] [-moveToLocal <src> <localdst>] [-metaDiff <path>] [-put [-f] <localsrc> ... <dst>] [-rm [-f] [-r|-R] <src> ...] [-rmdir <src> ...] [-renameSnapshot <snapshotDir> <oldName> <newName>] [-restore [-days value] <path>] [-stat [format] <path> ...] [-setStoragePolicy -path <path> -policy <policy>] [-setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>]] [-setfattr {-n name [-v value] | -x name} <path>] [-touchz <path> ...] [-truncate [-w] <length> <path> ...] [-unarchive [-i] [-a] <path>] [-uncache <path>] [-unsetStoragePolicy -path <path>] jindo admin [-mount <mountpoint> <path>] [-report] [-reportMetrics] [-umount <mountpoint>] jindo admin [-addProxyUser -dlsUri <path> -proxyUser <value> -users|-groups <value> -hosts <value>] [-allowSnapshot -dlsUri <path>] [-addUserGroupsMapping -dlsUri <path> -user <value> -groups <value>] [-disallowSnapshot -dlsUri <path>] [-deleteProxyUser -dlsUri <path> -proxyUser <value>] [-deleteUserGroupsMapping -dlsUri <path> -user <value>] [-dumpFile -dlsUri <path>] [-dumpBlockFiles -in <path> -out <path>] [-listProxyUsers -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-listUserGroupsMappings -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-snapshotDiff -dlsUri <path> -fromSnapshot <value> -toSnapshot <value>] jindo benchmark [-io -read -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-memCheck]] [-io -pread -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-seekTimes <value>] [-memCheck]] [-io -write -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-memCheck]] [-meta -mkdir -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-memCheck]] [-meta -listDirectory -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]] [-meta -getFileStatus -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]] [-meta -renameFile -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]] \u4f7f\u7528 jindo-util \u8bbf\u95ee\u6807\u51c6 OSS \u00a7 1\u3001\u65b0\u5efa\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg [common] logger.dir = /tmp/jindo-util/ logger.sync = false logger.consolelogger = false logger.level = 0 logger.verbose = 0 logger.cleaner.enable = true hadoopConf.enable = false [jindosdk] fs.oss.endpoint = <OSS_ENDPOINT> fs.oss.accessKeyId = <OSS_ACCESSKEY_ID> fs.oss.accessKeySecret = <OSS_ACCESSKEY_SECRET> : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u7684 endpoint : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u7684 AccessKeyId : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u7684 AccessKeySecret 2\u3001\u6dfb\u52a0\u73af\u5883\u53d8\u91cf export JINDOSDK_CONF_DIR=<JINDOSDK_CFG_DIR> : jindosdk.cfg \u914d\u7f6e\u6587\u4ef6\u6240\u5728\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u6bd4\u5982 /etc/ 3\u3001\u4f7f\u7528 jindo-util \u8bbf\u95ee OSS jindo fs -ls oss://<bucket>/<dir> \u4f7f\u7528 jindo-util \u8bbf\u95ee OSS-HDFS \u00a7 1\u3001\u65b0\u5efa\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg [common] logger.dir = /tmp/jindo-util/ logger.sync = false logger.consolelogger = false logger.level = 0 logger.verbose = 0 logger.cleaner.enable = true hadoopConf.enable = false [jindosdk] fs.oss.bucket.<OSS_HDFS_BUCKET>.accessKeyId = <OSS_HDFS_ACCESSKEY_ID> fs.oss.bucket.<OSS_HDFS_BUCKET>.accessKeySecret = <OSS_HDFS_ACCESSKEY_SECRET> fs.oss.bucket.<OSS_HDFS_BUCKET>.endpoint = <OSS_HDFS_ENDPOINT> fs.oss.bucket.<OSS_HDFS_BUCKET>.data.lake.storage.enable = true : \u9700\u8981\u8bbf\u95ee\u7684 OSS-HDFS \u7684 bucket : \u9700\u8981\u8bbf\u95ee\u7684 OSS-HDFS \u7684 endpoint : \u9700\u8981\u8bbf\u95ee\u7684 OSS-HDFS \u7684 AccessKeyId : \u9700\u8981\u8bbf\u95ee\u7684 OSS-HDFS \u7684 AccessKeySecret 2\u3001\u6dfb\u52a0\u73af\u5883\u53d8\u91cf export JINDOSDK_CONF_DIR=<JINDOSDK_CFG_DIR> : jindosdk.cfg \u914d\u7f6e\u6587\u4ef6\u6240\u5728\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u6bd4\u5982 /etc/ 3\u3001\u4f7f\u7528 jindo-util \u8bbf\u95ee OSS-HDFS \u670d\u52a1 jindo fs -ls oss://<bucket>/<dir>","title":"jindo-util \u4f7f\u7528\u65b9\u6cd5"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_util_intro/#jindo-util","text":"","title":"jindo-util \u4f7f\u7528\u65b9\u6cd5"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_util_intro/#jindo-util-linux","text":"1\u3001\u4e0b\u8f7d jindofsx-4.5.2.tar.gz 2\u3001\u89e3\u538b tar \u5305 tar -zxvf jindofsx-4.5.2.tar.gz 3\u3001\u5728 /jindofsx-4.5.2/bin/ \u76ee\u5f55\u4e0b\u627e\u5230 jindo-util \u4e8c\u8fdb\u5236\u6587\u4ef6 chmod 700 jindo-util mv jindo-util jindo 4\u3001\u67e5\u770b jindo-util \u6240\u6709\u652f\u6301\u7684\u547d\u4ee4\u548c\u4f7f\u7528\u65b9\u6cd5 # jindo fs -help Usage: jindo fs [-appendToFile <localsrc> ... <dst>] [-archive [-i] [-c] <path>] [-cat <src> ...] [-count [-p] [-r] <path> ...] [-cp [-f] <src> ... <dst>] [-checksum <src> ...] [-chgrp [-R] GROUP PATH...] [-chmod [-R] <MODE[,MODE]... | OCTALMODE> PATH...] [-chown [-R] [OWNER][:[GROUP]] PATH...] [-copyFromLocal [-f] <localsrc> ... <dst>] [-copyToLocal [-f] <src> ... <localdst>] [-createSnapshot <snapshotDir> [<snapshotName>]] [-checkStoragePolicy -path <path>] [-deleteSnapshot <snapshotDir> <snapshotName>] [-du [-s] [-p] [-r] <path> ...] [-decommission -hosts <hosts>] [-formatCache] [-get [-f] <src> ... <localdst>] [-getfacl [-R] <path>] [-getfattr [-R] {-n name | -d} <path>] [-getStoragePolicy -path <path>] [-ls [-R] <path>] [-listPolicies] [-load [-meta] [-data] [-s] [-m] [-R] [-replica value] <path>] [-mkdir <path>] [-mv <src> ... <dst>] [-moveFromLocal <localsrc> ... <dst>] [-moveToLocal <src> <localdst>] [-metaDiff <path>] [-put [-f] <localsrc> ... <dst>] [-rm [-f] [-r|-R] <src> ...] [-rmdir <src> ...] [-renameSnapshot <snapshotDir> <oldName> <newName>] [-restore [-days value] <path>] [-stat [format] <path> ...] [-setStoragePolicy -path <path> -policy <policy>] [-setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>]] [-setfattr {-n name [-v value] | -x name} <path>] [-touchz <path> ...] [-truncate [-w] <length> <path> ...] [-unarchive [-i] [-a] <path>] [-uncache <path>] [-unsetStoragePolicy -path <path>] jindo admin [-mount <mountpoint> <path>] [-report] [-reportMetrics] [-umount <mountpoint>] jindo admin [-addProxyUser -dlsUri <path> -proxyUser <value> -users|-groups <value> -hosts <value>] [-allowSnapshot -dlsUri <path>] [-addUserGroupsMapping -dlsUri <path> -user <value> -groups <value>] [-disallowSnapshot -dlsUri <path>] [-deleteProxyUser -dlsUri <path> -proxyUser <value>] [-deleteUserGroupsMapping -dlsUri <path> -user <value>] [-dumpFile -dlsUri <path>] [-dumpBlockFiles -in <path> -out <path>] [-listProxyUsers -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-listUserGroupsMappings -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-snapshotDiff -dlsUri <path> -fromSnapshot <value> -toSnapshot <value>] jindo benchmark [-io -read -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-memCheck]] [-io -pread -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-seekTimes <value>] [-memCheck]] [-io -write -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-memCheck]] [-meta -mkdir -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-memCheck]] [-meta -listDirectory -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]] [-meta -getFileStatus -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]] [-meta -renameFile -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]]","title":"\u4e0b\u8f7d jindo-util \u5de5\u5177\uff08\u5f53\u524d\u53ea\u652f\u6301 Linux \u73af\u5883\u4f7f\u7528\uff09"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_util_intro/#jindo-util-oss","text":"1\u3001\u65b0\u5efa\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg [common] logger.dir = /tmp/jindo-util/ logger.sync = false logger.consolelogger = false logger.level = 0 logger.verbose = 0 logger.cleaner.enable = true hadoopConf.enable = false [jindosdk] fs.oss.endpoint = <OSS_ENDPOINT> fs.oss.accessKeyId = <OSS_ACCESSKEY_ID> fs.oss.accessKeySecret = <OSS_ACCESSKEY_SECRET> : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u7684 endpoint : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u7684 AccessKeyId : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u7684 AccessKeySecret 2\u3001\u6dfb\u52a0\u73af\u5883\u53d8\u91cf export JINDOSDK_CONF_DIR=<JINDOSDK_CFG_DIR> : jindosdk.cfg \u914d\u7f6e\u6587\u4ef6\u6240\u5728\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u6bd4\u5982 /etc/ 3\u3001\u4f7f\u7528 jindo-util \u8bbf\u95ee OSS jindo fs -ls oss://<bucket>/<dir>","title":"\u4f7f\u7528 jindo-util \u8bbf\u95ee\u6807\u51c6 OSS"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_util_intro/#jindo-util-oss-hdfs","text":"1\u3001\u65b0\u5efa\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg [common] logger.dir = /tmp/jindo-util/ logger.sync = false logger.consolelogger = false logger.level = 0 logger.verbose = 0 logger.cleaner.enable = true hadoopConf.enable = false [jindosdk] fs.oss.bucket.<OSS_HDFS_BUCKET>.accessKeyId = <OSS_HDFS_ACCESSKEY_ID> fs.oss.bucket.<OSS_HDFS_BUCKET>.accessKeySecret = <OSS_HDFS_ACCESSKEY_SECRET> fs.oss.bucket.<OSS_HDFS_BUCKET>.endpoint = <OSS_HDFS_ENDPOINT> fs.oss.bucket.<OSS_HDFS_BUCKET>.data.lake.storage.enable = true : \u9700\u8981\u8bbf\u95ee\u7684 OSS-HDFS \u7684 bucket : \u9700\u8981\u8bbf\u95ee\u7684 OSS-HDFS \u7684 endpoint : \u9700\u8981\u8bbf\u95ee\u7684 OSS-HDFS \u7684 AccessKeyId : \u9700\u8981\u8bbf\u95ee\u7684 OSS-HDFS \u7684 AccessKeySecret 2\u3001\u6dfb\u52a0\u73af\u5883\u53d8\u91cf export JINDOSDK_CONF_DIR=<JINDOSDK_CFG_DIR> : jindosdk.cfg \u914d\u7f6e\u6587\u4ef6\u6240\u5728\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u6bd4\u5982 /etc/ 3\u3001\u4f7f\u7528 jindo-util \u8bbf\u95ee OSS-HDFS \u670d\u52a1 jindo fs -ls oss://<bucket>/<dir>","title":"\u4f7f\u7528 jindo-util \u8bbf\u95ee OSS-HDFS"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/","text":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002JindoFuse \u4f7f\u7528\u539f\u6765\u7684\u5730\u5740 (oss://) \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c\u8be5\u670d\u52a1\u4e0a\u7684\u6587\u4ef6\uff0c\u540c\u65f6\u901a\u8fc7\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002 \u57fa\u672c\u4f7f\u7528 \u00a7 \u73af\u5883\u51c6\u5907 \u00a7 # CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u914d\u7f6e\u670d\u52a1\u7aef \u00a7 \u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-filestore] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = cn-xxx.oss-dls.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e\u5ba2\u6237\u7aef \u00a7 \u914d\u7f6e\u76ee\u5f55 \u00a7 \u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf \u914d\u7f6e\u6587\u4ef6 \u00a7 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u914d\u7f6e\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint= <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1\u7684 AccessKey ID \u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7 AccessKey \u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> # \u5982\u679c\u6709\u591a\u4e2a bucket \u5bf9\u5e94\u4e0d\u540c\u7684AccessKey ID \u548c AccessKey Secret\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u914d\u7f6e\u65b9\u5f0f fs.oss.bucket.<your-bucket>.accessKeyId = <your_key_id> fs.oss.bucket.<your-bucket>.accessKeySecret = <your_key_secret> # \u914d\u7f6eNamespace service \u5730\u5740 fs.jindofsx.namespace.rpc.address = <hostname>:<port> # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable = false \u6302\u8f7d JindoFuse \u00a7 \u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002 \u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 mkdir -p <mount_point> \u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 jindo-fuse <mount_point> -ouri=[<oss_path>] -oxengine=jindofsx -ouri \u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684 oss \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a Bucket \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002 \u8bbf\u95ee JindoFuse \u00a7 \u5982\u679c\u5c06 OSS-HDFS \u670d\u52a1\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1 ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/ \u5378\u8f7d JindoFuse \u00a7 \u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point> \u81ea\u52a8\u5378\u8f7d JindoFuse \u00a7 \u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u9ad8\u9636\u4f7f\u7528 \u00a7 \u914d\u7f6e\u9009\u9879 \u00a7 \u66f4\u591a jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<oss_path>] -ofs.jindofsx.data.cache.enable=false","title":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#jindofuse-oss-hdfs-jindofs","text":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002JindoFuse \u4f7f\u7528\u539f\u6765\u7684\u5730\u5740 (oss://) \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c\u8be5\u670d\u52a1\u4e0a\u7684\u6587\u4ef6\uff0c\u540c\u65f6\u901a\u8fc7\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002","title":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#_1","text":"","title":"\u57fa\u672c\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#_2","text":"# CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev","title":"\u73af\u5883\u51c6\u5907"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#_3","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#_4","text":"\u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-filestore] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = cn-xxx.oss-dls.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002","title":"\u914d\u7f6e\u670d\u52a1\u7aef"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#jindofsx","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#_5","text":"","title":"\u914d\u7f6e\u5ba2\u6237\u7aef"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#_6","text":"\u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf","title":"\u914d\u7f6e\u76ee\u5f55"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#_7","text":"\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u914d\u7f6e\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint= <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1\u7684 AccessKey ID \u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7 AccessKey \u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> # \u5982\u679c\u6709\u591a\u4e2a bucket \u5bf9\u5e94\u4e0d\u540c\u7684AccessKey ID \u548c AccessKey Secret\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u914d\u7f6e\u65b9\u5f0f fs.oss.bucket.<your-bucket>.accessKeyId = <your_key_id> fs.oss.bucket.<your-bucket>.accessKeySecret = <your_key_secret> # \u914d\u7f6eNamespace service \u5730\u5740 fs.jindofsx.namespace.rpc.address = <hostname>:<port> # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable = false","title":"\u914d\u7f6e\u6587\u4ef6"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#jindofuse","text":"\u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002","title":"\u6302\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#_8","text":"mkdir -p <mount_point>","title":"\u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#fuse","text":"jindo-fuse <mount_point> -ouri=[<oss_path>] -oxengine=jindofsx -ouri \u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684 oss \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a Bucket \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002","title":"\u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#jindofuse_1","text":"\u5982\u679c\u5c06 OSS-HDFS \u670d\u52a1\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1 ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/","title":"\u8bbf\u95ee JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#jindofuse_2","text":"\u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point>","title":"\u5378\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#jindofuse_3","text":"\u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002","title":"\u81ea\u52a8\u5378\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#_9","text":"","title":"\u9ad8\u9636\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#_10","text":"\u66f4\u591a jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<oss_path>] -ofs.jindofsx.data.cache.enable=false","title":"\u914d\u7f6e\u9009\u9879"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/","text":"JindoFuse \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f \u00a7 \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002JindoFuse \u4f7f\u7528\u7edf\u4e00\u6302\u8f7d\u7684\u65b9\u5f0f (jindo://) \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c\u8be5\u670d\u52a1\u4e0a\u7684\u6587\u4ef6\uff0c\u540c\u65f6\u901a\u8fc7\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002 \u57fa\u672c\u4f7f\u7528 \u00a7 \u73af\u5883\u51c6\u5907 \u00a7 # CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u914d\u7f6e\u670d\u52a1\u7aef \u00a7 \u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-filestore] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = cn-xxx.oss-dls.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e\u5ba2\u6237\u7aef \u00a7 \u914d\u7f6e\u76ee\u5f55 \u00a7 \u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf \u914d\u7f6e\u6587\u4ef6 \u00a7 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u914d\u7f6e\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 # \u6216\u914d\u7f6e\u963f\u91cc\u4e91 OSS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou-internal.aliyuncs.com\u3002 fs.oss.endpoint= <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1\u7684 AccessKey ID \u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7 AccessKey \u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> # \u914d\u7f6eNamespace service \u5730\u5740 fs.jindofsx.namespace.rpc.address = <hostname>:<port> # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable = false \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55 \u00a7 \u6302\u8f7d\u547d\u4ee4 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>/ \u6302\u8f7d JindoFuse \u00a7 \u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002 \u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 mkdir -p <mount_point> \u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 jindo-fuse <mount_point> -ouri=[<jindo_path>] -oxengine=jindofsx -ouri \u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684 jindo \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a\u7edf\u4e00\u6302\u8f7d\uff0c\u6bd4\u5982\u4e4b\u524d\u6302\u8f7d\u7684 jindo:///jindooss \u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002 \u8bbf\u95ee JindoFuse \u00a7 \u5982\u679c\u5c06 jindo \u8def\u5f84\uff08jindo://\uff09\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/jindo/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/jindo/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/jindo/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/jindo/dir1 ls /mnt/jindo/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/jindo/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/jindo/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/jindo/dir1/ \u5378\u8f7d JindoFuse \u00a7 \u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point> \u81ea\u52a8\u5378\u8f7d JindoFuse \u00a7 \u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u9ad8\u9636\u4f7f\u7528 \u00a7 \u914d\u7f6e\u9009\u9879 \u00a7 \u66f4\u591a jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<jindo_path>] -ofs.jindofsx.data.cache.enable=false","title":"JindoFuse \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#jindofuse","text":"\u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002JindoFuse \u4f7f\u7528\u7edf\u4e00\u6302\u8f7d\u7684\u65b9\u5f0f (jindo://) \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c\u8be5\u670d\u52a1\u4e0a\u7684\u6587\u4ef6\uff0c\u540c\u65f6\u901a\u8fc7\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002","title":"JindoFuse \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#_1","text":"","title":"\u57fa\u672c\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#_2","text":"# CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev","title":"\u73af\u5883\u51c6\u5907"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#_3","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#_4","text":"\u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-filestore] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = cn-xxx.oss-dls.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002","title":"\u914d\u7f6e\u670d\u52a1\u7aef"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#jindofsx","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#_5","text":"","title":"\u914d\u7f6e\u5ba2\u6237\u7aef"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#_6","text":"\u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf","title":"\u914d\u7f6e\u76ee\u5f55"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#_7","text":"\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u914d\u7f6e\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 # \u6216\u914d\u7f6e\u963f\u91cc\u4e91 OSS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou-internal.aliyuncs.com\u3002 fs.oss.endpoint= <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1\u7684 AccessKey ID \u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7 AccessKey \u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> # \u914d\u7f6eNamespace service \u5730\u5740 fs.jindofsx.namespace.rpc.address = <hostname>:<port> # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable = false","title":"\u914d\u7f6e\u6587\u4ef6"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#oss-oss-hdfs","text":"\u6302\u8f7d\u547d\u4ee4 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>/","title":"\u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#jindofuse_1","text":"\u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002","title":"\u6302\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#_8","text":"mkdir -p <mount_point>","title":"\u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#fuse","text":"jindo-fuse <mount_point> -ouri=[<jindo_path>] -oxengine=jindofsx -ouri \u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684 jindo \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a\u7edf\u4e00\u6302\u8f7d\uff0c\u6bd4\u5982\u4e4b\u524d\u6302\u8f7d\u7684 jindo:///jindooss \u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002","title":"\u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#jindofuse_2","text":"\u5982\u679c\u5c06 jindo \u8def\u5f84\uff08jindo://\uff09\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/jindo/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/jindo/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/jindo/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/jindo/dir1 ls /mnt/jindo/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/jindo/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/jindo/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/jindo/dir1/","title":"\u8bbf\u95ee JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#jindofuse_3","text":"\u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point>","title":"\u5378\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#jindofuse_4","text":"\u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002","title":"\u81ea\u52a8\u5378\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#_9","text":"","title":"\u9ad8\u9636\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#_10","text":"\u66f4\u591a jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<jindo_path>] -ofs.jindofsx.data.cache.enable=false","title":"\u914d\u7f6e\u9009\u9879"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/","text":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f NAS \u00a7 JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002JindoFuse \u4f7f\u7528 NAS \u7684\u5730\u5740 (local://) \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c\u8be5\u670d\u52a1\u4e0a\u7684\u6587\u4ef6\uff0c\u540c\u65f6\u901a\u8fc7\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002 \u57fa\u672c\u4f7f\u7528 \u00a7 \u73af\u5883\u51c6\u5907 \u00a7 # CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u96c6\u7fa4\u5df2\u6302\u8f7d NAS \u6587\u4ef6\u7cfb\u7edf \u6302\u8f7d\u6b65\u9aa4\u8bf7\u53c2\u8003 Linux\u7cfb\u7edf\u6302\u8f7dNFS\u6587\u4ef6\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u914d\u7f6e\u5ba2\u6237\u7aef \u00a7 \u914d\u7f6e\u76ee\u5f55 \u00a7 \u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf \u914d\u7f6e\u6587\u4ef6 \u00a7 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u914d\u7f6eNamespace service \u5730\u5740 fs.jindofsx.namespace.rpc.address = <hostname>:<port> # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable = false \u6302\u8f7d JindoFuse \u00a7 \u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002 \u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 mkdir -p <mount_point> \u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 jindo-fuse <mount_point> -ouri=[<nas_path>] -oxengine=jindofsx -ouri \u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684 nas \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a nas \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\uff0c\u6bd4\u5982 local:///tmp/nas/ \u3002\u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002 \u8bbf\u95ee JindoFuse \u00a7 \u5982\u679c\u5c06 nas \u8def\u5f84\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/nas/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/nas/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/nas/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/nas/dir1 ls /mnt/nas/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/nas/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/nas/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/nas/dir1/ \u5378\u8f7d JindoFuse \u00a7 \u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point> \u81ea\u52a8\u5378\u8f7d JindoFuse \u00a7 \u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u9ad8\u9636\u4f7f\u7528 \u00a7 \u914d\u7f6e\u9009\u9879 \u00a7 \u66f4\u591a jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<nas_path>] -ofs.jindofsx.data.cache.enable=false","title":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f NAS"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#jindofuse-nas","text":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002JindoFuse \u4f7f\u7528 NAS \u7684\u5730\u5740 (local://) \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c\u8be5\u670d\u52a1\u4e0a\u7684\u6587\u4ef6\uff0c\u540c\u65f6\u901a\u8fc7\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002","title":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f NAS"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#_1","text":"","title":"\u57fa\u672c\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#_2","text":"# CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev","title":"\u73af\u5883\u51c6\u5907"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#_3","text":"\u96c6\u7fa4\u5df2\u6302\u8f7d NAS \u6587\u4ef6\u7cfb\u7edf \u6302\u8f7d\u6b65\u9aa4\u8bf7\u53c2\u8003 Linux\u7cfb\u7edf\u6302\u8f7dNFS\u6587\u4ef6\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#_4","text":"","title":"\u914d\u7f6e\u5ba2\u6237\u7aef"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#_5","text":"\u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf","title":"\u914d\u7f6e\u76ee\u5f55"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#_6","text":"\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u914d\u7f6eNamespace service \u5730\u5740 fs.jindofsx.namespace.rpc.address = <hostname>:<port> # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable = false","title":"\u914d\u7f6e\u6587\u4ef6"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#jindofuse","text":"\u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002","title":"\u6302\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#_7","text":"mkdir -p <mount_point>","title":"\u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#fuse","text":"jindo-fuse <mount_point> -ouri=[<nas_path>] -oxengine=jindofsx -ouri \u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684 nas \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a nas \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\uff0c\u6bd4\u5982 local:///tmp/nas/ \u3002\u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002","title":"\u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#jindofuse_1","text":"\u5982\u679c\u5c06 nas \u8def\u5f84\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/nas/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/nas/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/nas/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/nas/dir1 ls /mnt/nas/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/nas/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/nas/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/nas/dir1/","title":"\u8bbf\u95ee JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#jindofuse_2","text":"\u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point>","title":"\u5378\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#jindofuse_3","text":"\u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002","title":"\u81ea\u52a8\u5378\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#_8","text":"","title":"\u9ad8\u9636\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#_9","text":"\u66f4\u591a jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<nas_path>] -ofs.jindofsx.data.cache.enable=false","title":"\u914d\u7f6e\u9009\u9879"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/","text":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f OSS \u00a7 JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002JindoFuse \u4f7f\u7528\u539f\u6765\u7684\u5730\u5740 (oss://) \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c\u8be5\u670d\u52a1\u4e0a\u7684\u6587\u4ef6\uff0c\u540c\u65f6\u901a\u8fc7\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002 \u57fa\u672c\u4f7f\u7528 \u00a7 \u73af\u5883\u51c6\u5907 \u00a7 # CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u914d\u7f6e\u670d\u52a1\u7aef \u00a7 \u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-filestore] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = cn-xxx.oss-dls.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e\u5ba2\u6237\u7aef \u00a7 \u914d\u7f6e\u76ee\u5f55 \u00a7 \u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf \u914d\u7f6e\u6587\u4ef6 \u00a7 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u6216\u914d\u7f6e\u963f\u91cc\u4e91 OSS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou-internal.aliyuncs.com\u3002 fs.oss.endpoint= <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1\u7684 AccessKey ID \u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7 AccessKey \u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> # \u914d\u7f6eNamespace service \u5730\u5740 fs.jindofsx.namespace.rpc.address = <hostname>:<port> # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable = false \u6302\u8f7d JindoFuse \u00a7 \u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002 \u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 mkdir -p <mount_point> \u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 jindo-fuse <mount_point> -ouri=[<oss_path>] -oxengine=jindofsx -ouri \u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684 oss \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a Bucket \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002 \u8bbf\u95ee JindoFuse \u00a7 \u5982\u679c\u5c06 OSS-HDFS \u670d\u52a1\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1 ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/ \u5378\u8f7d JindoFuse \u00a7 \u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point> \u81ea\u52a8\u5378\u8f7d JindoFuse \u00a7 \u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u9ad8\u9636\u4f7f\u7528 \u00a7 \u914d\u7f6e\u9009\u9879 \u00a7 \u66f4\u591a jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<oss_path>] -ofs.jindofsx.data.cache.enable=false","title":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f OSS"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#jindofuse-oss","text":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002JindoFuse \u4f7f\u7528\u539f\u6765\u7684\u5730\u5740 (oss://) \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c\u8be5\u670d\u52a1\u4e0a\u7684\u6587\u4ef6\uff0c\u540c\u65f6\u901a\u8fc7\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002","title":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f OSS"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#_1","text":"","title":"\u57fa\u672c\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#_2","text":"# CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev","title":"\u73af\u5883\u51c6\u5907"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#_3","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#_4","text":"\u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-filestore] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = cn-xxx.oss-dls.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002","title":"\u914d\u7f6e\u670d\u52a1\u7aef"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#jindofsx","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#_5","text":"","title":"\u914d\u7f6e\u5ba2\u6237\u7aef"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#_6","text":"\u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf","title":"\u914d\u7f6e\u76ee\u5f55"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#_7","text":"\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u6216\u914d\u7f6e\u963f\u91cc\u4e91 OSS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou-internal.aliyuncs.com\u3002 fs.oss.endpoint= <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1\u7684 AccessKey ID \u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7 AccessKey \u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> # \u914d\u7f6eNamespace service \u5730\u5740 fs.jindofsx.namespace.rpc.address = <hostname>:<port> # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable = false","title":"\u914d\u7f6e\u6587\u4ef6"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#jindofuse","text":"\u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002","title":"\u6302\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#_8","text":"mkdir -p <mount_point>","title":"\u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#fuse","text":"jindo-fuse <mount_point> -ouri=[<oss_path>] -oxengine=jindofsx -ouri \u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684 oss \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a Bucket \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002","title":"\u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#jindofuse_1","text":"\u5982\u679c\u5c06 OSS-HDFS \u670d\u52a1\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1 ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/","title":"\u8bbf\u95ee JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#jindofuse_2","text":"\u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point>","title":"\u5378\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#jindofuse_3","text":"\u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002","title":"\u81ea\u52a8\u5378\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#_9","text":"","title":"\u9ad8\u9636\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#_10","text":"\u66f4\u591a jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<oss_path>] -ofs.jindofsx.data.cache.enable=false","title":"\u914d\u7f6e\u9009\u9879"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u900f\u660e\u7f13\u5b58\u52a0\u901f \u00a7 \u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u900f\u660e\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u670d\u52a1\u7aef\u914d\u7f6e OSS-HDFS \u670d\u52a1 AccessKey \u00a7 \u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS-HDFS \u670d\u52a1 bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = cn-xxx.oss-dls.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = cn-xxx.oss-dls.aliyuncs.com jindofsx.oss.bucket.YYY.user = xxx #Storage Service \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u4f7f\u7528\u7684\u7528\u6237\u540d \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS-HDFS \u670d\u52a1 bucket \u540d\u79f0\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS-HDFS \u670d\u52a1 bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06 OSS-HDFS \u670d\u52a1 \u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u914d\u7f6e\u7f13\u5b58\u52a0\u901f \u00a7 \u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u900f\u660e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs/#oss-hdfs-jindofs","text":"\u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u900f\u660e\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u900f\u660e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs/#oss-hdfs-accesskey","text":"\u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS-HDFS \u670d\u52a1 bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = cn-xxx.oss-dls.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = cn-xxx.oss-dls.aliyuncs.com jindofsx.oss.bucket.YYY.user = xxx #Storage Service \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u4f7f\u7528\u7684\u7528\u6237\u540d \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS-HDFS \u670d\u52a1 bucket \u540d\u79f0\u3002","title":"\u670d\u52a1\u7aef\u914d\u7f6e OSS-HDFS \u670d\u52a1 AccessKey"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs/#jindofsx","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS-HDFS \u670d\u52a1 bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs/#jindosdk","text":"\u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06 OSS-HDFS \u670d\u52a1 \u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs/#_2","text":"\u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e","title":"\u914d\u7f6e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs/#_3","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs_jindo/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f \u00a7 \u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u670d\u52a1\u7aef\u914d\u7f6e OSS-HDFS \u670d\u52a1 AccessKey \u00a7 \u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS-HDFS \u670d\u52a1 bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = cn-xxx.oss-dls.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.user = xxx #Storage Service \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u4f7f\u7528\u7684\u7528\u6237\u540d \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS-HDFS \u670d\u52a1 bucket \u7684\u540d\u79f0\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS-HDFS \u670d\u52a1 bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AccessKey \u5c06 OSS-HDFS \u670d\u52a1 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002\u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u6302\u8f7d OSS-HDFS \u670d\u52a1 \u00a7 \u6302\u8f7d OSS-HDFS \u670d\u52a1\u76ee\u5f55 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindodls oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219/jindodls \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/ [root@emr-header-1 ~]# hdfs dfs -ls jindo://emr-header-1:8101/ Found 1 items ---------- 1 0 1970-01-01 08:00 jindo://emr-header-1:8101/jindodls \u5373\u8bbf\u95ee jindo://emr-header-1:8101/jindodls/ \u7b49\u4ef7\u4e8e\u8bbf\u95ee oss://<Bucket>.<Endpoint>/ \u914d\u7f6e\u7f13\u5b58\u52a0\u901f \u00a7 \u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u00a7 \u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u5728\u6570\u636e\u7f13\u5b58\u5f00\u5173\u6253\u5f00\u65f6\u901a\u8fc7 jindo:// \u524d\u7f00\u8bfb\u53d6 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 jindo:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002 \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs_jindo/#oss-hdfs-jindofs","text":"\u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs_jindo/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs_jindo/#oss-hdfs-accesskey","text":"\u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS-HDFS \u670d\u52a1 bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = cn-xxx.oss-dls.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.user = xxx #Storage Service \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u4f7f\u7528\u7684\u7528\u6237\u540d \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS-HDFS \u670d\u52a1 bucket \u7684\u540d\u79f0\u3002","title":"\u670d\u52a1\u7aef\u914d\u7f6e OSS-HDFS \u670d\u52a1 AccessKey"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs_jindo/#jindofsx","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS-HDFS \u670d\u52a1 bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs_jindo/#jindosdk","text":"\u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AccessKey \u5c06 OSS-HDFS \u670d\u52a1 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002\u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs_jindo/#oss-hdfs","text":"\u6302\u8f7d OSS-HDFS \u670d\u52a1\u76ee\u5f55 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindodls oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219/jindodls \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/ [root@emr-header-1 ~]# hdfs dfs -ls jindo://emr-header-1:8101/ Found 1 items ---------- 1 0 1970-01-01 08:00 jindo://emr-header-1:8101/jindodls \u5373\u8bbf\u95ee jindo://emr-header-1:8101/jindodls/ \u7b49\u4ef7\u4e8e\u8bbf\u95ee oss://<Bucket>.<Endpoint>/","title":"\u6302\u8f7d OSS-HDFS \u670d\u52a1"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs_jindo/#_2","text":"\u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e","title":"\u914d\u7f6e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs_jindo/#oss-hdfs_1","text":"\u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u5728\u6570\u636e\u7f13\u5b58\u5f00\u5173\u6253\u5f00\u65f6\u901a\u8fc7 jindo:// \u524d\u7f00\u8bfb\u53d6 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 jindo:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002","title":"\u8bbf\u95ee OSS-HDFS \u670d\u52a1"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs_jindo/#_3","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/jindotable/table_cache/","text":"\u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e \u00a7 JindoTable \u53ef\u4ee5\u4f7f\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e\uff0c\u4ece\u800c\u63d0\u9ad8\u8bbf\u95ee\u901f\u5ea6\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u00a7 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u00a7 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883 \u00a7 \u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e cache \u547d\u4ee4 \u00a7 \u529f\u80fd \u00a7 \u8868\u793a\u7f13\u5b58\u6307\u5b9a\u8868\u6216\u5206\u533a\u7684\u6570\u636e\u81f3\u96c6\u7fa4\u672c\u5730\u78c1\u76d8\u4e0a\u3002 \u8868\u6216\u5206\u533a\u7684\u8def\u5f84\u9700\u8981\u4f4d\u4e8e OSS \u6216 OSS-HDFS \u670d\u52a1\u3002 \u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f \u00a7 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help cache \u53c2\u6570\u8bf4\u660e \u00a7 jindotable -cache [-pin] [-s] [-l] [-m] [-r replica] [-b batchsize] -t <dbName.tableName> [-p <partitionSpec>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u6307\u5b9a\u8981\u7f13\u5b58\u7684\u8868 \u662f -p <partitionSpec> \u6307\u5b9a\u8981\u7f13\u5b58\u7684\u5206\u533a\uff0c\u4f7f\u7528partitionCol1=1,partitionCol2=2,...\u7684\u683c\u5f0f \u5426 -pin \u5728\u7f13\u5b58\u7a7a\u95f4\u4e0d\u8db3\u65f6\u5c3d\u91cf\u4e0d\u5220\u9664\u76f8\u5173\u6570\u636e\u3002 \u5426 -s \u662f\u5426\u540c\u6b65 \u5426 -l \u662f\u5426\u662f\u5c0f\u6587\u4ef6 \u5426 -m \u662f\u5426\u5728\u5185\u5b58\u7f13\u5b58 \u5426 -r replica \u7f13\u5b58\u7684\u526f\u672c\u6570 \u5426 -b batchsize \u7f13\u5b58\u7684\u5e76\u53d1\u6570 \u5426 \u793a\u4f8b \u00a7 \u7f13\u5b582020-03-16\u65e5 db1.t1 \u8868\u7684\u6570\u636e\u81f3\u672c\u5730\u78c1\u76d8\u4e0a\u3002 jindotable -cache -t db1.t1 -p date=2020-03-16 uncache \u547d\u4ee4 \u00a7 \u529f\u80fd \u00a7 \u8868\u793a\u5220\u9664\u96c6\u7fa4\u672c\u5730\u78c1\u76d8\u4e0a\u6307\u5b9a\u8868\u6216\u5206\u533a\u7684\u7f13\u5b58\u6570\u636e\u3002 \u5bf9\u5e94\u7684\u8def\u5f84\u9700\u8981\u4f4d\u4e8e OSS \u6216 OSS-HDFS \u670d\u52a1\u3002 \u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f \u00a7 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help uncache \u53c2\u6570\u8bf4\u660e \u00a7 jindotable -uncache [-s] -t <dbName.tableName> [-p <partitionSpec>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u6307\u5b9a\u8981\u5220\u9664\u7f13\u5b58\u7684\u8868 \u662f -p <partitionSpec> \u6307\u5b9a\u8981\u5220\u9664\u7f13\u5b58\u7684\u5206\u533a\uff0c\u4f7f\u7528partitionCol1=1,partitionCol2=2,...\u7684\u683c\u5f0f \u5426 \u793a\u4f8b \u00a7 \u5220\u9664\u96c6\u7fa4\u672c\u5730\u78c1\u76d8\u4e0a\u8868 db1.t2 \u7684\u7f13\u5b58\u6570\u636e\u3002 jindotable -uncache -t db1.t2 \u5220\u9664\u96c6\u7fa4\u672c\u5730\u78c1\u76d8\u4e0a\u8868 db1.t1 \u4e2d\u6307\u5b9a\u5206\u533a\u7684\u7f13\u5b58\u6570\u636e\u3002 jindotable -uncache -t db1.t1 -p date=2020-03-16,category=1","title":"\u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e"},{"location":"backup/jindofsx/jindotable/table_cache/#jindotable-hive","text":"JindoTable \u53ef\u4ee5\u4f7f\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e\uff0c\u4ece\u800c\u63d0\u9ad8\u8bbf\u95ee\u901f\u5ea6\u3002","title":"\u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e"},{"location":"backup/jindofsx/jindotable/table_cache/#_1","text":"","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/jindofsx/jindotable/table_cache/#jindofsx","text":"\u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf","title":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf"},{"location":"backup/jindofsx/jindotable/table_cache/#jindosdk","text":"\u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u5df2\u90e8\u7f72 JindoSDK"},{"location":"backup/jindofsx/jindotable/table_cache/#hadoop-hive","text":"\u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e","title":"\u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883"},{"location":"backup/jindofsx/jindotable/table_cache/#cache","text":"","title":"cache \u547d\u4ee4"},{"location":"backup/jindofsx/jindotable/table_cache/#_2","text":"\u8868\u793a\u7f13\u5b58\u6307\u5b9a\u8868\u6216\u5206\u533a\u7684\u6570\u636e\u81f3\u96c6\u7fa4\u672c\u5730\u78c1\u76d8\u4e0a\u3002 \u8868\u6216\u5206\u533a\u7684\u8def\u5f84\u9700\u8981\u4f4d\u4e8e OSS \u6216 OSS-HDFS \u670d\u52a1\u3002","title":"\u529f\u80fd"},{"location":"backup/jindofsx/jindotable/table_cache/#_3","text":"\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help cache","title":"\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f"},{"location":"backup/jindofsx/jindotable/table_cache/#_4","text":"jindotable -cache [-pin] [-s] [-l] [-m] [-r replica] [-b batchsize] -t <dbName.tableName> [-p <partitionSpec>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u6307\u5b9a\u8981\u7f13\u5b58\u7684\u8868 \u662f -p <partitionSpec> \u6307\u5b9a\u8981\u7f13\u5b58\u7684\u5206\u533a\uff0c\u4f7f\u7528partitionCol1=1,partitionCol2=2,...\u7684\u683c\u5f0f \u5426 -pin \u5728\u7f13\u5b58\u7a7a\u95f4\u4e0d\u8db3\u65f6\u5c3d\u91cf\u4e0d\u5220\u9664\u76f8\u5173\u6570\u636e\u3002 \u5426 -s \u662f\u5426\u540c\u6b65 \u5426 -l \u662f\u5426\u662f\u5c0f\u6587\u4ef6 \u5426 -m \u662f\u5426\u5728\u5185\u5b58\u7f13\u5b58 \u5426 -r replica \u7f13\u5b58\u7684\u526f\u672c\u6570 \u5426 -b batchsize \u7f13\u5b58\u7684\u5e76\u53d1\u6570 \u5426","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"backup/jindofsx/jindotable/table_cache/#_5","text":"\u7f13\u5b582020-03-16\u65e5 db1.t1 \u8868\u7684\u6570\u636e\u81f3\u672c\u5730\u78c1\u76d8\u4e0a\u3002 jindotable -cache -t db1.t1 -p date=2020-03-16","title":"\u793a\u4f8b"},{"location":"backup/jindofsx/jindotable/table_cache/#uncache","text":"","title":"uncache \u547d\u4ee4"},{"location":"backup/jindofsx/jindotable/table_cache/#_6","text":"\u8868\u793a\u5220\u9664\u96c6\u7fa4\u672c\u5730\u78c1\u76d8\u4e0a\u6307\u5b9a\u8868\u6216\u5206\u533a\u7684\u7f13\u5b58\u6570\u636e\u3002 \u5bf9\u5e94\u7684\u8def\u5f84\u9700\u8981\u4f4d\u4e8e OSS \u6216 OSS-HDFS \u670d\u52a1\u3002","title":"\u529f\u80fd"},{"location":"backup/jindofsx/jindotable/table_cache/#_7","text":"\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help uncache","title":"\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f"},{"location":"backup/jindofsx/jindotable/table_cache/#_8","text":"jindotable -uncache [-s] -t <dbName.tableName> [-p <partitionSpec>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u6307\u5b9a\u8981\u5220\u9664\u7f13\u5b58\u7684\u8868 \u662f -p <partitionSpec> \u6307\u5b9a\u8981\u5220\u9664\u7f13\u5b58\u7684\u5206\u533a\uff0c\u4f7f\u7528partitionCol1=1,partitionCol2=2,...\u7684\u683c\u5f0f \u5426","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"backup/jindofsx/jindotable/table_cache/#_9","text":"\u5220\u9664\u96c6\u7fa4\u672c\u5730\u78c1\u76d8\u4e0a\u8868 db1.t2 \u7684\u7f13\u5b58\u6570\u636e\u3002 jindotable -uncache -t db1.t2 \u5220\u9664\u96c6\u7fa4\u672c\u5730\u78c1\u76d8\u4e0a\u8868 db1.t1 \u4e2d\u6307\u5b9a\u5206\u533a\u7684\u7f13\u5b58\u6570\u636e\u3002 jindotable -uncache -t db1.t1 -p date=2020-03-16,category=1","title":"\u793a\u4f8b"},{"location":"backup/jindofsx/multi-cloud/jindofsx_on_multi_cloud/","text":"S3\u3001COS\u3001OBS \u591a\u4e91\u5b58\u50a8\u652f\u6301 \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u4e86\u900f\u660e\u7f13\u5b58\u548c\u7edf\u4e00\u6302\u8f7d\u547d\u540d\u7a7a\u95f4\u7684\u4f7f\u7528\u65b9\u5f0f\uff0c\u652f\u6301\u591a\u79cd scheme \u7684\u4e91\u5b58\u50a8\u3002\u6587\u4ef6\u90fd\u4ee5\u5bf9\u8c61\u7684\u5f62\u5f0f\u5b58\u50a8\u5728\u4e91\u5b58\u50a8\u4e0a\uff0c\u6bcf\u4e2a\u6587\u4ef6\u6839\u636e\u5b9e\u9645\u8bbf\u95ee\u60c5\u51b5\u4f1a\u5728\u672c\u5730\u8fdb\u884c\u7f13\u5b58\uff0c\u63d0\u5347\u8bbf\u95ee\u4e91\u5b58\u50a8\u7684\u6548\u7387\uff0c\u540c\u65f6\u517c\u5bb9\u4e86\u539f\u6709\u4e91\u5b58\u50a8\u6587\u4ef6\u5f62\u5f0f\uff0c\u6570\u636e\u8bbf\u95ee\u4e0a\u80fd\u591f\u4e0e\u5176\u4ed6\u4e91\u5b58\u50a8\u5ba2\u6237\u7aef\u5b8c\u5168\u517c\u5bb9\uff0c\u4f5c\u4e1a\u8bbf\u95ee\u4e91\u5b58\u50a8\u7684\u65b9\u5f0f\u65e0\u9700\u505a\u4efb\u4f55\u4fee\u6539\u3002 \u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301 S3\u3001COS\u3001OBS \u900f\u660e\u7f13\u5b58\u52a0\u901f\u548c\u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u670d\u52a1\u7aef\u914d\u7f6e\u591a\u4e91 AccessKey \u00a7 \u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] jindofsx.s3.bucket.XXX.accessKeyId = xxx jindofsx.s3.bucket.XXX.accessKeySecret = xxx jindofsx.s3.bucket.XXX.endpoint = s3.xxx.amazonaws.com jindofsx.cos.bucket.YYY.accessKeyId = xxx jindofsx.cos.bucket.YYY.accessKeySecret = xxx jindofsx.cos.bucket.YYY.endpoint = cos.xxx.myqcloud.com jindofsx.obs.bucket.ZZZ.accessKeyId = xxx jindofsx.obs.bucket.ZZZ.accessKeySecret = xxx jindofsx.obs.bucket.ZZZ.endpoint = obs.xxx.myhuaweicloud.com \u8bf4\u660e: XXX\u3001YYY\u3001ZZZ \u5206\u522b\u4e3a S3 bucket\u3001COS bucket\u3001OBS bucket \u7684\u540d\u79f0\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u591a\u4e91\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1\u591a\u4e91\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> <!-- S3 --> <property> <name>fs.AbstractFileSystem.s3.impl</name> <value>com.aliyun.jindodata.s3.S3</value> </property> <property> <name>fs.s3.impl</name> <value>com.aliyun.jindodata.s3.JindoS3FileSystem</value> </property> <!-- COS --> <property> <name>fs.AbstractFileSystem.cos.impl</name> <value>com.aliyun.jindodata.cos.Cos</value> </property> <property> <name>fs.cos.impl</name> <value>com.aliyun.jindodata.cos.JindoCosFileSystem</value> </property> <!-- OBS --> <property> <name>fs.AbstractFileSystem.obs.impl</name> <value>com.aliyun.jindodata.obs.OBS</value> </property> <property> <name>fs.obs.impl</name> <value>com.aliyun.jindodata.obs.JindoObsFileSystem</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06\u591a\u4e91\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <!-- S3 --> <property> <name>fs.s3.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.s3.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.s3.endpoint</name> <value>xxx</value> </property> <!-- COS --> <property> <name>fs.cos.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.cos.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.cos.endpoint</name> <value>xxx</value> </property> <!-- OBS --> <property> <name>fs.obs.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.obs.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.obs.endpoint</name> <value>xxx</value> </property> </configuration> \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u6302\u8f7d\u591a\u4e91\u5b58\u50a8 \u00a7 \u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u4ee5\u6302\u8f7d AWS S3 \u76ee\u5f55\u4e3a\u4f8b\uff0ccos\u548cobs\u4f7f\u7528\u65b9\u6cd5\u7c7b\u4f3c\u3002 * \u6302\u8f7d S3 \u76ee\u5f55 jindo admin -mount <path> <realpath> \u4f8b\uff1a jindo admin -mount /jindos3 s3://<Bucket>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindos3 \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f s3://<Bucket>/ [root@emr-header-1 ~]# hdfs dfs -ls jindo://emr-header-1:8101/ Found 1 items ---------- 1 0 1970-01-01 08:00 jindo://emr-header-1:8101/jindos3 \u5373\u8bbf\u95ee jindo://emr-header-1:8101/jindos3/ \u7b49\u4ef7\u4e8e\u8bbf\u95ee s3://<Bucket>/ \u914d\u7f6e\u7f13\u5b58\u52a0\u901f \u00a7 \u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u8bbf\u95ee AWS S3 \u00a7 \u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u5728\u6570\u636e\u7f13\u5b58\u5f00\u5173\u6253\u5f00\u65f6\u901a\u8fc7 jindo:// \u524d\u7f00\u8bfb\u53d6 S3 \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 jindo:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002 \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"S3\u3001COS\u3001OBS \u591a\u4e91\u5b58\u50a8\u652f\u6301"},{"location":"backup/jindofsx/multi-cloud/jindofsx_on_multi_cloud/#s3cosobs","text":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u4e86\u900f\u660e\u7f13\u5b58\u548c\u7edf\u4e00\u6302\u8f7d\u547d\u540d\u7a7a\u95f4\u7684\u4f7f\u7528\u65b9\u5f0f\uff0c\u652f\u6301\u591a\u79cd scheme \u7684\u4e91\u5b58\u50a8\u3002\u6587\u4ef6\u90fd\u4ee5\u5bf9\u8c61\u7684\u5f62\u5f0f\u5b58\u50a8\u5728\u4e91\u5b58\u50a8\u4e0a\uff0c\u6bcf\u4e2a\u6587\u4ef6\u6839\u636e\u5b9e\u9645\u8bbf\u95ee\u60c5\u51b5\u4f1a\u5728\u672c\u5730\u8fdb\u884c\u7f13\u5b58\uff0c\u63d0\u5347\u8bbf\u95ee\u4e91\u5b58\u50a8\u7684\u6548\u7387\uff0c\u540c\u65f6\u517c\u5bb9\u4e86\u539f\u6709\u4e91\u5b58\u50a8\u6587\u4ef6\u5f62\u5f0f\uff0c\u6570\u636e\u8bbf\u95ee\u4e0a\u80fd\u591f\u4e0e\u5176\u4ed6\u4e91\u5b58\u50a8\u5ba2\u6237\u7aef\u5b8c\u5168\u517c\u5bb9\uff0c\u4f5c\u4e1a\u8bbf\u95ee\u4e91\u5b58\u50a8\u7684\u65b9\u5f0f\u65e0\u9700\u505a\u4efb\u4f55\u4fee\u6539\u3002 \u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301 S3\u3001COS\u3001OBS \u900f\u660e\u7f13\u5b58\u52a0\u901f\u548c\u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002","title":"S3\u3001COS\u3001OBS \u591a\u4e91\u5b58\u50a8\u652f\u6301"},{"location":"backup/jindofsx/multi-cloud/jindofsx_on_multi_cloud/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/multi-cloud/jindofsx_on_multi_cloud/#accesskey","text":"\u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] jindofsx.s3.bucket.XXX.accessKeyId = xxx jindofsx.s3.bucket.XXX.accessKeySecret = xxx jindofsx.s3.bucket.XXX.endpoint = s3.xxx.amazonaws.com jindofsx.cos.bucket.YYY.accessKeyId = xxx jindofsx.cos.bucket.YYY.accessKeySecret = xxx jindofsx.cos.bucket.YYY.endpoint = cos.xxx.myqcloud.com jindofsx.obs.bucket.ZZZ.accessKeyId = xxx jindofsx.obs.bucket.ZZZ.accessKeySecret = xxx jindofsx.obs.bucket.ZZZ.endpoint = obs.xxx.myhuaweicloud.com \u8bf4\u660e: XXX\u3001YYY\u3001ZZZ \u5206\u522b\u4e3a S3 bucket\u3001COS bucket\u3001OBS bucket \u7684\u540d\u79f0\u3002","title":"\u670d\u52a1\u7aef\u914d\u7f6e\u591a\u4e91 AccessKey"},{"location":"backup/jindofsx/multi-cloud/jindofsx_on_multi_cloud/#jindofsx","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/multi-cloud/jindofsx_on_multi_cloud/#jindosdk","text":"\u914d\u7f6e\u591a\u4e91\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1\u591a\u4e91\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> <!-- S3 --> <property> <name>fs.AbstractFileSystem.s3.impl</name> <value>com.aliyun.jindodata.s3.S3</value> </property> <property> <name>fs.s3.impl</name> <value>com.aliyun.jindodata.s3.JindoS3FileSystem</value> </property> <!-- COS --> <property> <name>fs.AbstractFileSystem.cos.impl</name> <value>com.aliyun.jindodata.cos.Cos</value> </property> <property> <name>fs.cos.impl</name> <value>com.aliyun.jindodata.cos.JindoCosFileSystem</value> </property> <!-- OBS --> <property> <name>fs.AbstractFileSystem.obs.impl</name> <value>com.aliyun.jindodata.obs.OBS</value> </property> <property> <name>fs.obs.impl</name> <value>com.aliyun.jindodata.obs.JindoObsFileSystem</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06\u591a\u4e91\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <!-- S3 --> <property> <name>fs.s3.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.s3.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.s3.endpoint</name> <value>xxx</value> </property> <!-- COS --> <property> <name>fs.cos.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.cos.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.cos.endpoint</name> <value>xxx</value> </property> <!-- OBS --> <property> <name>fs.obs.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.obs.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.obs.endpoint</name> <value>xxx</value> </property> </configuration> \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/multi-cloud/jindofsx_on_multi_cloud/#_2","text":"\u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u4ee5\u6302\u8f7d AWS S3 \u76ee\u5f55\u4e3a\u4f8b\uff0ccos\u548cobs\u4f7f\u7528\u65b9\u6cd5\u7c7b\u4f3c\u3002 * \u6302\u8f7d S3 \u76ee\u5f55 jindo admin -mount <path> <realpath> \u4f8b\uff1a jindo admin -mount /jindos3 s3://<Bucket>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindos3 \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f s3://<Bucket>/ [root@emr-header-1 ~]# hdfs dfs -ls jindo://emr-header-1:8101/ Found 1 items ---------- 1 0 1970-01-01 08:00 jindo://emr-header-1:8101/jindos3 \u5373\u8bbf\u95ee jindo://emr-header-1:8101/jindos3/ \u7b49\u4ef7\u4e8e\u8bbf\u95ee s3://<Bucket>/","title":"\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u6302\u8f7d\u591a\u4e91\u5b58\u50a8"},{"location":"backup/jindofsx/multi-cloud/jindofsx_on_multi_cloud/#_3","text":"\u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e","title":"\u914d\u7f6e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/multi-cloud/jindofsx_on_multi_cloud/#aws-s3","text":"\u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u5728\u6570\u636e\u7f13\u5b58\u5f00\u5173\u6253\u5f00\u65f6\u901a\u8fc7 jindo:// \u524d\u7f00\u8bfb\u53d6 S3 \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 jindo:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002","title":"\u8bbf\u95ee AWS S3"},{"location":"backup/jindofsx/multi-cloud/jindofsx_on_multi_cloud/#_4","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/nas/jindofsx_on_nas_jindo/","text":"\u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8 NAS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f \u00a7 \u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8NAS\uff08Apsara File Storage NAS\uff09\u662f\u4e00\u4e2a\u53ef\u5927\u89c4\u6a21\u5171\u4eab\u8bbf\u95ee\uff0c\u5f39\u6027\u6269\u5c55\u7684\u9ad8\u6027\u80fd\u4e91\u539f\u751f\u5206\u5e03\u5f0f\u6587\u4ef6\u7cfb\u7edf\u3002\u652f\u6301\u667a\u80fd\u51b7\u70ed\u6570\u636e\u5206\u5c42\uff0c\u6709\u6548\u964d\u4f4e\u6570\u636e\u5b58\u50a8\u6210\u672c\u3002\u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301\u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8 NAS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u96c6\u7fa4\u5df2\u6302\u8f7d NAS \u6587\u4ef6\u7cfb\u7edf \u6302\u8f7d\u6b65\u9aa4\u8bf7\u53c2\u8003 Linux\u7cfb\u7edf\u6302\u8f7dNFS\u6587\u4ef6\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u6302\u8f7d NAS \u6587\u4ef6\u7cfb\u7edf\u76ee\u5f55 \u00a7 jindo admin -mount <path> <realpath> \u4f8b\uff1a jindo admin -mount /jindonas local:///mnt/nas \u5047\u8bbe NAS \u6587\u4ef6\u7cfb\u7edf\u6302\u8f7d\u5728\u670d\u52a1\u5668\u4e0a\u7684\u672c\u5730\u8def\u5f84\u4e3a /mnt/nas , \u5728\u8fdb\u884c\u6302\u8f7d\u65f6\u9700\u8981\u52a0\u4e0a local:// \u524d\u7f00\u3002\u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindonas \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f /mnt/nas [root@emr-header-1 ~]# hdfs dfs -ls jindo://emr-header-1:8101/ Found 2 items ---------- 1 0 1970-01-01 08:00 jindo://emr-header-1:8101/jindonas \u5373\u8bbf\u95ee jindo://emr-header-1:8101/jindonas/ \u7b49\u4ef7\u4e8e\u8bbf\u95ee /mnt/nas/ \u914d\u7f6e\u7f13\u5b58\u52a0\u901f \u00a7 \u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u8bbf\u95ee NAS \u6587\u4ef6\u7cfb\u7edf \u00a7 \u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u5728\u6570\u636e\u7f13\u5b58\u5f00\u5173\u6253\u5f00\u65f6\u901a\u8fc7 jindo:// \u524d\u7f00\u8bfb\u53d6 NAS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 jindo:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002","title":"\u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8 NAS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/nas/jindofsx_on_nas_jindo/#nas","text":"\u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8NAS\uff08Apsara File Storage NAS\uff09\u662f\u4e00\u4e2a\u53ef\u5927\u89c4\u6a21\u5171\u4eab\u8bbf\u95ee\uff0c\u5f39\u6027\u6269\u5c55\u7684\u9ad8\u6027\u80fd\u4e91\u539f\u751f\u5206\u5e03\u5f0f\u6587\u4ef6\u7cfb\u7edf\u3002\u652f\u6301\u667a\u80fd\u51b7\u70ed\u6570\u636e\u5206\u5c42\uff0c\u6709\u6548\u964d\u4f4e\u6570\u636e\u5b58\u50a8\u6210\u672c\u3002\u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301\u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8 NAS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002","title":"\u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8 NAS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/nas/jindofsx_on_nas_jindo/#_1","text":"\u96c6\u7fa4\u5df2\u6302\u8f7d NAS \u6587\u4ef6\u7cfb\u7edf \u6302\u8f7d\u6b65\u9aa4\u8bf7\u53c2\u8003 Linux\u7cfb\u7edf\u6302\u8f7dNFS\u6587\u4ef6\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/nas/jindofsx_on_nas_jindo/#jindosdk","text":"\u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/nas/jindofsx_on_nas_jindo/#nas_1","text":"jindo admin -mount <path> <realpath> \u4f8b\uff1a jindo admin -mount /jindonas local:///mnt/nas \u5047\u8bbe NAS \u6587\u4ef6\u7cfb\u7edf\u6302\u8f7d\u5728\u670d\u52a1\u5668\u4e0a\u7684\u672c\u5730\u8def\u5f84\u4e3a /mnt/nas , \u5728\u8fdb\u884c\u6302\u8f7d\u65f6\u9700\u8981\u52a0\u4e0a local:// \u524d\u7f00\u3002\u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindonas \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f /mnt/nas [root@emr-header-1 ~]# hdfs dfs -ls jindo://emr-header-1:8101/ Found 2 items ---------- 1 0 1970-01-01 08:00 jindo://emr-header-1:8101/jindonas \u5373\u8bbf\u95ee jindo://emr-header-1:8101/jindonas/ \u7b49\u4ef7\u4e8e\u8bbf\u95ee /mnt/nas/","title":"\u6302\u8f7d NAS \u6587\u4ef6\u7cfb\u7edf\u76ee\u5f55"},{"location":"backup/jindofsx/nas/jindofsx_on_nas_jindo/#_2","text":"\u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e","title":"\u914d\u7f6e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/nas/jindofsx_on_nas_jindo/#nas_2","text":"\u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u5728\u6570\u636e\u7f13\u5b58\u5f00\u5173\u6253\u5f00\u65f6\u901a\u8fc7 jindo:// \u524d\u7f00\u8bfb\u53d6 NAS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 jindo:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002","title":"\u8bbf\u95ee NAS \u6587\u4ef6\u7cfb\u7edf"},{"location":"backup/jindofsx/oss/jindofsx_on_oss/","text":"\u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u4e86\u900f\u660e\u7f13\u5b58\u7684\u4f7f\u7528\u65b9\u5f0f\uff0c\u517c\u5bb9\u539f\u751f OSS \u5b58\u50a8\u65b9\u5f0f\uff0c\u6587\u4ef6\u4ee5\u5bf9\u8c61\u7684\u5f62\u5f0f\u5b58\u50a8\u5728 OSS \u4e0a\uff0c\u6bcf\u4e2a\u6587\u4ef6\u6839\u636e\u5b9e\u9645\u8bbf\u95ee\u60c5\u51b5\u4f1a\u5728\u672c\u5730\u8fdb\u884c\u7f13\u5b58\uff0c\u63d0\u5347\u8bbf\u95eeOSS\u7684\u6548\u7387\uff0c\u540c\u65f6\u517c\u5bb9\u4e86\u539f\u6709OSS\u539f\u6709\u6587\u4ef6\u5f62\u5f0f\uff0c\u6570\u636e\u8bbf\u95ee\u4e0a\u80fd\u591f\u4e0e\u5176\u4ed6OSS\u5ba2\u6237\u7aef\u5b8c\u5168\u517c\u5bb9\uff0c\u4f5c\u4e1a\u8bbf\u95ee OSS \u7684\u65b9\u5f0f\u65e0\u9700\u505a\u4efb\u4f55\u4fee\u6539\u3002 \u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u670d\u52a1\u7aef\u914d\u7f6e\u963f\u91cc\u4e91 OSS AccessKey \u00a7 \u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = oss-cn-xxx-internal.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1 OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06 OSS \u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u914d\u7f6e\u7f13\u5b58\u52a0\u901f \u00a7 \u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/oss/jindofsx_on_oss/#oss","text":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u4e86\u900f\u660e\u7f13\u5b58\u7684\u4f7f\u7528\u65b9\u5f0f\uff0c\u517c\u5bb9\u539f\u751f OSS \u5b58\u50a8\u65b9\u5f0f\uff0c\u6587\u4ef6\u4ee5\u5bf9\u8c61\u7684\u5f62\u5f0f\u5b58\u50a8\u5728 OSS \u4e0a\uff0c\u6bcf\u4e2a\u6587\u4ef6\u6839\u636e\u5b9e\u9645\u8bbf\u95ee\u60c5\u51b5\u4f1a\u5728\u672c\u5730\u8fdb\u884c\u7f13\u5b58\uff0c\u63d0\u5347\u8bbf\u95eeOSS\u7684\u6548\u7387\uff0c\u540c\u65f6\u517c\u5bb9\u4e86\u539f\u6709OSS\u539f\u6709\u6587\u4ef6\u5f62\u5f0f\uff0c\u6570\u636e\u8bbf\u95ee\u4e0a\u80fd\u591f\u4e0e\u5176\u4ed6OSS\u5ba2\u6237\u7aef\u5b8c\u5168\u517c\u5bb9\uff0c\u4f5c\u4e1a\u8bbf\u95ee OSS \u7684\u65b9\u5f0f\u65e0\u9700\u505a\u4efb\u4f55\u4fee\u6539\u3002 \u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002","title":"\u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/oss/jindofsx_on_oss/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/oss/jindofsx_on_oss/#oss-accesskey","text":"\u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = oss-cn-xxx-internal.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002","title":"\u670d\u52a1\u7aef\u914d\u7f6e\u963f\u91cc\u4e91 OSS AccessKey"},{"location":"backup/jindofsx/oss/jindofsx_on_oss/#jindofsx","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/oss/jindofsx_on_oss/#jindosdk","text":"\u914d\u7f6e OSS \u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1 OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06 OSS \u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/oss/jindofsx_on_oss/#_2","text":"\u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e","title":"\u914d\u7f6e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/oss/jindofsx_on_oss/#_3","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/oss/jindofsx_on_oss_jindo/","text":"\u963f\u91cc\u4e91 OSS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f \u00a7 \u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u670d\u52a1\u7aef\u914d\u7f6e\u963f\u91cc\u4e91 OSS AccessKey \u00a7 \u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = oss-cn-xxx-internal.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS AccessKey \u5c06 OSS Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u6302\u8f7d\u963f\u91cc\u4e91 OSS \u00a7 \u6302\u8f7d OSS \u76ee\u5f55 jindo admin -mount <path> <realpath> \u4f8b\uff1a jindo admin -mount /jindooss oss://<Bucket>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>/ [root@emr-header-1 ~]# hdfs dfs -ls jindo://emr-header-1:8101/ Found 1 items ---------- 1 0 1970-01-01 08:00 jindo://emr-header-1:8101/jindooss \u5373\u8bbf\u95ee jindo://emr-header-1:8101/jindooss/ \u7b49\u4ef7\u4e8e\u8bbf\u95ee oss://<Bucket>/ \u914d\u7f6e\u7f13\u5b58\u52a0\u901f \u00a7 \u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u8bbf\u95ee\u963f\u91cc\u4e91 OSS \u00a7 \u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u5728\u6570\u636e\u7f13\u5b58\u5f00\u5173\u6253\u5f00\u65f6\u901a\u8fc7 jindo:// \u524d\u7f00\u8bfb\u53d6 OSS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 jindo:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002 \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u963f\u91cc\u4e91 OSS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/oss/jindofsx_on_oss_jindo/#oss","text":"\u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002","title":"\u963f\u91cc\u4e91 OSS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/oss/jindofsx_on_oss_jindo/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/oss/jindofsx_on_oss_jindo/#oss-accesskey","text":"\u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = oss-cn-xxx-internal.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002","title":"\u670d\u52a1\u7aef\u914d\u7f6e\u963f\u91cc\u4e91 OSS AccessKey"},{"location":"backup/jindofsx/oss/jindofsx_on_oss_jindo/#jindofsx","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/oss/jindofsx_on_oss_jindo/#jindosdk","text":"\u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS AccessKey \u5c06 OSS Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/oss/jindofsx_on_oss_jindo/#oss_1","text":"\u6302\u8f7d OSS \u76ee\u5f55 jindo admin -mount <path> <realpath> \u4f8b\uff1a jindo admin -mount /jindooss oss://<Bucket>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>/ [root@emr-header-1 ~]# hdfs dfs -ls jindo://emr-header-1:8101/ Found 1 items ---------- 1 0 1970-01-01 08:00 jindo://emr-header-1:8101/jindooss \u5373\u8bbf\u95ee jindo://emr-header-1:8101/jindooss/ \u7b49\u4ef7\u4e8e\u8bbf\u95ee oss://<Bucket>/","title":"\u6302\u8f7d\u963f\u91cc\u4e91 OSS"},{"location":"backup/jindofsx/oss/jindofsx_on_oss_jindo/#_2","text":"\u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e","title":"\u914d\u7f6e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/oss/jindofsx_on_oss_jindo/#oss_2","text":"\u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u5728\u6570\u636e\u7f13\u5b58\u5f00\u5173\u6253\u5f00\u65f6\u901a\u8fc7 jindo:// \u524d\u7f00\u8bfb\u53d6 OSS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 jindo:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002","title":"\u8bbf\u95ee\u963f\u91cc\u4e91 OSS"},{"location":"backup/jindofsx/oss/jindofsx_on_oss_jindo/#_3","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/permission/jindofsx_ranger/","text":"\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u00a7 \u672c\u6587\u4ecb\u7ecd\u5728\u81ea\u5efa\u96c6\u7fa4\u914d\u7f6e\u963f\u91cc\u4e91 OSS \u548c OSS-HDFS \u670d\u52a1\u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\u3002 \u80cc\u666f \u00a7 Apache Ranger \u63d0\u4f9b\u96c6\u4e2d\u5f0f\u7684\u6743\u9650\u7ba1\u7406\u6846\u67b6\uff0c\u53ef\u4ee5\u5bf9 Hadoop \u751f\u6001\u4e2d\u7684\u591a\u4e2a\u7ec4\u4ef6\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u8bbf\u95ee\u63a7\u5236\u3002\u5f53\u7528\u6237\u5c06\u6570\u636e\u5b58\u653e\u5728\u963f\u91cc\u4e91 OSS \u65f6\uff0c\u5219\u662f\u901a\u8fc7\u963f\u91cc\u4e91 RAM \u4ea7\u54c1\u521b\u5efa\u6216\u7ba1\u7406 RAM \u7528\u6237\uff0c\u5bf9RAM\u7528\u6237\u5b9e\u73b0\u5bf9 OSS \u8d44\u6e90\u7684\u8bbf\u95ee\u63a7\u5236\u3002 \u4e3a\u7ef4\u6301\u5927\u6570\u636e\u5ba2\u6237\u7684\u4f7f\u7528\u4e60\u60ef\uff0c\u901a\u8fc7 JindoFSx Namespace \u63a5\u5165 Ranger \u7684\u5ba2\u6237\u7aef\uff0c\u65b9\u4fbf\u7528\u6237\u7edf\u4e00\u7ba1\u7406\u5927\u6570\u636e\u7ec4\u4ef6\u6743\u9650\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u5df2\u90e8\u7f72 JindoFSx Namespace \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx Namespace\uff0c\u8bf7\u53c2\u8003 \u5feb\u901f\u90e8\u7f72\u4e00\u4e2a\u7b80\u5355\u7684 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf(\u4ec5 Namespace) \u5b89\u88c5 Kerberos \u548c Sasl2 \u76f8\u5173\u4f9d\u8d56 \u5982\u679c\u60a8\u7684\u73af\u5883\u6ca1\u6709 Kerberos \u548c Sasl2 \u76f8\u5173\u4f9d\u8d56\uff0c\u8bf7\u5b89\u88c5\u76f8\u5173\u4f9d\u8d56 \u5b89\u88c5\u8bf4\u660e 1. \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u3002 \u00a7 1.1 \u914d\u7f6e Jindo SASL \u63d2\u4ef6 \u00a7 \u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 jindofsx-4.6.12/plugins/ \u8def\u5f84\u4e0b\u3002 \u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\uff0c \u5728 [jindofsx-namespace] section \u4e0b\u6dfb\u52a0\u52a8\u6001\u5e93\u7684\u7edd\u5bf9\u8def\u5f84\u3002 \u53c2\u6570 \u503c namespace.plugin.dir ${JINDDATA_HOME}/plugins/ 1.2 \u542f\u7528 Ranger \u9274\u6743 \u00a7 \u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\uff0c \u5728 [jindofsx-namespace] section \u4e0b\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.oss.authorization.method ranger \u6ce8\u610f\uff1a \u5c06 jindofsx.cfg \u914d\u7f6e\u6587\u4ef6\u90e8\u7f72\u5230 NamespaceService \u6240\u5728\u7684\u6240\u6709\u8282\u70b9\u3002 1.3 \u914d\u7f6e\u4f7f\u7528 Assume Role \u9881\u53d1\u4e34\u65f6 AK \u00a7 \u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 \u5728 [jindofsx-namespace] section \u4e0b\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.oss.credential.provider ASSUME_ROLE \u5728\u963f\u91cc\u4e91 RAM \u521b\u5efa\u7528\u6237\u9881\u53d1\u4e34\u65f6 AK \u7684 Role, \u9009\u62e9\u53ef\u4fe1\u5b9e\u4f53\u7c7b\u578b\u4e3a\"\u963f\u91cc\u4e91\u8d26\u53f7\", \u5e76\u4e3a\u89d2\u8272\u6388\u6743\u3002 \u5728 [jindofsx-common] section \u4e0b\u6dfb\u52a0\u6216\u66f4\u6539\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c default.credential.provider ASSUME_ROLE sts.access.key STS \u7684 accessKey sts.access.secret STS \u7684 accessKeySecret sts.access.endpoint STS \u7684 endpoint, \u5982\uff1a sts.cn-shanghai.aliyuncs.com sts.role.name \u5728 RAM \u4e2d\u521b\u5efa\u7684\u89d2\u8272\uff0c \u5982 JindoFsxTestRole sts.role.arn \u89d2\u8272\u7684ARN\uff0c\u5982\uff1aacs:ram::xxx:role/JindoFsxTestRole \u6ce8\u610f\uff1a \u5c06 jindofsx.cfg \u914d\u7f6e\u6587\u4ef6\u90e8\u7f72\u5230 NamespaceService \u6240\u5728\u7684\u6240\u6709\u8282\u70b9\u3002 1.4 \u914d\u7f6e Principal \u548c Keytab\uff08\u9002\u7528\u4e8e Kerberos \u96c6\u7fa4\uff09 \u00a7 \u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 \u5728 [jindofsx-namespace] section \u4e0b\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.authentication.enable true namespace.authentication.keytab \u914d\u7f6e keytab \u6240\u5728\u7684\u8def\u5f84\uff0c \u5982\uff1a /tmp/jindofsx.keytab namespace.authentication.principal \u914d\u7f6e keytab \u4e2d\u7684 principal, \u5982\uff1a jindofsx/localhost 2. \u91cd\u542f JindoFSx Namespace \u670d\u52a1\u3002 \u00a7 \u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\uff1a cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh 3. \u914d\u7f6e JindoSDK\u3002 \u00a7 3.1 \u914d\u7f6e xengine \u00a7 \u5728 Hadoop \u7684 core-site.xml \u4e2d \u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.xengine jindofsx 3.2 \u914d\u7f6e Jindo SASL \u63d2\u4ef6 \u00a7 \u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 jindosdk-4.6.12/plugins/ \u8def\u5f84\u4e0b\uff0c\u5c06\u8be5\u8def\u5f84\u7684\u7edd\u5bf9\u8def\u5f84\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 \u53c2\u6570 \u503c fs.jdo.plugin.dir ${JINDOSDK_HOME}/plugins/ 3.3 \u542f\u7528 Ranger \u9274\u6743 \u00a7 \u652f\u6301\u7edf\u4e00\u914d\u7f6e\u548c\u6309 bucket \u914d\u7f6e\u4e24\u79cd\u65b9\u5f0f\u3002 a. \u5bf9 OSS \u548c OSS-HDFS \u7684\u5168\u90e8 bucket \u9274\u6743 \u00a7 \u5728Hadoop \u7684 core-site.xml \u4e2d\u6dfb\u52a0\u6216\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.authorization.method ranger fs.oss.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider b. \u5bf9 OSS \u6216 OSS-HDFS \u7684\u90e8\u5206 bucket \u9274\u6743 \u00a7 \u5728Hadoop \u7684 core-site.xml \u4e2d\u6dfb\u52a0\u6216\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.bucket.XXX.authorization.method ranger fs.oss.bucket.XXX.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider \u8bf4\u660e\uff1aXXX \u4e3a \u6216 OSS \u6216 OSS-HDFS \u670d\u52a1 bucket\u7684\u540d\u79f0\u3002 4. \u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 5. \u5b89\u88c5\u652f\u6301\u914d\u7f6e OSS \u6743\u9650\u7684 Ranger \u63d2\u4ef6 \u00a7 Ranger \u7ec4\u4ef6\u9700\u8981\u5b89\u88c5\u65b0\u63d2\u4ef6\u4ee5\u652f\u6301\u5bf9 OSS \u6216 OSS-HDFS \u670d\u52a1\u5bf9\u9274\u6743\u3002 5.1 \u4e0b\u8f7d\u548c\u62f7\u8d1d\u63d2\u4ef6 \u00a7 \u6839\u636e\u96c6\u7fa4\u4e2d\u5b89\u88c5\u7684 Ranger \u7248\u672c\u4e0b\u8f7d\u5bf9\u5e94\u7684\u63d2\u4ef6\u7248\u672c\u3002 1.x \u7248\u672c \u4e0b\u8f7d\u5730\u5740 2.x \u7248\u672c \u4e0b\u8f7d\u5730\u5740 cd /xxx/ranger-admin-current/ews/webapp/WEB-INF/classes/ranger-plugins mkdir oss cp ranger-oss-plugin-1.2.0.jar /xxx/ranger-admin-current/ews/webapp/WEB-INF/classes/ranger-plugins/oss \u6ce8\u610f\uff1a\u9700\u8981\u5b89\u88c5\u5728\u90e8\u7f72 Ranger Admin \u7684\u6240\u6709\u8282\u70b9\u3002 5.2 \u5b89\u88c5\u63d2\u4ef6 \u00a7 \u521b\u5efa\u5982\u4e0branger-oss.json\u6587\u4ef6\uff1a { \"id\": 500, \"name\": \"oss\", \"implClass\": \"org.apache.ranger.services.oss.RangerServiceOSS\", \"label\": \"OSS\", \"description\": \"OSS\", \"guid\": \"0d047247-bafe-4cf8-8e9b-d5d377284b2h\", \"resources\": [ { \"itemId\": 1, \"name\": \"path\", \"type\": \"path\", \"level\": 10, \"parent\": \"\", \"mandatory\": true, \"lookupSupported\": true, \"recursiveSupported\": true, \"excludesSupported\": false, \"matcher\": \"org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher\", \"matcherOptions\": { \"wildCard\": true, \"ignoreCase\": false }, \"validationRegEx\": \"\", \"validationMessage\": \"\", \"uiHint\": \"\", \"label\": \"Path\", \"description\": \"OSS Path, Should Not Start With /, exp. dir/test.txt\" } ], \"accessTypes\": [ { \"itemId\": 1, \"name\": \"read\", \"label\": \"Read\" }, { \"itemId\": 2, \"name\": \"write\", \"label\": \"Write\" }, { \"itemId\": 3, \"name\": \"execute\", \"label\": \"Execute\" } ], \"configs\": [ ], \"enums\": [ ], \"contextEnrichers\": [], \"policyConditions\": [] } \u6267\u884c\u5982\u4e0b\u547d\u4ee4\u88c5\u8f7d ranger-oss\uff0c\u6ce8\u610f\u5176\u4e2d json \u6587\u4ef6\u7684\u8def\u5f84\u548c ranger admin \u7684 url\uff0cadminUser \u9ed8\u8ba4\u4e3a admin\uff0cadminPasswd \u9ed8\u8ba4\u4e3a admin\uff1a curl -v -u${adminUser}:${adminPasswd} -X POST -H \"Accept:application/json\" -H \"Content-Type:application/json\" -d @ranger-oss.json http://emr-header-1:6080/service/plugins/definitions \u63d2\u4ef6\u6210\u529f\u5b89\u88c5\u540e\uff0c\u53ef\u4ee5\u5728 Ranger UI \u4e0a\u770b\u5230\u65b0\u589e\u9879\uff1a \u521b\u5efa\u4e00\u4e2aService\uff0c\u4f8b\u5982emr-oss\uff0c\u521b\u5efa\u5b8c\u6210\u540e\u5982\u4e0b\uff0c\u540e\u7eed\u5c31\u53ef\u4ee5\u5728\u8be5Service\u4e0b\u521b\u5efaPolicy\uff1a 6. \u5728 Ranger WebUI \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1\u6743\u9650 \u00a7 Ranger \u89c4\u5219\u793a\u4f8b \u00a7 \u4f8b\uff1a\u914d\u7f6e\u7528\u6237test\u62e5\u6709\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u6240\u4ee5\u6743\u9650\u7684\u6b65\u9aa4\uff1a a. \u914d\u7f6e test \u7528\u6237\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u8bbf\u95ee\u6743\u9650\u4e3a ALL\u3002 \u00a7 \u8bf4\u660e\uff1a \u00a7 \u89c4\u5219\u914d\u7f6e\u9875\u9762\u4e2d\uff0c\u914d\u7f6e\u7684 path \u6ca1\u6709 oss:// \u7684\u524d\u7f00\u3002 recursive\u6309\u94ae\u4e0d\u53ef\u5173\u95ed\u3002 \u8def\u5f84\u672b\u5c3e\u65e0\u9700\u5e26\u6b63\u659c\u7ebf\uff08/\uff09\u3002 b. \u9700\u8981\u914d\u7f6e\u8bbf\u95ee\u8def\u5f84\u7684\u7236\u76ee\u5f55 oss://bucket-test-hangzhou/user \u7684\u6743\u9650\u4e3a Execute\u3002 \u00a7 7. \u8bbf\u95eeOSS\u3002 \u00a7 \u82e5\u7528\u6237\u8bbf\u95ee Ranger \u6ca1\u6709\u6388\u6743\u7684\u8def\u5f84\uff0c\u5219\u4f1a\u62a5\u5982\u4e0b\u9519\u8bef\uff1a org.apache.hadoop.security.AccessControlException: Permission denied: user=test, access=READ_EXECUTE, resourcePath=\"bucket-test-hangzhou/\" \u6ce8\u610f\u4e8b\u9879 \u00a7 \u9488\u5bf9\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7684\u9274\u6743\uff0c\u76ee\u524d\u4f7f\u7528\u7684\u662f\u5bf9\u8c61\u5b58\u50a8\u7684\u9274\u6743\u65b9\u5f0f\uff08\u901a\u8fc7\u5339\u914d\u8def\u5f84\uff09\uff0c\u5bf9\u8c61\u5b58\u50a8\u6ca1\u6709\u76ee\u5f55\u6216\u6587\u4ef6\u7684 owner \u548c permission \u4fe1\u606f\uff0c\u5982\u9700\u5bf9 OSS-HDFS \u670d\u52a1\u4f7f\u7528 chmod , chown \u7b49\u64cd\u4f5c\uff0c\u53ea\u80fd\u901a\u8fc7 hadoop \u7528\u6237\u6267\u884c\u3002","title":"\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#oss-ranger","text":"\u672c\u6587\u4ecb\u7ecd\u5728\u81ea\u5efa\u96c6\u7fa4\u914d\u7f6e\u963f\u91cc\u4e91 OSS \u548c OSS-HDFS \u670d\u52a1\u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\u3002","title":"\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#_1","text":"Apache Ranger \u63d0\u4f9b\u96c6\u4e2d\u5f0f\u7684\u6743\u9650\u7ba1\u7406\u6846\u67b6\uff0c\u53ef\u4ee5\u5bf9 Hadoop \u751f\u6001\u4e2d\u7684\u591a\u4e2a\u7ec4\u4ef6\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u8bbf\u95ee\u63a7\u5236\u3002\u5f53\u7528\u6237\u5c06\u6570\u636e\u5b58\u653e\u5728\u963f\u91cc\u4e91 OSS \u65f6\uff0c\u5219\u662f\u901a\u8fc7\u963f\u91cc\u4e91 RAM \u4ea7\u54c1\u521b\u5efa\u6216\u7ba1\u7406 RAM \u7528\u6237\uff0c\u5bf9RAM\u7528\u6237\u5b9e\u73b0\u5bf9 OSS \u8d44\u6e90\u7684\u8bbf\u95ee\u63a7\u5236\u3002 \u4e3a\u7ef4\u6301\u5927\u6570\u636e\u5ba2\u6237\u7684\u4f7f\u7528\u4e60\u60ef\uff0c\u901a\u8fc7 JindoFSx Namespace \u63a5\u5165 Ranger \u7684\u5ba2\u6237\u7aef\uff0c\u65b9\u4fbf\u7528\u6237\u7edf\u4e00\u7ba1\u7406\u5927\u6570\u636e\u7ec4\u4ef6\u6743\u9650\u3002","title":"\u80cc\u666f"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#_2","text":"\u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u5df2\u90e8\u7f72 JindoFSx Namespace \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx Namespace\uff0c\u8bf7\u53c2\u8003 \u5feb\u901f\u90e8\u7f72\u4e00\u4e2a\u7b80\u5355\u7684 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf(\u4ec5 Namespace) \u5b89\u88c5 Kerberos \u548c Sasl2 \u76f8\u5173\u4f9d\u8d56 \u5982\u679c\u60a8\u7684\u73af\u5883\u6ca1\u6709 Kerberos \u548c Sasl2 \u76f8\u5173\u4f9d\u8d56\uff0c\u8bf7\u5b89\u88c5\u76f8\u5173\u4f9d\u8d56 \u5b89\u88c5\u8bf4\u660e","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#1-jindofsx-namespace","text":"","title":"1. \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#11-jindo-sasl","text":"\u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 jindofsx-4.6.12/plugins/ \u8def\u5f84\u4e0b\u3002 \u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\uff0c \u5728 [jindofsx-namespace] section \u4e0b\u6dfb\u52a0\u52a8\u6001\u5e93\u7684\u7edd\u5bf9\u8def\u5f84\u3002 \u53c2\u6570 \u503c namespace.plugin.dir ${JINDDATA_HOME}/plugins/","title":"1.1 \u914d\u7f6e Jindo SASL \u63d2\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#12-ranger","text":"\u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\uff0c \u5728 [jindofsx-namespace] section \u4e0b\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.oss.authorization.method ranger \u6ce8\u610f\uff1a \u5c06 jindofsx.cfg \u914d\u7f6e\u6587\u4ef6\u90e8\u7f72\u5230 NamespaceService \u6240\u5728\u7684\u6240\u6709\u8282\u70b9\u3002","title":"1.2 \u542f\u7528 Ranger \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#13-assume-role-ak","text":"\u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 \u5728 [jindofsx-namespace] section \u4e0b\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.oss.credential.provider ASSUME_ROLE \u5728\u963f\u91cc\u4e91 RAM \u521b\u5efa\u7528\u6237\u9881\u53d1\u4e34\u65f6 AK \u7684 Role, \u9009\u62e9\u53ef\u4fe1\u5b9e\u4f53\u7c7b\u578b\u4e3a\"\u963f\u91cc\u4e91\u8d26\u53f7\", \u5e76\u4e3a\u89d2\u8272\u6388\u6743\u3002 \u5728 [jindofsx-common] section \u4e0b\u6dfb\u52a0\u6216\u66f4\u6539\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c default.credential.provider ASSUME_ROLE sts.access.key STS \u7684 accessKey sts.access.secret STS \u7684 accessKeySecret sts.access.endpoint STS \u7684 endpoint, \u5982\uff1a sts.cn-shanghai.aliyuncs.com sts.role.name \u5728 RAM \u4e2d\u521b\u5efa\u7684\u89d2\u8272\uff0c \u5982 JindoFsxTestRole sts.role.arn \u89d2\u8272\u7684ARN\uff0c\u5982\uff1aacs:ram::xxx:role/JindoFsxTestRole \u6ce8\u610f\uff1a \u5c06 jindofsx.cfg \u914d\u7f6e\u6587\u4ef6\u90e8\u7f72\u5230 NamespaceService \u6240\u5728\u7684\u6240\u6709\u8282\u70b9\u3002","title":"1.3 \u914d\u7f6e\u4f7f\u7528 Assume Role \u9881\u53d1\u4e34\u65f6 AK"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#14-principal-keytab-kerberos","text":"\u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 \u5728 [jindofsx-namespace] section \u4e0b\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.authentication.enable true namespace.authentication.keytab \u914d\u7f6e keytab \u6240\u5728\u7684\u8def\u5f84\uff0c \u5982\uff1a /tmp/jindofsx.keytab namespace.authentication.principal \u914d\u7f6e keytab \u4e2d\u7684 principal, \u5982\uff1a jindofsx/localhost","title":"1.4 \u914d\u7f6e Principal \u548c Keytab\uff08\u9002\u7528\u4e8e Kerberos \u96c6\u7fa4\uff09"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#2-jindofsx-namespace","text":"\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\uff1a cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"2. \u91cd\u542f JindoFSx Namespace \u670d\u52a1\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#31-xengine","text":"\u5728 Hadoop \u7684 core-site.xml \u4e2d \u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.xengine jindofsx","title":"3.1 \u914d\u7f6e xengine"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#32-jindo-sasl","text":"\u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 jindosdk-4.6.12/plugins/ \u8def\u5f84\u4e0b\uff0c\u5c06\u8be5\u8def\u5f84\u7684\u7edd\u5bf9\u8def\u5f84\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 \u53c2\u6570 \u503c fs.jdo.plugin.dir ${JINDOSDK_HOME}/plugins/","title":"3.2 \u914d\u7f6e Jindo SASL \u63d2\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#33-ranger","text":"\u652f\u6301\u7edf\u4e00\u914d\u7f6e\u548c\u6309 bucket \u914d\u7f6e\u4e24\u79cd\u65b9\u5f0f\u3002","title":"3.3 \u542f\u7528 Ranger \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#a-oss-oss-hdfs-bucket","text":"\u5728Hadoop \u7684 core-site.xml \u4e2d\u6dfb\u52a0\u6216\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.authorization.method ranger fs.oss.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider","title":"a. \u5bf9 OSS \u548c OSS-HDFS \u7684\u5168\u90e8 bucket \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#b-oss-oss-hdfs-bucket","text":"\u5728Hadoop \u7684 core-site.xml \u4e2d\u6dfb\u52a0\u6216\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.bucket.XXX.authorization.method ranger fs.oss.bucket.XXX.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider \u8bf4\u660e\uff1aXXX \u4e3a \u6216 OSS \u6216 OSS-HDFS \u670d\u52a1 bucket\u7684\u540d\u79f0\u3002","title":"b. \u5bf9 OSS \u6216 OSS-HDFS \u7684\u90e8\u5206 bucket \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#4-hive","text":"","title":"4. \u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#5-oss-ranger","text":"Ranger \u7ec4\u4ef6\u9700\u8981\u5b89\u88c5\u65b0\u63d2\u4ef6\u4ee5\u652f\u6301\u5bf9 OSS \u6216 OSS-HDFS \u670d\u52a1\u5bf9\u9274\u6743\u3002","title":"5. \u5b89\u88c5\u652f\u6301\u914d\u7f6e OSS \u6743\u9650\u7684 Ranger \u63d2\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#51","text":"\u6839\u636e\u96c6\u7fa4\u4e2d\u5b89\u88c5\u7684 Ranger \u7248\u672c\u4e0b\u8f7d\u5bf9\u5e94\u7684\u63d2\u4ef6\u7248\u672c\u3002 1.x \u7248\u672c \u4e0b\u8f7d\u5730\u5740 2.x \u7248\u672c \u4e0b\u8f7d\u5730\u5740 cd /xxx/ranger-admin-current/ews/webapp/WEB-INF/classes/ranger-plugins mkdir oss cp ranger-oss-plugin-1.2.0.jar /xxx/ranger-admin-current/ews/webapp/WEB-INF/classes/ranger-plugins/oss \u6ce8\u610f\uff1a\u9700\u8981\u5b89\u88c5\u5728\u90e8\u7f72 Ranger Admin \u7684\u6240\u6709\u8282\u70b9\u3002","title":"5.1 \u4e0b\u8f7d\u548c\u62f7\u8d1d\u63d2\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#52","text":"\u521b\u5efa\u5982\u4e0branger-oss.json\u6587\u4ef6\uff1a { \"id\": 500, \"name\": \"oss\", \"implClass\": \"org.apache.ranger.services.oss.RangerServiceOSS\", \"label\": \"OSS\", \"description\": \"OSS\", \"guid\": \"0d047247-bafe-4cf8-8e9b-d5d377284b2h\", \"resources\": [ { \"itemId\": 1, \"name\": \"path\", \"type\": \"path\", \"level\": 10, \"parent\": \"\", \"mandatory\": true, \"lookupSupported\": true, \"recursiveSupported\": true, \"excludesSupported\": false, \"matcher\": \"org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher\", \"matcherOptions\": { \"wildCard\": true, \"ignoreCase\": false }, \"validationRegEx\": \"\", \"validationMessage\": \"\", \"uiHint\": \"\", \"label\": \"Path\", \"description\": \"OSS Path, Should Not Start With /, exp. dir/test.txt\" } ], \"accessTypes\": [ { \"itemId\": 1, \"name\": \"read\", \"label\": \"Read\" }, { \"itemId\": 2, \"name\": \"write\", \"label\": \"Write\" }, { \"itemId\": 3, \"name\": \"execute\", \"label\": \"Execute\" } ], \"configs\": [ ], \"enums\": [ ], \"contextEnrichers\": [], \"policyConditions\": [] } \u6267\u884c\u5982\u4e0b\u547d\u4ee4\u88c5\u8f7d ranger-oss\uff0c\u6ce8\u610f\u5176\u4e2d json \u6587\u4ef6\u7684\u8def\u5f84\u548c ranger admin \u7684 url\uff0cadminUser \u9ed8\u8ba4\u4e3a admin\uff0cadminPasswd \u9ed8\u8ba4\u4e3a admin\uff1a curl -v -u${adminUser}:${adminPasswd} -X POST -H \"Accept:application/json\" -H \"Content-Type:application/json\" -d @ranger-oss.json http://emr-header-1:6080/service/plugins/definitions \u63d2\u4ef6\u6210\u529f\u5b89\u88c5\u540e\uff0c\u53ef\u4ee5\u5728 Ranger UI \u4e0a\u770b\u5230\u65b0\u589e\u9879\uff1a \u521b\u5efa\u4e00\u4e2aService\uff0c\u4f8b\u5982emr-oss\uff0c\u521b\u5efa\u5b8c\u6210\u540e\u5982\u4e0b\uff0c\u540e\u7eed\u5c31\u53ef\u4ee5\u5728\u8be5Service\u4e0b\u521b\u5efaPolicy\uff1a","title":"5.2 \u5b89\u88c5\u63d2\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#6-ranger-webui-oss-oss-hdfs","text":"","title":"6. \u5728 Ranger WebUI \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1\u6743\u9650"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#ranger","text":"\u4f8b\uff1a\u914d\u7f6e\u7528\u6237test\u62e5\u6709\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u6240\u4ee5\u6743\u9650\u7684\u6b65\u9aa4\uff1a","title":"Ranger \u89c4\u5219\u793a\u4f8b"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#a-test-ossbucket-test-hangzhouusertest-all","text":"","title":"a. \u914d\u7f6e test \u7528\u6237\u8bbf\u95eeoss://bucket-test-hangzhou/user/test\u76ee\u5f55\u7684\u8bbf\u95ee\u6743\u9650\u4e3a ALL\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#_3","text":"\u89c4\u5219\u914d\u7f6e\u9875\u9762\u4e2d\uff0c\u914d\u7f6e\u7684 path \u6ca1\u6709 oss:// \u7684\u524d\u7f00\u3002 recursive\u6309\u94ae\u4e0d\u53ef\u5173\u95ed\u3002 \u8def\u5f84\u672b\u5c3e\u65e0\u9700\u5e26\u6b63\u659c\u7ebf\uff08/\uff09\u3002","title":"\u8bf4\u660e\uff1a"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#b-ossbucket-test-hangzhouuser-execute","text":"","title":"b. \u9700\u8981\u914d\u7f6e\u8bbf\u95ee\u8def\u5f84\u7684\u7236\u76ee\u5f55oss://bucket-test-hangzhou/user\u7684\u6743\u9650\u4e3a Execute\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#7-oss","text":"\u82e5\u7528\u6237\u8bbf\u95ee Ranger \u6ca1\u6709\u6388\u6743\u7684\u8def\u5f84\uff0c\u5219\u4f1a\u62a5\u5982\u4e0b\u9519\u8bef\uff1a org.apache.hadoop.security.AccessControlException: Permission denied: user=test, access=READ_EXECUTE, resourcePath=\"bucket-test-hangzhou/\"","title":"7. \u8bbf\u95eeOSS\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#_4","text":"\u9488\u5bf9\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7684\u9274\u6743\uff0c\u76ee\u524d\u4f7f\u7528\u7684\u662f\u5bf9\u8c61\u5b58\u50a8\u7684\u9274\u6743\u65b9\u5f0f\uff08\u901a\u8fc7\u5339\u914d\u8def\u5f84\uff09\uff0c\u5bf9\u8c61\u5b58\u50a8\u6ca1\u6709\u76ee\u5f55\u6216\u6587\u4ef6\u7684 owner \u548c permission \u4fe1\u606f\uff0c\u5982\u9700\u5bf9 OSS-HDFS \u670d\u52a1\u4f7f\u7528 chmod , chown \u7b49\u64cd\u4f5c\uff0c\u53ea\u80fd\u901a\u8fc7 hadoop \u7528\u6237\u6267\u884c\u3002","title":"\u6ce8\u610f\u4e8b\u9879"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/","text":"\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u00a7 \u672c\u6587\u4ecb\u7ecd\u5728\u963f\u91cc\u4e91 E-MapReduce \u96c6\u7fa4\u914d\u7f6e\u963f\u91cc\u4e91 OSS \u548c OSS-HDFS \u670d\u52a1\u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\u3002 \u80cc\u666f \u00a7 Apache Ranger \u63d0\u4f9b\u96c6\u4e2d\u5f0f\u7684\u6743\u9650\u7ba1\u7406\u6846\u67b6\uff0c\u53ef\u4ee5\u5bf9 Hadoop \u751f\u6001\u4e2d\u7684\u591a\u4e2a\u7ec4\u4ef6\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u8bbf\u95ee\u63a7\u5236\u3002\u5f53\u7528\u6237\u5c06\u6570\u636e\u5b58\u653e\u5728\u963f\u91cc\u4e91 OSS \u65f6\uff0c\u5219\u662f\u901a\u8fc7\u963f\u91cc\u4e91 RAM \u4ea7\u54c1\u521b\u5efa\u6216\u7ba1\u7406 RAM \u7528\u6237\uff0c\u5bf9RAM\u7528\u6237\u5b9e\u73b0\u5bf9 OSS \u8d44\u6e90\u7684\u8bbf\u95ee\u63a7\u5236\u3002 \u4e3a\u7ef4\u6301\u5927\u6570\u636e\u5ba2\u6237\u7684\u4f7f\u7528\u4e60\u60ef\uff0c\u901a\u8fc7 JindoFSx Namespace \u63a5\u5165 Ranger \u7684\u5ba2\u6237\u7aef\uff0c\u65b9\u4fbf\u7528\u6237\u7edf\u4e00\u7ba1\u7406\u5927\u6570\u636e\u7ec4\u4ef6\u6743\u9650\u3002 \u8bbf\u95ee OSS \u9274\u6743\u6d41\u7a0b \u00a7 OSS \u7684\u8bbf\u95ee\u5bc6\u94a5 AccessKey\uff08AK\uff09\u7edf\u4e00\u5728 JindoFSx Namespace \u4e2d\u8bbe\u7f6e\uff0c\u907f\u514d\u7528\u6237\u5728\u5ba2\u6237\u7aef\u914d\u7f6e\u660e\u6587\u5bc6\u94a5\uff0c\u5efa\u8bae\u53ea\u5141\u8bb8\u7ba1\u7406\u5458\u64cd\u4f5c\u548c\u7ba1\u7406 JindoFSx Namespace \u670d\u52a1\uff1b JindoSDK \u63d0\u4f9b\u6807\u51c6 Hadoop Filesystem \u5ba2\u6237\u7aef\uff0c\u5c06\u8bbf\u95ee OSS \u7684\u8bf7\u6c42\u4f1a\u53d1\u9001\u81f3 Namespace \u670d\u52a1\uff1b JindoFSx Namespace Service \u8d1f\u8d23\u96c6\u6210 Ranger \u5ba2\u6237\u7aef\uff0c\u5468\u671f\u6027\u5c06\u6743\u9650\u7b56\u7565\u4ece Ranger \u670d\u52a1\u7aef\u540c\u6b65\u5230\u672c\u5730\uff1b Namespace \u670d\u52a1\u5728\u6536\u5230 JindoSDK \u7684\u9274\u6743\u8bf7\u6c42\u540e\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u6821\u9a8c\uff1b \u901a\u8fc7\u6743\u9650\u6821\u9a8c\u540e JindoSDK \u5219\u53ef\u4ee5\u4f7f\u7528 JindoFSx Namespace \u670d\u52a1\u9881\u53d1\u7684\u4e34\u65f6 AK \u8bbf\u95ee OSS\u3002 \u6ce8\uff1a\u82e5 Namespace \u90e8\u7f72\u7684\u8282\u70b9\u4e3a\u963f\u91cc\u4e91 ECS \u8282\u70b9\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u5b89\u5168\u7ec4\uff0c\u9650\u5236\u8bbf\u95ee Namespace \u7684\u5ba2\u6237\u7aef\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u521b\u5efa E-MapReduce EMR-5.6.0/EMR-3.40.0 \u6216\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\uff0c\u5e76\u4e14\u9009\u62e9\u4e86 Ranger \u670d\u52a1\u3002 \u542f\u7528 Kerberos \u8ba4\u8bc1\u6ee1\u8db3\u8ba4\u8bc1\u9700\u6c42\u3002 \u82e5\u4f7f\u7528\u96c6\u7fa4\u90fd\u662f\u53ef\u4fe1\u7528\u6237\uff0c\u5219\u53ef\u4e0d\u542f\u7528 Kerberos \u8ba4\u8bc1\u3002 \u6ce8\u610f\uff1aEMR \u96c6\u7fa4\u5f00\u542f Kerberos \u540e\uff0c\u5c06\u65e0\u6cd5\u5173\u95ed Kerberos \u8ba4\u8bc1\uff0c\u8bf7\u8c28\u614e\u9009\u62e9\u3002\u5f00\u542f Kerberos \u540e\uff0c\u96c6\u7fa4\u6240\u6709\u5927\u6570\u636e\u7ec4\u4ef6\u7684\u670d\u52a1\u5747\u9700\u8981\u7ecf\u8fc7 Kerberos \u8ba4\u8bc1\uff0c\u5927\u6570\u636e\u4f5c\u4e1a\u63d0\u4ea4\u5230\u96c6\u7fa4\u4f1a\u5148\u7ecf\u8fc7\u8eab\u4efd\u8ba4\u8bc1\u3002Kerberos \u8be6\u60c5\u4fe1\u606f\uff0c\u8bf7\u53c2\u89c1 \u9ad8\u5b89\u5168\u7c7b\u578b\u96c6\u7fa4 \u521b\u5efa\u9ad8\u5b89\u5168\u96c6\u7fa4 \u00a7 \u521b\u5efa\u96c6\u7fa4\u65f6\uff0c\u5728\u8f6f\u4ef6\u914d\u7f6e\u9875\u9762\u7684\u9ad8\u7ea7\u8bbe\u7f6e\u533a\u57df\u4e2d\uff0c\u6253\u5f00Kerberos\u96c6\u7fa4\u6a21\u5f0f\u5f00\u5173\u3002 \u82e5\u6240\u5728\u96c6\u7fa4\u90fd\u662f\u53ef\u4fe1\u7528\u6237\uff0c\u5219\u53ef\u4e0d\u542f\u7528 Kerberos \u8ba4\u8bc1\u3002 \u3010\u6ce8\u610f\u3011\uff1a\u82e5\u4e3a\u975e\u9ad8\u5b89\u5168\u96c6\u7fa4\uff0c\u5219\u4e0d\u9700\u8981\u6b65\u9aa4 5 \u548c\u6b65\u9aa4 6\u3002 1. \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u3002 \u00a7 1.1 \u914d\u7f6e Jindo SASL \u63d2\u4ef6 \u00a7 \u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 jindodata-4.6.12/plugins/ \u8def\u5f84\u4e0b\uff0c\u5c06\u8be5\u8def\u5f84\u7684\u7edd\u5bf9\u8def\u5f84\u6dfb\u52a0\u5230 JindoData \u7684 namespace \u4e2d\u3002 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.plugin.dir ${JINDDATA_HOME}/plugins/ 1.2 \u542f\u7528 Ranger \u9274\u6743 \u00a7 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.oss.authorization.method ranger 2. \u91cd\u542f JindoFSx Namespace \u670d\u52a1\u3002 \u00a7 3. \u914d\u7f6e JindoSDK\u3002 \u00a7 3.1 \u914d\u7f6e xengine \u00a7 \u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.xengine jindofsx 3.2 \u914d\u7f6e Jindo SASL \u63d2\u4ef6 \u00a7 \u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 jindosdk-4.6.12/plugins/ \u8def\u5f84\u4e0b\uff0c\u5c06\u8be5\u8def\u5f84\u7684\u7edd\u5bf9\u8def\u5f84\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 \u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.jdo.plugin.dir ${JINDOSDK_HOME}/plugins/ 3.3 \u542f\u7528 Ranger \u9274\u6743 \u00a7 \u652f\u6301\u7edf\u4e00\u914d\u7f6e\u548c\u6309 bucket \u914d\u7f6e\u4e24\u79cd\u65b9\u5f0f\u3002 a. \u5bf9 OSS \u548c OSS-HDFS \u7684\u5168\u90e8 bucket \u9274\u6743 \u00a7 \u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.authorization.method ranger \u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site \uff0c\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider b. \u5bf9 OSS \u6216 OSS-HDFS \u7684\u90e8\u5206 bucket \u9274\u6743 \u00a7 \u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.bucket.XXX.authorization.method ranger \u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site \uff0c\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.bucket.XXX.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider \u8bf4\u660e\uff1aXXX \u4e3a \u6216 OSS \u6216 OSS-HDFS \u670d\u52a1 bucket\u7684\u540d\u79f0\u3002 4. \u91cd\u542fHiveServer2\u3002 \u00a7 5. \u521b\u5efa\u7528\u6237 Principal\u3002\uff08\u9002\u7528\u4e8e\u9ad8\u5b89\u5168\u96c6\u7fa4\uff09 \u00a7 a. \u901a\u8fc7SSH\u65b9\u5f0f\u8fde\u63a5\u96c6\u7fa4\u7684emr-header-1\u8282\u70b9\u3002 \u00a7 b. \u6267\u884c\u5982\u4e0b\u547d\u4ee4\uff0c\u8fdb\u5165Kerberos\u7684admin\u5de5\u5177\u3002 \u00a7 sh /usr/lib/has-current/bin/admin-local.sh /etc/ecm/has-conf -k /etc/ecm/has-conf/admin.keytab \u672c\u793a\u4f8b\u5bc6\u7801\u8bbe\u7f6e\u4e3a 123456\u3002 addprinc -pw 123456 test \u8bf4\u660e: \u9700\u8981\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5728\u521b\u5efaTGT\u65f6\u4f1a\u7528\u5230\u3002\u5982\u679c\u60a8\u4e0d\u60f3\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5219\u53ef\u4ee5\u6267\u884c\u4e0b\u4e00\u6b65\uff0c\u628aPrincipal\u7684\u7528\u6237\u540d\u548c\u5bc6\u7801\u5bfc\u5165\u5230keytab\u6587\u4ef6\u4e2d\u3002 \u00a7 c. \u53ef\u9009\uff1a\u6267\u884c\u5982\u4e0b\u547d\u4ee4\uff0c\u751f\u6210keytab\u6587\u4ef6\u3002 \u00a7 ktadd -k /root/test.keytab test \u6267\u884c quit \u547d\u4ee4\uff0c\u53ef\u4ee5\u9000\u51fa Kerberos \u7684 admin \u5de5\u5177\u3002 6. \u521b\u5efa TGT\u3002\uff08\u9002\u7528\u4e8e\u9ad8\u5b89\u5168\u96c6\u7fa4\uff09 \u00a7 \u521b\u5efa TGT \u7684\u673a\u5668\uff0c\u53ef\u4ee5\u662f\u4efb\u610f\u4e00\u53f0\u9700\u8981\u8bbf\u95ee OSS \u7684\u673a\u5668\u3002 a. \u4f7f\u7528 root \u7528\u6237\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u521b\u5efa test \u7528\u6237\u3002 \u00a7 useradd test b. \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u5207\u6362\u4e3a test \u7528\u6237\u3002 \u00a7 su test c. \u751f\u6210TGT\u3002 \u00a7 \u65b9\u5f0f\u4e00\uff1a\u4f7f\u7528\u7528\u6237\u540d\u548c\u5bc6\u7801\u65b9\u5f0f\uff0c\u521b\u5efa TGT\u3002 \u6267\u884c kinit \u547d\u4ee4\uff0c\u56de\u8f66\u540e\u8f93\u5165 test \u7684\u5bc6\u7801 123456\u3002 \u65b9\u5f0f\u4e8c\uff1a\u4f7f\u7528 keytab \u6587\u4ef6\uff0c\u521b\u5efa TGT\u3002 \u5728\u6b65\u9aa4 4 \u4e2d\u7684 test.keytab \u6587\u4ef6\uff0c\u5df2\u7ecf\u4fdd\u5b58\u5728 emr-header-1 \u673a\u5668\u7684 /root/ \u76ee\u5f55\u4e0b\uff0c\u9700\u8981\u4f7f\u7528 scp \u547d\u4ee4\u62f7\u8d1d\u5230\u5f53\u524d\u673a\u5668\u7684 /home/test/ \u76ee\u5f55\u4e0b\u3002 kinit -kt /home/test/test.keytab test d. \u67e5\u770bTGT\u3002 \u00a7 \u4f7f\u7528 klist \u547d\u4ee4\uff0c\u5982\u679c\u51fa\u73b0\u5982\u4e0b\u4fe1\u606f\uff0c\u5219\u8bf4\u660eTGT\u521b\u5efa\u6210\u529f\uff0c\u5373\u53ef\u4ee5\u8bbf\u95eeOSS\u4e86\u3002 Ticket cache: FILE:/tmp/krb5cc_1012 Default principal: test@EMR.23****.COM Valid starting Expires Service principal 03/27/2022 23:20:44 03/28/2022 23:20:44 krbtgt/EMR.238075.COM@EMR.238075.COM renew until 03/28/2022 23:20:44 \u6ce8\u610f: \u9700\u8981\u8bb0\u5f55\u4e0b\u56de\u663e\u4fe1\u606f EMR.23****.COM \u4e2d\u7684\u6570\u5b57 23**** \uff0c\u5373\u4e3a cluster_id \u7684\u503c\uff0c\u540e\u9762\u8bbf\u95ee OSS \u65f6\u9700\u8981\u3002 7. \u5728 Ranger WebUI \u914d\u7f6e OSS \u6743\u9650 \u00a7 Ranger \u89c4\u5219\u793a\u4f8b \u00a7 \u4f8b\uff1a\u914d\u7f6e\u7528\u6237test\u62e5\u6709\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u6240\u4ee5\u6743\u9650\u7684\u6b65\u9aa4\uff1a a. \u914d\u7f6e test \u7528\u6237\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u8bbf\u95ee\u6743\u9650\u4e3a ALL\u3002 \u00a7 \u8bf4\u660e\uff1a \u00a7 \u89c4\u5219\u914d\u7f6e\u9875\u9762\u4e2d\uff0c\u914d\u7f6e\u7684 path \u6ca1\u6709 oss:// \u7684\u524d\u7f00\u3002 recursive\u6309\u94ae\u4e0d\u53ef\u5173\u95ed\u3002 \u8def\u5f84\u672b\u5c3e\u65e0\u9700\u5e26\u6b63\u659c\u7ebf\uff08/\uff09\u3002 b. \u9700\u8981\u914d\u7f6e\u8bbf\u95ee\u8def\u5f84\u7684\u7236\u76ee\u5f55 oss://bucket-test-hangzhou/user \u7684\u6743\u9650\u4e3a Execute\u3002 \u00a7 8. \u8bbf\u95eeOSS\u3002 \u00a7 \u82e5\u7528\u6237\u8bbf\u95ee Ranger \u6ca1\u6709\u6388\u6743\u7684\u8def\u5f84\uff0c\u5219\u4f1a\u62a5\u5982\u4e0b\u9519\u8bef\uff1a org.apache.hadoop.security.AccessControlException: Permission denied: user=test, access=READ_EXECUTE, resourcePath=\"bucket-test-hangzhou/\"","title":"\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#oss-ranger","text":"\u672c\u6587\u4ecb\u7ecd\u5728\u963f\u91cc\u4e91 E-MapReduce \u96c6\u7fa4\u914d\u7f6e\u963f\u91cc\u4e91 OSS \u548c OSS-HDFS \u670d\u52a1\u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\u3002","title":"\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#_1","text":"Apache Ranger \u63d0\u4f9b\u96c6\u4e2d\u5f0f\u7684\u6743\u9650\u7ba1\u7406\u6846\u67b6\uff0c\u53ef\u4ee5\u5bf9 Hadoop \u751f\u6001\u4e2d\u7684\u591a\u4e2a\u7ec4\u4ef6\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u8bbf\u95ee\u63a7\u5236\u3002\u5f53\u7528\u6237\u5c06\u6570\u636e\u5b58\u653e\u5728\u963f\u91cc\u4e91 OSS \u65f6\uff0c\u5219\u662f\u901a\u8fc7\u963f\u91cc\u4e91 RAM \u4ea7\u54c1\u521b\u5efa\u6216\u7ba1\u7406 RAM \u7528\u6237\uff0c\u5bf9RAM\u7528\u6237\u5b9e\u73b0\u5bf9 OSS \u8d44\u6e90\u7684\u8bbf\u95ee\u63a7\u5236\u3002 \u4e3a\u7ef4\u6301\u5927\u6570\u636e\u5ba2\u6237\u7684\u4f7f\u7528\u4e60\u60ef\uff0c\u901a\u8fc7 JindoFSx Namespace \u63a5\u5165 Ranger \u7684\u5ba2\u6237\u7aef\uff0c\u65b9\u4fbf\u7528\u6237\u7edf\u4e00\u7ba1\u7406\u5927\u6570\u636e\u7ec4\u4ef6\u6743\u9650\u3002","title":"\u80cc\u666f"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#oss","text":"OSS \u7684\u8bbf\u95ee\u5bc6\u94a5 AccessKey\uff08AK\uff09\u7edf\u4e00\u5728 JindoFSx Namespace \u4e2d\u8bbe\u7f6e\uff0c\u907f\u514d\u7528\u6237\u5728\u5ba2\u6237\u7aef\u914d\u7f6e\u660e\u6587\u5bc6\u94a5\uff0c\u5efa\u8bae\u53ea\u5141\u8bb8\u7ba1\u7406\u5458\u64cd\u4f5c\u548c\u7ba1\u7406 JindoFSx Namespace \u670d\u52a1\uff1b JindoSDK \u63d0\u4f9b\u6807\u51c6 Hadoop Filesystem \u5ba2\u6237\u7aef\uff0c\u5c06\u8bbf\u95ee OSS \u7684\u8bf7\u6c42\u4f1a\u53d1\u9001\u81f3 Namespace \u670d\u52a1\uff1b JindoFSx Namespace Service \u8d1f\u8d23\u96c6\u6210 Ranger \u5ba2\u6237\u7aef\uff0c\u5468\u671f\u6027\u5c06\u6743\u9650\u7b56\u7565\u4ece Ranger \u670d\u52a1\u7aef\u540c\u6b65\u5230\u672c\u5730\uff1b Namespace \u670d\u52a1\u5728\u6536\u5230 JindoSDK \u7684\u9274\u6743\u8bf7\u6c42\u540e\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u6821\u9a8c\uff1b \u901a\u8fc7\u6743\u9650\u6821\u9a8c\u540e JindoSDK \u5219\u53ef\u4ee5\u4f7f\u7528 JindoFSx Namespace \u670d\u52a1\u9881\u53d1\u7684\u4e34\u65f6 AK \u8bbf\u95ee OSS\u3002 \u6ce8\uff1a\u82e5 Namespace \u90e8\u7f72\u7684\u8282\u70b9\u4e3a\u963f\u91cc\u4e91 ECS \u8282\u70b9\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u5b89\u5168\u7ec4\uff0c\u9650\u5236\u8bbf\u95ee Namespace \u7684\u5ba2\u6237\u7aef\u3002","title":"\u8bbf\u95ee OSS \u9274\u6743\u6d41\u7a0b"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#_2","text":"\u5df2\u521b\u5efa E-MapReduce EMR-5.6.0/EMR-3.40.0 \u6216\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\uff0c\u5e76\u4e14\u9009\u62e9\u4e86 Ranger \u670d\u52a1\u3002 \u542f\u7528 Kerberos \u8ba4\u8bc1\u6ee1\u8db3\u8ba4\u8bc1\u9700\u6c42\u3002 \u82e5\u4f7f\u7528\u96c6\u7fa4\u90fd\u662f\u53ef\u4fe1\u7528\u6237\uff0c\u5219\u53ef\u4e0d\u542f\u7528 Kerberos \u8ba4\u8bc1\u3002 \u6ce8\u610f\uff1aEMR \u96c6\u7fa4\u5f00\u542f Kerberos \u540e\uff0c\u5c06\u65e0\u6cd5\u5173\u95ed Kerberos \u8ba4\u8bc1\uff0c\u8bf7\u8c28\u614e\u9009\u62e9\u3002\u5f00\u542f Kerberos \u540e\uff0c\u96c6\u7fa4\u6240\u6709\u5927\u6570\u636e\u7ec4\u4ef6\u7684\u670d\u52a1\u5747\u9700\u8981\u7ecf\u8fc7 Kerberos \u8ba4\u8bc1\uff0c\u5927\u6570\u636e\u4f5c\u4e1a\u63d0\u4ea4\u5230\u96c6\u7fa4\u4f1a\u5148\u7ecf\u8fc7\u8eab\u4efd\u8ba4\u8bc1\u3002Kerberos \u8be6\u60c5\u4fe1\u606f\uff0c\u8bf7\u53c2\u89c1 \u9ad8\u5b89\u5168\u7c7b\u578b\u96c6\u7fa4","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#_3","text":"\u521b\u5efa\u96c6\u7fa4\u65f6\uff0c\u5728\u8f6f\u4ef6\u914d\u7f6e\u9875\u9762\u7684\u9ad8\u7ea7\u8bbe\u7f6e\u533a\u57df\u4e2d\uff0c\u6253\u5f00Kerberos\u96c6\u7fa4\u6a21\u5f0f\u5f00\u5173\u3002 \u82e5\u6240\u5728\u96c6\u7fa4\u90fd\u662f\u53ef\u4fe1\u7528\u6237\uff0c\u5219\u53ef\u4e0d\u542f\u7528 Kerberos \u8ba4\u8bc1\u3002 \u3010\u6ce8\u610f\u3011\uff1a\u82e5\u4e3a\u975e\u9ad8\u5b89\u5168\u96c6\u7fa4\uff0c\u5219\u4e0d\u9700\u8981\u6b65\u9aa4 5 \u548c\u6b65\u9aa4 6\u3002","title":"\u521b\u5efa\u9ad8\u5b89\u5168\u96c6\u7fa4"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#1-jindofsx-namespace","text":"","title":"1. \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#11-jindo-sasl","text":"\u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 jindodata-4.6.12/plugins/ \u8def\u5f84\u4e0b\uff0c\u5c06\u8be5\u8def\u5f84\u7684\u7edd\u5bf9\u8def\u5f84\u6dfb\u52a0\u5230 JindoData \u7684 namespace \u4e2d\u3002 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.plugin.dir ${JINDDATA_HOME}/plugins/","title":"1.1 \u914d\u7f6e Jindo SASL \u63d2\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#12-ranger","text":"\u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.oss.authorization.method ranger","title":"1.2 \u542f\u7528 Ranger \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#2-jindofsx-namespace","text":"","title":"2. \u91cd\u542f JindoFSx Namespace \u670d\u52a1\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#31-xengine","text":"\u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.xengine jindofsx","title":"3.1 \u914d\u7f6e xengine"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#32-jindo-sasl","text":"\u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 jindosdk-4.6.12/plugins/ \u8def\u5f84\u4e0b\uff0c\u5c06\u8be5\u8def\u5f84\u7684\u7edd\u5bf9\u8def\u5f84\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 \u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.jdo.plugin.dir ${JINDOSDK_HOME}/plugins/","title":"3.2 \u914d\u7f6e Jindo SASL \u63d2\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#33-ranger","text":"\u652f\u6301\u7edf\u4e00\u914d\u7f6e\u548c\u6309 bucket \u914d\u7f6e\u4e24\u79cd\u65b9\u5f0f\u3002","title":"3.3 \u542f\u7528 Ranger \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#a-oss-oss-hdfs-bucket","text":"\u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.authorization.method ranger \u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site \uff0c\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider","title":"a. \u5bf9 OSS \u548c OSS-HDFS \u7684\u5168\u90e8 bucket \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#b-oss-oss-hdfs-bucket","text":"\u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.bucket.XXX.authorization.method ranger \u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site \uff0c\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.bucket.XXX.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider \u8bf4\u660e\uff1aXXX \u4e3a \u6216 OSS \u6216 OSS-HDFS \u670d\u52a1 bucket\u7684\u540d\u79f0\u3002","title":"b. \u5bf9 OSS \u6216 OSS-HDFS \u7684\u90e8\u5206 bucket \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#4-hiveserver2","text":"","title":"4. \u91cd\u542fHiveServer2\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#5-principal","text":"","title":"5. \u521b\u5efa\u7528\u6237 Principal\u3002\uff08\u9002\u7528\u4e8e\u9ad8\u5b89\u5168\u96c6\u7fa4\uff09"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#a-sshemr-header-1","text":"","title":"a. \u901a\u8fc7SSH\u65b9\u5f0f\u8fde\u63a5\u96c6\u7fa4\u7684emr-header-1\u8282\u70b9\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#b-kerberosadmin","text":"sh /usr/lib/has-current/bin/admin-local.sh /etc/ecm/has-conf -k /etc/ecm/has-conf/admin.keytab \u672c\u793a\u4f8b\u5bc6\u7801\u8bbe\u7f6e\u4e3a 123456\u3002 addprinc -pw 123456 test","title":"b. \u6267\u884c\u5982\u4e0b\u547d\u4ee4\uff0c\u8fdb\u5165Kerberos\u7684admin\u5de5\u5177\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#tgtprincipalkeytab","text":"","title":"\u8bf4\u660e: \u9700\u8981\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5728\u521b\u5efaTGT\u65f6\u4f1a\u7528\u5230\u3002\u5982\u679c\u60a8\u4e0d\u60f3\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5219\u53ef\u4ee5\u6267\u884c\u4e0b\u4e00\u6b65\uff0c\u628aPrincipal\u7684\u7528\u6237\u540d\u548c\u5bc6\u7801\u5bfc\u5165\u5230keytab\u6587\u4ef6\u4e2d\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#c-keytab","text":"ktadd -k /root/test.keytab test \u6267\u884c quit \u547d\u4ee4\uff0c\u53ef\u4ee5\u9000\u51fa Kerberos \u7684 admin \u5de5\u5177\u3002","title":"c. \u53ef\u9009\uff1a\u6267\u884c\u5982\u4e0b\u547d\u4ee4\uff0c\u751f\u6210keytab\u6587\u4ef6\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#6-tgt","text":"\u521b\u5efa TGT \u7684\u673a\u5668\uff0c\u53ef\u4ee5\u662f\u4efb\u610f\u4e00\u53f0\u9700\u8981\u8bbf\u95ee OSS \u7684\u673a\u5668\u3002","title":"6. \u521b\u5efa TGT\u3002\uff08\u9002\u7528\u4e8e\u9ad8\u5b89\u5168\u96c6\u7fa4\uff09"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#a-root-test","text":"useradd test","title":"a. \u4f7f\u7528 root \u7528\u6237\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u521b\u5efa test \u7528\u6237\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#b-test","text":"su test","title":"b. \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u5207\u6362\u4e3a test \u7528\u6237\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#c-tgt","text":"\u65b9\u5f0f\u4e00\uff1a\u4f7f\u7528\u7528\u6237\u540d\u548c\u5bc6\u7801\u65b9\u5f0f\uff0c\u521b\u5efa TGT\u3002 \u6267\u884c kinit \u547d\u4ee4\uff0c\u56de\u8f66\u540e\u8f93\u5165 test \u7684\u5bc6\u7801 123456\u3002 \u65b9\u5f0f\u4e8c\uff1a\u4f7f\u7528 keytab \u6587\u4ef6\uff0c\u521b\u5efa TGT\u3002 \u5728\u6b65\u9aa4 4 \u4e2d\u7684 test.keytab \u6587\u4ef6\uff0c\u5df2\u7ecf\u4fdd\u5b58\u5728 emr-header-1 \u673a\u5668\u7684 /root/ \u76ee\u5f55\u4e0b\uff0c\u9700\u8981\u4f7f\u7528 scp \u547d\u4ee4\u62f7\u8d1d\u5230\u5f53\u524d\u673a\u5668\u7684 /home/test/ \u76ee\u5f55\u4e0b\u3002 kinit -kt /home/test/test.keytab test","title":"c. \u751f\u6210TGT\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#d-tgt","text":"\u4f7f\u7528 klist \u547d\u4ee4\uff0c\u5982\u679c\u51fa\u73b0\u5982\u4e0b\u4fe1\u606f\uff0c\u5219\u8bf4\u660eTGT\u521b\u5efa\u6210\u529f\uff0c\u5373\u53ef\u4ee5\u8bbf\u95eeOSS\u4e86\u3002 Ticket cache: FILE:/tmp/krb5cc_1012 Default principal: test@EMR.23****.COM Valid starting Expires Service principal 03/27/2022 23:20:44 03/28/2022 23:20:44 krbtgt/EMR.238075.COM@EMR.238075.COM renew until 03/28/2022 23:20:44 \u6ce8\u610f: \u9700\u8981\u8bb0\u5f55\u4e0b\u56de\u663e\u4fe1\u606f EMR.23****.COM \u4e2d\u7684\u6570\u5b57 23**** \uff0c\u5373\u4e3a cluster_id \u7684\u503c\uff0c\u540e\u9762\u8bbf\u95ee OSS \u65f6\u9700\u8981\u3002","title":"d. \u67e5\u770bTGT\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#7-ranger-webui-oss","text":"","title":"7. \u5728 Ranger WebUI \u914d\u7f6e OSS \u6743\u9650"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#ranger","text":"\u4f8b\uff1a\u914d\u7f6e\u7528\u6237test\u62e5\u6709\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u6240\u4ee5\u6743\u9650\u7684\u6b65\u9aa4\uff1a","title":"Ranger \u89c4\u5219\u793a\u4f8b"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#a-test-ossbucket-test-hangzhouusertest-all","text":"","title":"a. \u914d\u7f6e test \u7528\u6237\u8bbf\u95eeoss://bucket-test-hangzhou/user/test\u76ee\u5f55\u7684\u8bbf\u95ee\u6743\u9650\u4e3a ALL\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#_4","text":"\u89c4\u5219\u914d\u7f6e\u9875\u9762\u4e2d\uff0c\u914d\u7f6e\u7684 path \u6ca1\u6709 oss:// \u7684\u524d\u7f00\u3002 recursive\u6309\u94ae\u4e0d\u53ef\u5173\u95ed\u3002 \u8def\u5f84\u672b\u5c3e\u65e0\u9700\u5e26\u6b63\u659c\u7ebf\uff08/\uff09\u3002","title":"\u8bf4\u660e\uff1a"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#b-ossbucket-test-hangzhouuser-execute","text":"","title":"b. \u9700\u8981\u914d\u7f6e\u8bbf\u95ee\u8def\u5f84\u7684\u7236\u76ee\u5f55oss://bucket-test-hangzhou/user\u7684\u6743\u9650\u4e3a Execute\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#8-oss","text":"\u82e5\u7528\u6237\u8bbf\u95ee Ranger \u6ca1\u6709\u6388\u6743\u7684\u8def\u5f84\uff0c\u5219\u4f1a\u62a5\u5982\u4e0b\u9519\u8bef\uff1a org.apache.hadoop.security.AccessControlException: Permission denied: user=test, access=READ_EXECUTE, resourcePath=\"bucket-test-hangzhou/\"","title":"8. \u8bbf\u95eeOSS\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/","text":"EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u00a7 \u672c\u6587\u4ecb\u7ecd\u5728 EMR \u96c6\u7fa4\u914d\u7f6e\u963f\u91cc\u4e91 OSS \u548c OSS-HDFS \u670d\u52a1\u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\u3002 \u80cc\u666f \u00a7 Apache Ranger \u63d0\u4f9b\u96c6\u4e2d\u5f0f\u7684\u6743\u9650\u7ba1\u7406\u6846\u67b6\uff0c\u53ef\u4ee5\u5bf9 Hadoop \u751f\u6001\u4e2d\u7684\u591a\u4e2a\u7ec4\u4ef6\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u8bbf\u95ee\u63a7\u5236\u3002\u5f53\u7528\u6237\u5c06\u6570\u636e\u5b58\u653e\u5728\u963f\u91cc\u4e91 OSS \u65f6\uff0c\u5219\u662f\u901a\u8fc7\u963f\u91cc\u4e91 RAM \u4ea7\u54c1\u521b\u5efa\u6216\u7ba1\u7406 RAM \u7528\u6237\uff0c\u5bf9RAM\u7528\u6237\u5b9e\u73b0\u5bf9 OSS \u8d44\u6e90\u7684\u8bbf\u95ee\u63a7\u5236\u3002 \u4e3a\u7ef4\u6301\u5927\u6570\u636e\u5ba2\u6237\u7684\u4f7f\u7528\u4e60\u60ef\uff0c\u901a\u8fc7 JindoFSx Namespace \u63a5\u5165 Ranger \u7684\u5ba2\u6237\u7aef\uff0c\u65b9\u4fbf\u7528\u6237\u7edf\u4e00\u7ba1\u7406\u5927\u6570\u636e\u7ec4\u4ef6\u6743\u9650\u3002 \u8bbf\u95ee OSS \u9274\u6743\u6d41\u7a0b \u00a7 OSS \u7684\u8bbf\u95ee\u5bc6\u94a5 AccessKey\uff08AK\uff09\u7edf\u4e00\u5728 JindoFSx Namespace \u4e2d\u8bbe\u7f6e\uff0c\u907f\u514d\u7528\u6237\u5728\u5ba2\u6237\u7aef\u914d\u7f6e\u660e\u6587\u5bc6\u94a5\uff0c\u5efa\u8bae\u53ea\u5141\u8bb8\u7ba1\u7406\u5458\u64cd\u4f5c\u548c\u7ba1\u7406 JindoFSx Namespace \u670d\u52a1\uff1b JindoSDK \u63d0\u4f9b\u6807\u51c6 Hadoop Filesystem \u5ba2\u6237\u7aef\uff0c\u5c06\u8bbf\u95ee OSS \u7684\u8bf7\u6c42\u4f1a\u53d1\u9001\u81f3 Namespace \u670d\u52a1\uff1b JindoFSx Namespace Service \u8d1f\u8d23\u96c6\u6210 Ranger \u5ba2\u6237\u7aef\uff0c\u5468\u671f\u6027\u5c06\u6743\u9650\u7b56\u7565\u4ece Ranger \u670d\u52a1\u7aef\u540c\u6b65\u5230\u672c\u5730\uff1b Namespace \u670d\u52a1\u5728\u6536\u5230 JindoSDK \u7684\u9274\u6743\u8bf7\u6c42\u540e\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u6821\u9a8c\uff1b \u901a\u8fc7\u6743\u9650\u6821\u9a8c\u540e JindoSDK \u5219\u53ef\u4ee5\u4f7f\u7528 JindoFSx Namespace \u670d\u52a1\u9881\u53d1\u7684\u4e34\u65f6 AK \u8bbf\u95ee OSS\u3002 \u6ce8\uff1a\u82e5 Namespace \u90e8\u7f72\u7684\u8282\u70b9\u4e3a\u963f\u91cc\u4e91 ECS \u8282\u70b9\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u5b89\u5168\u7ec4\uff0c\u9650\u5236\u8bbf\u95ee Namespace \u7684\u5ba2\u6237\u7aef\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u521b\u5efa E-MapReduce EMR-5.9.1/EMR-3.43.1 \u6216\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\uff0c\u5e76\u4e14\u9009\u62e9\u4e86 Ranger \u670d\u52a1\u3002 \u542f\u7528 Kerberos \u8ba4\u8bc1\u6ee1\u8db3\u8ba4\u8bc1\u9700\u6c42\u3002 \u82e5\u4f7f\u7528\u96c6\u7fa4\u90fd\u662f\u53ef\u4fe1\u7528\u6237\uff0c\u5219\u53ef\u4e0d\u542f\u7528 Kerberos \u8ba4\u8bc1\u3002 \u6ce8\u610f\uff1aEMR \u96c6\u7fa4\u5f00\u542f Kerberos \u540e\uff0c\u5c06\u65e0\u6cd5\u5173\u95ed Kerberos \u8ba4\u8bc1\uff0c\u8bf7\u8c28\u614e\u9009\u62e9\u3002\u5f00\u542f Kerberos \u540e\uff0c\u96c6\u7fa4\u6240\u6709\u5927\u6570\u636e\u7ec4\u4ef6\u7684\u670d\u52a1\u5747\u9700\u8981\u7ecf\u8fc7 Kerberos \u8ba4\u8bc1\uff0c\u5927\u6570\u636e\u4f5c\u4e1a\u63d0\u4ea4\u5230\u96c6\u7fa4\u4f1a\u5148\u7ecf\u8fc7\u8eab\u4efd\u8ba4\u8bc1\u3002Kerberos \u8be6\u60c5\u4fe1\u606f\uff0c\u8bf7\u53c2\u89c1 \u9ad8\u5b89\u5168\u7c7b\u578b\u96c6\u7fa4 1. \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u3002 \u00a7 1.1 \u914d\u7f6e Jindo SASL \u63d2\u4ef6 \u00a7 \u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 /opt/apps/JINDODATA/jindodata-current/plugins \u8def\u5f84\u4e0b\u3002 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.plugin.dir /opt/apps/JINDODATA/jindodata-current/plugins 1.2 \u542f\u7528 Ranger \u9274\u6743 \u00a7 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.oss.authorization.method ranger 1.3 \u914d\u7f6e\u4f7f\u7528 Assume Role \u9881\u53d1\u4e34\u65f6 AK\uff08\u53ef\u9009\uff0c\u9ed8\u8ba4\u4f7f\u7528 ECS_ROLE \u514d\u5bc6\uff09 \u00a7 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.oss.credential.provider ASSUME_ROLE \u5728\u963f\u91cc\u4e91 RAM \u521b\u5efa\u7528\u6237\u9881\u53d1\u4e34\u65f6 AK \u7684 Role, \u9009\u62e9\u53ef\u4fe1\u5b9e\u4f53\u7c7b\u578b\u4e3a\"\u963f\u91cc\u4e91\u8d26\u53f7\", \u5e76\u4e3a\u89d2\u8272\u6388\u6743\u3002 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > common > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c default.credential.provider ASSUME_ROLE sts.access.key STS \u7684 accessKey sts.access.secret STS \u7684 accessKeySecret sts.access.endpoint STS \u7684 endpoint, \u5982\uff1a sts.cn-shanghai.aliyuncs.com sts.role.name \u5728 RAM \u4e2d\u521b\u5efa\u7684\u89d2\u8272\uff0c \u5982 JindoFsxTestRole sts.role.arn \u89d2\u8272\u7684ARN\uff0c\u5982\uff1aacs:ram::xxx:role/JindoFsxTestRole 1.4 \u914d\u7f6e Principal \u548c Keytab\uff08\u9002\u7528\u4e8e Kerberos \u96c6\u7fa4\uff09 \u00a7 a. \u521b\u5efa principal \u548c keytab \u00a7 \u5982\u679c\u60a8\u662f\u4f7f\u7528root\u7528\u6237\uff0c\u767b\u5f55KDC\uff08Kerberos\u7684\u670d\u52a1\u7aef\u7a0b\u5e8f\uff09\u6240\u5728\u7684master-1-1\u8282\u70b9\uff0c\u5219\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u76f4\u63a5\u8fdb\u5165admin\u5de5\u5177\u3002 kadmin.local \u5f53\u8fd4\u56de\u4fe1\u606f\u4e2d\u5305\u542b\u5982\u4e0b\u4fe1\u606f\u65f6\uff0c\u8868\u793a\u5df2\u8fdb\u5165admin.local\u547d\u4ee4\u884c\u3002 Authenticating as principal hadoop/admin@EMR.C-85D4B8D74296****.COM with password. kadmin.local: \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u521b\u5efa\u7528\u6237\u540d\u4e3a jindodata \u7684 principal, \u683c\u5f0f\u4e3a jindodata/ . .emr.aliyuncs.com\u3002 addprinc -randkey jindodata/master-1-1.c-9e4fc2dcc****.cn-shanghai.emr.aliyuncs.com \u5bfc\u51fa keytab \u6587\u4ef6\u3002 ktadd -k jindodata.keytab jindodata/master-1-1.c-9e4fc2dcc****.cn-shanghai.emr.aliyuncs.com b. \u914d\u7f6e \u00a7 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.authentication.enable true namespace.authentication.keytab \u914d\u7f6e keytab \u6240\u5728\u7684\u8def\u5f84\uff0c \u5982\uff1a /opt/apps/JINDODATA/jindodata-current/conf/jindodata.keytab namespace.authentication.principal \u914d\u7f6e keytab \u4e2d\u7684 principal, \u5982\uff1a jindodata/master-1-1.c-9e4fc2dcc *.cn-shanghai.emr.aliyuncs.com@EMR.C-85D4B8D74296 *.COM 1.5 \u914d\u7f6e Ranger Admin Service URL \u00a7 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c ranger.plugin.jindo-auth.policy.rest.url \u914d\u7f6e\u83b7\u53d6 Ranger Policy \u7684 URL, \u5982\uff1ahttp://master-1-1:6080\uff0c \u9ed8\u8ba4\u4e3a http://emr-header-1:6080 1.6 \u91cd\u542f JindoFSx Namespace \u670d\u52a1\u3002 \u00a7 2. \u914d\u7f6e JindoSDK\u3002 \u00a7 2.1 \u914d\u7f6e xengine \u00a7 \u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \uff0c \u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.xengine jindofsx 2.2 \u914d\u7f6e Jindo SASL \u63d2\u4ef6 \u00a7 \u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 /opt/apps/JINDOSDK/jindosdk-current/plugins/ \u8def\u5f84\u4e0b\uff0c \u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c \u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.jdo.plugin.dir /opt/apps/JINDOSDK/jindosdk-current/plugins 2.3 \u542f\u7528 Ranger \u9274\u6743 \u00a7 \u652f\u6301\u7edf\u4e00\u914d\u7f6e\u548c\u6309 bucket \u914d\u7f6e\u4e24\u79cd\u65b9\u5f0f\u3002 a. \u5bf9 OSS \u548c OSS-HDFS \u7684\u5168\u90e8 bucket \u9274\u6743 \u00a7 \u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \uff0c \u65b0\u589e\u6216\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.authorization.method ranger fs.oss.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider b. \u5bf9 OSS \u6216 OSS-HDFS \u7684\u90e8\u5206 bucket \u9274\u6743 \u00a7 \u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \uff0c \u65b0\u589e\u6216\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.bucket.XXX.authorization.method ranger fs.oss.bucket.XXX.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider \u8bf4\u660e\uff1aXXX \u4e3a \u6216 OSS \u6216 OSS-HDFS \u670d\u52a1 bucket\u7684\u540d\u79f0\u3002 2.4. \u91cd\u542f HiveServer2 \u7b49\u6240\u6709\u5e38\u9a7b\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 3. \u521b\u5efa\u7528\u6237 Principal\u3002 \u00a7 \u5982\u679c\u60a8\u662f\u4f7f\u7528root\u7528\u6237\uff0c\u767b\u5f55KDC\uff08Kerberos\u7684\u670d\u52a1\u7aef\u7a0b\u5e8f\uff09\u6240\u5728\u7684 master-1-1 \u8282\u70b9\uff0c\u5219\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u76f4\u63a5\u8fdb\u5165admin\u5de5\u5177\u3002 kadmin.local \u5f53\u8fd4\u56de\u4fe1\u606f\u4e2d\u5305\u542b\u5982\u4e0b\u4fe1\u606f\u65f6\uff0c\u8868\u793a\u5df2\u8fdb\u5165admin.local\u547d\u4ee4\u884c\u3002 Authenticating as principal hadoop/admin@EMR.C-85D4B8D74296****.COM with password. kadmin.local: \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u521b\u5efa\u7528\u6237\u540d\u4e3a test \u7684 principal\u3002 addprinc -pw 123456 test 4. \u521b\u5efa TGT\u3002 \u00a7 \u521b\u5efa TGT \u7684\u673a\u5668\uff0c\u53ef\u4ee5\u662f\u4efb\u610f\u4e00\u53f0\u9700\u8981\u8bbf\u95ee OSS \u7684\u673a\u5668\u3002 a. \u4f7f\u7528 root \u7528\u6237\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u521b\u5efa test \u7528\u6237\u3002 \u00a7 useradd test \u8bf4\u660e: \u9700\u8981\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5728\u521b\u5efaTGT\u65f6\u4f1a\u7528\u5230\u3002\u5982\u679c\u60a8\u4e0d\u60f3\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5219\u53ef\u4ee5\u6267\u884c\u4e0b\u4e00\u6b65\uff0c\u628a principal \u7684\u7528\u6237\u540d\u548c\u5bc6\u7801\u5bfc\u5165\u5230 keytab \u6587\u4ef6\u4e2d\u3002 \u00a7 b. \u53ef\u9009\uff1a\u6267\u884c\u5982\u4e0b\u547d\u4ee4\uff0c\u751f\u6210keytab\u6587\u4ef6\u3002 \u00a7 ktadd -k /root/test.keytab test c. \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u5207\u6362\u4e3a test \u7528\u6237\u3002 \u00a7 su test d. \u751f\u6210TGT\u3002 \u00a7 \u65b9\u5f0f\u4e00\uff1a\u4f7f\u7528\u7528\u6237\u540d\u548c\u5bc6\u7801\u65b9\u5f0f\uff0c\u521b\u5efa TGT\u3002 \u6267\u884c kinit \u547d\u4ee4\uff0c\u56de\u8f66\u540e\u8f93\u5165 test \u7684\u5bc6\u7801 123456\u3002 \u65b9\u5f0f\u4e8c\uff1a\u4f7f\u7528 keytab \u6587\u4ef6\uff0c\u521b\u5efa TGT\u3002 \u5728\u6b65\u9aa4 4 \u4e2d\u7684 test.keytab \u6587\u4ef6\uff0c\u5df2\u7ecf\u4fdd\u5b58\u5728 emr-header-1 \u673a\u5668\u7684 /root/ \u76ee\u5f55\u4e0b\uff0c\u9700\u8981\u4f7f\u7528 scp \u547d\u4ee4\u62f7\u8d1d\u5230\u5f53\u524d\u673a\u5668\u7684 /home/test/ \u76ee\u5f55\u4e0b\u3002 kinit -kt /home/test/test.keytab test e. \u67e5\u770bTGT\u3002 \u00a7 \u4f7f\u7528 klist \u547d\u4ee4\uff0c\u5982\u679c\u51fa\u73b0\u5982\u4e0b\u4fe1\u606f\uff0c\u5219\u8bf4\u660eTGT\u521b\u5efa\u6210\u529f\uff0c\u5373\u53ef\u4ee5\u8bbf\u95eeOSS\u4e86\u3002 Ticket cache: FILE:/tmp/krb5cc_1012 Default principal: test@EMR.C-85D4B8D74296****.COM Valid starting Expires Service principal 11/15/22 15:22:57 11/16/22 15:22:57 krbtgt/EMR.EMR.C-85D4B8D74296****.COM@EMR.C-85D4B8D74296****.COM renew until 11/22/22 15:22:57 5. \u5728 Ranger WebUI \u914d\u7f6e OSS \u6743\u9650 \u00a7 a. \u521b\u5efa\u4e00\u4e2a\u540d\u4e3a emr-oss \u7684 service\uff0c \u5e76\u6dfb\u52a0\u4e0b\u5217\u914d\u7f6e\u3002 \u00a7 \u53c2\u6570 \u503c policy.download.auth.users jindodata \u521b\u5efa\u5b8c\u6210\u540e\u5982\u4e0b\uff0c\u540e\u7eed\u5c31\u53ef\u4ee5\u5728\u8be5Service\u4e0b\u521b\u5efaPolicy\u3002 b. Ranger \u89c4\u5219\u793a\u4f8b \u00a7 \u4f8b\uff1a\u914d\u7f6e\u7528\u6237test\u62e5\u6709\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u6240\u4ee5\u6743\u9650\u7684\u6b65\u9aa4\uff1a \u914d\u7f6e test \u7528\u6237\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u8bbf\u95ee\u6743\u9650\u4e3a ALL\u3002 \u00a7 \u8bf4\u660e\uff1a \u00a7 \u89c4\u5219\u914d\u7f6e\u9875\u9762\u4e2d\uff0c\u914d\u7f6e\u7684 path \u6ca1\u6709 oss:// \u7684\u524d\u7f00\u3002 recursive\u6309\u94ae\u4e0d\u53ef\u5173\u95ed\u3002 \u8def\u5f84\u672b\u5c3e\u65e0\u9700\u5e26\u6b63\u659c\u7ebf\uff08/\uff09\u3002 6. \u8bbf\u95eeOSS\u3002 \u00a7 \u82e5\u7528\u6237\u8bbf\u95ee Ranger \u6ca1\u6709\u6388\u6743\u7684\u8def\u5f84\uff0c\u5219\u4f1a\u62a5\u5982\u4e0b\u9519\u8bef\uff1a org.apache.hadoop.security.AccessControlException: Permission denied: user=test, access=READ_EXECUTE, resourcePath=\"bucket-test-hangzhou/\"","title":"EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#emr-oss-ranger","text":"\u672c\u6587\u4ecb\u7ecd\u5728 EMR \u96c6\u7fa4\u914d\u7f6e\u963f\u91cc\u4e91 OSS \u548c OSS-HDFS \u670d\u52a1\u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\u3002","title":"EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#_1","text":"Apache Ranger \u63d0\u4f9b\u96c6\u4e2d\u5f0f\u7684\u6743\u9650\u7ba1\u7406\u6846\u67b6\uff0c\u53ef\u4ee5\u5bf9 Hadoop \u751f\u6001\u4e2d\u7684\u591a\u4e2a\u7ec4\u4ef6\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u8bbf\u95ee\u63a7\u5236\u3002\u5f53\u7528\u6237\u5c06\u6570\u636e\u5b58\u653e\u5728\u963f\u91cc\u4e91 OSS \u65f6\uff0c\u5219\u662f\u901a\u8fc7\u963f\u91cc\u4e91 RAM \u4ea7\u54c1\u521b\u5efa\u6216\u7ba1\u7406 RAM \u7528\u6237\uff0c\u5bf9RAM\u7528\u6237\u5b9e\u73b0\u5bf9 OSS \u8d44\u6e90\u7684\u8bbf\u95ee\u63a7\u5236\u3002 \u4e3a\u7ef4\u6301\u5927\u6570\u636e\u5ba2\u6237\u7684\u4f7f\u7528\u4e60\u60ef\uff0c\u901a\u8fc7 JindoFSx Namespace \u63a5\u5165 Ranger \u7684\u5ba2\u6237\u7aef\uff0c\u65b9\u4fbf\u7528\u6237\u7edf\u4e00\u7ba1\u7406\u5927\u6570\u636e\u7ec4\u4ef6\u6743\u9650\u3002","title":"\u80cc\u666f"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#oss","text":"OSS \u7684\u8bbf\u95ee\u5bc6\u94a5 AccessKey\uff08AK\uff09\u7edf\u4e00\u5728 JindoFSx Namespace \u4e2d\u8bbe\u7f6e\uff0c\u907f\u514d\u7528\u6237\u5728\u5ba2\u6237\u7aef\u914d\u7f6e\u660e\u6587\u5bc6\u94a5\uff0c\u5efa\u8bae\u53ea\u5141\u8bb8\u7ba1\u7406\u5458\u64cd\u4f5c\u548c\u7ba1\u7406 JindoFSx Namespace \u670d\u52a1\uff1b JindoSDK \u63d0\u4f9b\u6807\u51c6 Hadoop Filesystem \u5ba2\u6237\u7aef\uff0c\u5c06\u8bbf\u95ee OSS \u7684\u8bf7\u6c42\u4f1a\u53d1\u9001\u81f3 Namespace \u670d\u52a1\uff1b JindoFSx Namespace Service \u8d1f\u8d23\u96c6\u6210 Ranger \u5ba2\u6237\u7aef\uff0c\u5468\u671f\u6027\u5c06\u6743\u9650\u7b56\u7565\u4ece Ranger \u670d\u52a1\u7aef\u540c\u6b65\u5230\u672c\u5730\uff1b Namespace \u670d\u52a1\u5728\u6536\u5230 JindoSDK \u7684\u9274\u6743\u8bf7\u6c42\u540e\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u6821\u9a8c\uff1b \u901a\u8fc7\u6743\u9650\u6821\u9a8c\u540e JindoSDK \u5219\u53ef\u4ee5\u4f7f\u7528 JindoFSx Namespace \u670d\u52a1\u9881\u53d1\u7684\u4e34\u65f6 AK \u8bbf\u95ee OSS\u3002 \u6ce8\uff1a\u82e5 Namespace \u90e8\u7f72\u7684\u8282\u70b9\u4e3a\u963f\u91cc\u4e91 ECS \u8282\u70b9\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u5b89\u5168\u7ec4\uff0c\u9650\u5236\u8bbf\u95ee Namespace \u7684\u5ba2\u6237\u7aef\u3002","title":"\u8bbf\u95ee OSS \u9274\u6743\u6d41\u7a0b"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#_2","text":"\u5df2\u521b\u5efa E-MapReduce EMR-5.9.1/EMR-3.43.1 \u6216\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\uff0c\u5e76\u4e14\u9009\u62e9\u4e86 Ranger \u670d\u52a1\u3002 \u542f\u7528 Kerberos \u8ba4\u8bc1\u6ee1\u8db3\u8ba4\u8bc1\u9700\u6c42\u3002 \u82e5\u4f7f\u7528\u96c6\u7fa4\u90fd\u662f\u53ef\u4fe1\u7528\u6237\uff0c\u5219\u53ef\u4e0d\u542f\u7528 Kerberos \u8ba4\u8bc1\u3002 \u6ce8\u610f\uff1aEMR \u96c6\u7fa4\u5f00\u542f Kerberos \u540e\uff0c\u5c06\u65e0\u6cd5\u5173\u95ed Kerberos \u8ba4\u8bc1\uff0c\u8bf7\u8c28\u614e\u9009\u62e9\u3002\u5f00\u542f Kerberos \u540e\uff0c\u96c6\u7fa4\u6240\u6709\u5927\u6570\u636e\u7ec4\u4ef6\u7684\u670d\u52a1\u5747\u9700\u8981\u7ecf\u8fc7 Kerberos \u8ba4\u8bc1\uff0c\u5927\u6570\u636e\u4f5c\u4e1a\u63d0\u4ea4\u5230\u96c6\u7fa4\u4f1a\u5148\u7ecf\u8fc7\u8eab\u4efd\u8ba4\u8bc1\u3002Kerberos \u8be6\u60c5\u4fe1\u606f\uff0c\u8bf7\u53c2\u89c1 \u9ad8\u5b89\u5168\u7c7b\u578b\u96c6\u7fa4","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#1-jindofsx-namespace","text":"","title":"1. \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#11-jindo-sasl","text":"\u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 /opt/apps/JINDODATA/jindodata-current/plugins \u8def\u5f84\u4e0b\u3002 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.plugin.dir /opt/apps/JINDODATA/jindodata-current/plugins","title":"1.1 \u914d\u7f6e Jindo SASL \u63d2\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#12-ranger","text":"\u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.oss.authorization.method ranger","title":"1.2 \u542f\u7528 Ranger \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#13-assume-role-ak-ecs_role","text":"\u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.oss.credential.provider ASSUME_ROLE \u5728\u963f\u91cc\u4e91 RAM \u521b\u5efa\u7528\u6237\u9881\u53d1\u4e34\u65f6 AK \u7684 Role, \u9009\u62e9\u53ef\u4fe1\u5b9e\u4f53\u7c7b\u578b\u4e3a\"\u963f\u91cc\u4e91\u8d26\u53f7\", \u5e76\u4e3a\u89d2\u8272\u6388\u6743\u3002 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > common > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c default.credential.provider ASSUME_ROLE sts.access.key STS \u7684 accessKey sts.access.secret STS \u7684 accessKeySecret sts.access.endpoint STS \u7684 endpoint, \u5982\uff1a sts.cn-shanghai.aliyuncs.com sts.role.name \u5728 RAM \u4e2d\u521b\u5efa\u7684\u89d2\u8272\uff0c \u5982 JindoFsxTestRole sts.role.arn \u89d2\u8272\u7684ARN\uff0c\u5982\uff1aacs:ram::xxx:role/JindoFsxTestRole","title":"1.3 \u914d\u7f6e\u4f7f\u7528 Assume Role \u9881\u53d1\u4e34\u65f6 AK\uff08\u53ef\u9009\uff0c\u9ed8\u8ba4\u4f7f\u7528 ECS_ROLE \u514d\u5bc6\uff09"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#14-principal-keytab-kerberos","text":"","title":"1.4 \u914d\u7f6e Principal \u548c Keytab\uff08\u9002\u7528\u4e8e Kerberos \u96c6\u7fa4\uff09"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#a-principal-keytab","text":"\u5982\u679c\u60a8\u662f\u4f7f\u7528root\u7528\u6237\uff0c\u767b\u5f55KDC\uff08Kerberos\u7684\u670d\u52a1\u7aef\u7a0b\u5e8f\uff09\u6240\u5728\u7684master-1-1\u8282\u70b9\uff0c\u5219\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u76f4\u63a5\u8fdb\u5165admin\u5de5\u5177\u3002 kadmin.local \u5f53\u8fd4\u56de\u4fe1\u606f\u4e2d\u5305\u542b\u5982\u4e0b\u4fe1\u606f\u65f6\uff0c\u8868\u793a\u5df2\u8fdb\u5165admin.local\u547d\u4ee4\u884c\u3002 Authenticating as principal hadoop/admin@EMR.C-85D4B8D74296****.COM with password. kadmin.local: \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u521b\u5efa\u7528\u6237\u540d\u4e3a jindodata \u7684 principal, \u683c\u5f0f\u4e3a jindodata/ . .emr.aliyuncs.com\u3002 addprinc -randkey jindodata/master-1-1.c-9e4fc2dcc****.cn-shanghai.emr.aliyuncs.com \u5bfc\u51fa keytab \u6587\u4ef6\u3002 ktadd -k jindodata.keytab jindodata/master-1-1.c-9e4fc2dcc****.cn-shanghai.emr.aliyuncs.com","title":"a. \u521b\u5efa principal \u548c keytab"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#b","text":"\u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.authentication.enable true namespace.authentication.keytab \u914d\u7f6e keytab \u6240\u5728\u7684\u8def\u5f84\uff0c \u5982\uff1a /opt/apps/JINDODATA/jindodata-current/conf/jindodata.keytab namespace.authentication.principal \u914d\u7f6e keytab \u4e2d\u7684 principal, \u5982\uff1a jindodata/master-1-1.c-9e4fc2dcc *.cn-shanghai.emr.aliyuncs.com@EMR.C-85D4B8D74296 *.COM","title":"b. \u914d\u7f6e"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#15-ranger-admin-service-url","text":"\u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c ranger.plugin.jindo-auth.policy.rest.url \u914d\u7f6e\u83b7\u53d6 Ranger Policy \u7684 URL, \u5982\uff1ahttp://master-1-1:6080\uff0c \u9ed8\u8ba4\u4e3a http://emr-header-1:6080","title":"1.5 \u914d\u7f6e Ranger Admin Service URL"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#16-jindofsx-namespace","text":"","title":"1.6 \u91cd\u542f JindoFSx Namespace \u670d\u52a1\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#2-jindosdk","text":"","title":"2. \u914d\u7f6e JindoSDK\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#21-xengine","text":"\u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \uff0c \u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.xengine jindofsx","title":"2.1 \u914d\u7f6e xengine"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#22-jindo-sasl","text":"\u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 /opt/apps/JINDOSDK/jindosdk-current/plugins/ \u8def\u5f84\u4e0b\uff0c \u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c \u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.jdo.plugin.dir /opt/apps/JINDOSDK/jindosdk-current/plugins","title":"2.2 \u914d\u7f6e Jindo SASL \u63d2\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#23-ranger","text":"\u652f\u6301\u7edf\u4e00\u914d\u7f6e\u548c\u6309 bucket \u914d\u7f6e\u4e24\u79cd\u65b9\u5f0f\u3002","title":"2.3 \u542f\u7528 Ranger \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#a-oss-oss-hdfs-bucket","text":"\u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \uff0c \u65b0\u589e\u6216\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.authorization.method ranger fs.oss.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider","title":"a. \u5bf9 OSS \u548c OSS-HDFS \u7684\u5168\u90e8 bucket \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#b-oss-oss-hdfs-bucket","text":"\u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \uff0c \u65b0\u589e\u6216\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.bucket.XXX.authorization.method ranger fs.oss.bucket.XXX.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider \u8bf4\u660e\uff1aXXX \u4e3a \u6216 OSS \u6216 OSS-HDFS \u670d\u52a1 bucket\u7684\u540d\u79f0\u3002","title":"b. \u5bf9 OSS \u6216 OSS-HDFS \u7684\u90e8\u5206 bucket \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#24-hiveserver2","text":"","title":"2.4. \u91cd\u542f HiveServer2 \u7b49\u6240\u6709\u5e38\u9a7b\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#3-principal","text":"\u5982\u679c\u60a8\u662f\u4f7f\u7528root\u7528\u6237\uff0c\u767b\u5f55KDC\uff08Kerberos\u7684\u670d\u52a1\u7aef\u7a0b\u5e8f\uff09\u6240\u5728\u7684 master-1-1 \u8282\u70b9\uff0c\u5219\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u76f4\u63a5\u8fdb\u5165admin\u5de5\u5177\u3002 kadmin.local \u5f53\u8fd4\u56de\u4fe1\u606f\u4e2d\u5305\u542b\u5982\u4e0b\u4fe1\u606f\u65f6\uff0c\u8868\u793a\u5df2\u8fdb\u5165admin.local\u547d\u4ee4\u884c\u3002 Authenticating as principal hadoop/admin@EMR.C-85D4B8D74296****.COM with password. kadmin.local: \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u521b\u5efa\u7528\u6237\u540d\u4e3a test \u7684 principal\u3002 addprinc -pw 123456 test","title":"3. \u521b\u5efa\u7528\u6237 Principal\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#4-tgt","text":"\u521b\u5efa TGT \u7684\u673a\u5668\uff0c\u53ef\u4ee5\u662f\u4efb\u610f\u4e00\u53f0\u9700\u8981\u8bbf\u95ee OSS \u7684\u673a\u5668\u3002","title":"4. \u521b\u5efa TGT\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#a-root-test","text":"useradd test","title":"a. \u4f7f\u7528 root \u7528\u6237\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u521b\u5efa test \u7528\u6237\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#tgt-principal-keytab","text":"","title":"\u8bf4\u660e: \u9700\u8981\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5728\u521b\u5efaTGT\u65f6\u4f1a\u7528\u5230\u3002\u5982\u679c\u60a8\u4e0d\u60f3\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5219\u53ef\u4ee5\u6267\u884c\u4e0b\u4e00\u6b65\uff0c\u628a principal \u7684\u7528\u6237\u540d\u548c\u5bc6\u7801\u5bfc\u5165\u5230 keytab \u6587\u4ef6\u4e2d\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#b-keytab","text":"ktadd -k /root/test.keytab test","title":"b. \u53ef\u9009\uff1a\u6267\u884c\u5982\u4e0b\u547d\u4ee4\uff0c\u751f\u6210keytab\u6587\u4ef6\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#c-test","text":"su test","title":"c. \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u5207\u6362\u4e3a test \u7528\u6237\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#d-tgt","text":"\u65b9\u5f0f\u4e00\uff1a\u4f7f\u7528\u7528\u6237\u540d\u548c\u5bc6\u7801\u65b9\u5f0f\uff0c\u521b\u5efa TGT\u3002 \u6267\u884c kinit \u547d\u4ee4\uff0c\u56de\u8f66\u540e\u8f93\u5165 test \u7684\u5bc6\u7801 123456\u3002 \u65b9\u5f0f\u4e8c\uff1a\u4f7f\u7528 keytab \u6587\u4ef6\uff0c\u521b\u5efa TGT\u3002 \u5728\u6b65\u9aa4 4 \u4e2d\u7684 test.keytab \u6587\u4ef6\uff0c\u5df2\u7ecf\u4fdd\u5b58\u5728 emr-header-1 \u673a\u5668\u7684 /root/ \u76ee\u5f55\u4e0b\uff0c\u9700\u8981\u4f7f\u7528 scp \u547d\u4ee4\u62f7\u8d1d\u5230\u5f53\u524d\u673a\u5668\u7684 /home/test/ \u76ee\u5f55\u4e0b\u3002 kinit -kt /home/test/test.keytab test","title":"d. \u751f\u6210TGT\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#e-tgt","text":"\u4f7f\u7528 klist \u547d\u4ee4\uff0c\u5982\u679c\u51fa\u73b0\u5982\u4e0b\u4fe1\u606f\uff0c\u5219\u8bf4\u660eTGT\u521b\u5efa\u6210\u529f\uff0c\u5373\u53ef\u4ee5\u8bbf\u95eeOSS\u4e86\u3002 Ticket cache: FILE:/tmp/krb5cc_1012 Default principal: test@EMR.C-85D4B8D74296****.COM Valid starting Expires Service principal 11/15/22 15:22:57 11/16/22 15:22:57 krbtgt/EMR.EMR.C-85D4B8D74296****.COM@EMR.C-85D4B8D74296****.COM renew until 11/22/22 15:22:57","title":"e. \u67e5\u770bTGT\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#5-ranger-webui-oss","text":"","title":"5. \u5728 Ranger WebUI \u914d\u7f6e OSS \u6743\u9650"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#a-emr-oss-service","text":"\u53c2\u6570 \u503c policy.download.auth.users jindodata \u521b\u5efa\u5b8c\u6210\u540e\u5982\u4e0b\uff0c\u540e\u7eed\u5c31\u53ef\u4ee5\u5728\u8be5Service\u4e0b\u521b\u5efaPolicy\u3002","title":"a. \u521b\u5efa\u4e00\u4e2a\u540d\u4e3a emr-oss \u7684 service\uff0c \u5e76\u6dfb\u52a0\u4e0b\u5217\u914d\u7f6e\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#b-ranger","text":"\u4f8b\uff1a\u914d\u7f6e\u7528\u6237test\u62e5\u6709\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u6240\u4ee5\u6743\u9650\u7684\u6b65\u9aa4\uff1a","title":"b. Ranger \u89c4\u5219\u793a\u4f8b"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#test-ossbucket-test-hangzhouusertest-all","text":"","title":"\u914d\u7f6e test \u7528\u6237\u8bbf\u95eeoss://bucket-test-hangzhou/user/test\u76ee\u5f55\u7684\u8bbf\u95ee\u6743\u9650\u4e3a ALL\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#_3","text":"\u89c4\u5219\u914d\u7f6e\u9875\u9762\u4e2d\uff0c\u914d\u7f6e\u7684 path \u6ca1\u6709 oss:// \u7684\u524d\u7f00\u3002 recursive\u6309\u94ae\u4e0d\u53ef\u5173\u95ed\u3002 \u8def\u5f84\u672b\u5c3e\u65e0\u9700\u5e26\u6b63\u659c\u7ebf\uff08/\uff09\u3002","title":"\u8bf4\u660e\uff1a"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#6-oss","text":"\u82e5\u7528\u6237\u8bbf\u95ee Ranger \u6ca1\u6709\u6388\u6743\u7684\u8def\u5f84\uff0c\u5219\u4f1a\u62a5\u5982\u4e0b\u9519\u8bef\uff1a org.apache.hadoop.security.AccessControlException: Permission denied: user=test, access=READ_EXECUTE, resourcePath=\"bucket-test-hangzhou/\"","title":"6. \u8bbf\u95eeOSS\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/","text":"EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u00a7 \u672c\u6587\u4ecb\u7ecd\u5728 EMR \u96c6\u7fa4\u914d\u7f6e\u963f\u91cc\u4e91 OSS \u548c OSS-HDFS \u670d\u52a1\u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\u3002 \u80cc\u666f \u00a7 Apache Ranger \u63d0\u4f9b\u96c6\u4e2d\u5f0f\u7684\u6743\u9650\u7ba1\u7406\u6846\u67b6\uff0c\u53ef\u4ee5\u5bf9 Hadoop \u751f\u6001\u4e2d\u7684\u591a\u4e2a\u7ec4\u4ef6\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u8bbf\u95ee\u63a7\u5236\u3002\u5f53\u7528\u6237\u5c06\u6570\u636e\u5b58\u653e\u5728\u963f\u91cc\u4e91 OSS \u65f6\uff0c\u5219\u662f\u901a\u8fc7\u963f\u91cc\u4e91 RAM \u4ea7\u54c1\u521b\u5efa\u6216\u7ba1\u7406 RAM \u7528\u6237\uff0c\u5bf9RAM\u7528\u6237\u5b9e\u73b0\u5bf9 OSS \u8d44\u6e90\u7684\u8bbf\u95ee\u63a7\u5236\u3002 \u4e3a\u7ef4\u6301\u5927\u6570\u636e\u5ba2\u6237\u7684\u4f7f\u7528\u4e60\u60ef\uff0c\u901a\u8fc7 JindoFSx Namespace \u63a5\u5165 Ranger \u7684\u5ba2\u6237\u7aef\uff0c\u65b9\u4fbf\u7528\u6237\u7edf\u4e00\u7ba1\u7406\u5927\u6570\u636e\u7ec4\u4ef6\u6743\u9650\u3002 \u8bbf\u95ee OSS \u9274\u6743\u6d41\u7a0b \u00a7 OSS \u7684\u8bbf\u95ee\u5bc6\u94a5 AccessKey\uff08AK\uff09\u7edf\u4e00\u5728 JindoFSx Namespace \u4e2d\u8bbe\u7f6e\uff0c\u907f\u514d\u7528\u6237\u5728\u5ba2\u6237\u7aef\u914d\u7f6e\u660e\u6587\u5bc6\u94a5\uff0c\u5efa\u8bae\u53ea\u5141\u8bb8\u7ba1\u7406\u5458\u64cd\u4f5c\u548c\u7ba1\u7406 JindoFSx Namespace \u670d\u52a1\uff1b JindoSDK \u63d0\u4f9b\u6807\u51c6 Hadoop Filesystem \u5ba2\u6237\u7aef\uff0c\u5c06\u8bbf\u95ee OSS \u7684\u8bf7\u6c42\u4f1a\u53d1\u9001\u81f3 Namespace \u670d\u52a1\uff1b JindoFSx Namespace Service \u8d1f\u8d23\u96c6\u6210 Ranger \u5ba2\u6237\u7aef\uff0c\u5468\u671f\u6027\u5c06\u6743\u9650\u7b56\u7565\u4ece Ranger \u670d\u52a1\u7aef\u540c\u6b65\u5230\u672c\u5730\uff1b Namespace \u670d\u52a1\u5728\u6536\u5230 JindoSDK \u7684\u9274\u6743\u8bf7\u6c42\u540e\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u6821\u9a8c\uff1b \u901a\u8fc7\u6743\u9650\u6821\u9a8c\u540e JindoSDK \u5219\u53ef\u4ee5\u4f7f\u7528 JindoFSx Namespace \u670d\u52a1\u9881\u53d1\u7684\u4e34\u65f6 AK \u8bbf\u95ee OSS\u3002 \u6ce8\uff1a\u82e5 Namespace \u90e8\u7f72\u7684\u8282\u70b9\u4e3a\u963f\u91cc\u4e91 ECS \u8282\u70b9\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u5b89\u5168\u7ec4\uff0c\u9650\u5236\u8bbf\u95ee Namespace \u7684\u5ba2\u6237\u7aef\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u521b\u5efa E-MapReduce EMR-5.6.0/EMR-3.40.0 \u6216\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\uff0c\u5e76\u4e14\u9009\u62e9\u4e86 Ranger \u670d\u52a1\u3002 \u542f\u7528 Kerberos \u8ba4\u8bc1\u6ee1\u8db3\u8ba4\u8bc1\u9700\u6c42\u3002 \u82e5\u4f7f\u7528\u96c6\u7fa4\u90fd\u662f\u53ef\u4fe1\u7528\u6237\uff0c\u5219\u53ef\u4e0d\u542f\u7528 Kerberos \u8ba4\u8bc1\u3002 \u6ce8\u610f\uff1aEMR \u96c6\u7fa4\u5f00\u542f Kerberos \u540e\uff0c\u5c06\u65e0\u6cd5\u5173\u95ed Kerberos \u8ba4\u8bc1\uff0c\u8bf7\u8c28\u614e\u9009\u62e9\u3002\u5f00\u542f Kerberos \u540e\uff0c\u96c6\u7fa4\u6240\u6709\u5927\u6570\u636e\u7ec4\u4ef6\u7684\u670d\u52a1\u5747\u9700\u8981\u7ecf\u8fc7 Kerberos \u8ba4\u8bc1\uff0c\u5927\u6570\u636e\u4f5c\u4e1a\u63d0\u4ea4\u5230\u96c6\u7fa4\u4f1a\u5148\u7ecf\u8fc7\u8eab\u4efd\u8ba4\u8bc1\u3002Kerberos \u8be6\u60c5\u4fe1\u606f\uff0c\u8bf7\u53c2\u89c1 \u9ad8\u5b89\u5168\u7c7b\u578b\u96c6\u7fa4 \u521b\u5efa\u9ad8\u5b89\u5168\u96c6\u7fa4 \u00a7 \u521b\u5efa\u96c6\u7fa4\u65f6\uff0c\u5728\u8f6f\u4ef6\u914d\u7f6e\u9875\u9762\u7684\u9ad8\u7ea7\u8bbe\u7f6e\u533a\u57df\u4e2d\uff0c\u6253\u5f00Kerberos\u96c6\u7fa4\u6a21\u5f0f\u5f00\u5173\u3002 1. Ranger \u542f\u7528 OSS\u3002 \u00a7 2. \u90e8\u7f72\u5ba2\u6237\u7aef\u914d\u7f6e\u3002 \u00a7 \u5728 HDFS \u670d\u52a1\u9875\u9762\uff0c\u5355\u51fb\u53f3\u4e0a\u89d2\u7684\u90e8\u7f72\u5ba2\u6237\u7aef\u914d\u7f6e\u3002 3. \u91cd\u542f JindoFSx Namespace \u670d\u52a1\u3002 \u00a7 4. \u91cd\u542fHiveServer2\u3002 \u00a7 5. \u521b\u5efa\u7528\u6237 Principal\u3002 \u00a7 a. \u901a\u8fc7SSH\u65b9\u5f0f\u8fde\u63a5\u96c6\u7fa4\u7684emr-header-1\u8282\u70b9\u3002 \u00a7 b. \u6267\u884c\u5982\u4e0b\u547d\u4ee4\uff0c\u8fdb\u5165Kerberos\u7684admin\u5de5\u5177\u3002 \u00a7 sh /usr/lib/has-current/bin/admin-local.sh /etc/ecm/has-conf -k /etc/ecm/has-conf/admin.keytab \u672c\u793a\u4f8b\u5bc6\u7801\u8bbe\u7f6e\u4e3a 123456\u3002 addprinc -pw 123456 test \u8bf4\u660e: \u9700\u8981\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5728\u521b\u5efaTGT\u65f6\u4f1a\u7528\u5230\u3002\u5982\u679c\u60a8\u4e0d\u60f3\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5219\u53ef\u4ee5\u6267\u884c\u4e0b\u4e00\u6b65\uff0c\u628aPrincipal\u7684\u7528\u6237\u540d\u548c\u5bc6\u7801\u5bfc\u5165\u5230keytab\u6587\u4ef6\u4e2d\u3002 \u00a7 c. \u53ef\u9009\uff1a\u6267\u884c\u5982\u4e0b\u547d\u4ee4\uff0c\u751f\u6210keytab\u6587\u4ef6\u3002 \u00a7 ktadd -k /root/test.keytab test \u6267\u884c quit \u547d\u4ee4\uff0c\u53ef\u4ee5\u9000\u51fa Kerberos \u7684 admin \u5de5\u5177\u3002 6. \u521b\u5efa TGT\u3002 \u00a7 \u521b\u5efa TGT \u7684\u673a\u5668\uff0c\u53ef\u4ee5\u662f\u4efb\u610f\u4e00\u53f0\u9700\u8981\u8bbf\u95ee OSS \u7684\u673a\u5668\u3002 a. \u4f7f\u7528 root \u7528\u6237\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u521b\u5efa test \u7528\u6237\u3002 \u00a7 useradd test b. \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u5207\u6362\u4e3a test \u7528\u6237\u3002 \u00a7 su test c. \u751f\u6210TGT\u3002 \u00a7 \u65b9\u5f0f\u4e00\uff1a\u4f7f\u7528\u7528\u6237\u540d\u548c\u5bc6\u7801\u65b9\u5f0f\uff0c\u521b\u5efa TGT\u3002 \u6267\u884c kinit \u547d\u4ee4\uff0c\u56de\u8f66\u540e\u8f93\u5165 test \u7684\u5bc6\u7801 123456\u3002 \u65b9\u5f0f\u4e8c\uff1a\u4f7f\u7528 keytab \u6587\u4ef6\uff0c\u521b\u5efa TGT\u3002 \u5728\u6b65\u9aa4 4 \u4e2d\u7684 test.keytab \u6587\u4ef6\uff0c\u5df2\u7ecf\u4fdd\u5b58\u5728 emr-header-1 \u673a\u5668\u7684 /root/ \u76ee\u5f55\u4e0b\uff0c\u9700\u8981\u4f7f\u7528 scp \u547d\u4ee4\u62f7\u8d1d\u5230\u5f53\u524d\u673a\u5668\u7684 /home/test/ \u76ee\u5f55\u4e0b\u3002 kinit -kt /home/test/test.keytab test d. \u67e5\u770bTGT\u3002 \u00a7 \u4f7f\u7528 klist \u547d\u4ee4\uff0c\u5982\u679c\u51fa\u73b0\u5982\u4e0b\u4fe1\u606f\uff0c\u5219\u8bf4\u660eTGT\u521b\u5efa\u6210\u529f\uff0c\u5373\u53ef\u4ee5\u8bbf\u95eeOSS\u4e86\u3002 Ticket cache: FILE:/tmp/krb5cc_1012 Default principal: test@EMR.23****.COM Valid starting Expires Service principal 03/27/2022 23:20:44 03/28/2022 23:20:44 krbtgt/EMR.238075.COM@EMR.238075.COM renew until 03/28/2022 23:20:44 \u6ce8\u610f: \u9700\u8981\u8bb0\u5f55\u4e0b\u56de\u663e\u4fe1\u606f EMR.23****.COM \u4e2d\u7684\u6570\u5b57 23**** \uff0c\u5373\u4e3a cluster_id \u7684\u503c\uff0c\u540e\u9762\u8bbf\u95ee OSS \u65f6\u9700\u8981\u3002 7. \u5728 Ranger WebUI \u914d\u7f6e OSS \u6743\u9650 \u00a7 Ranger \u89c4\u5219\u793a\u4f8b \u00a7 \u4f8b\uff1a\u914d\u7f6e\u7528\u6237test\u62e5\u6709\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u6240\u4ee5\u6743\u9650\u7684\u6b65\u9aa4\uff1a a. \u914d\u7f6e test \u7528\u6237\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u8bbf\u95ee\u6743\u9650\u4e3a ALL\u3002 \u00a7 \u8bf4\u660e\uff1a \u00a7 \u89c4\u5219\u914d\u7f6e\u9875\u9762\u4e2d\uff0c\u914d\u7f6e\u7684 path \u6ca1\u6709 oss:// \u7684\u524d\u7f00\u3002 recursive\u6309\u94ae\u4e0d\u53ef\u5173\u95ed\u3002 \u8def\u5f84\u672b\u5c3e\u65e0\u9700\u5e26\u6b63\u659c\u7ebf\uff08/\uff09\u3002 b. \u9700\u8981\u914d\u7f6e\u8bbf\u95ee\u8def\u5f84\u7684\u7236\u76ee\u5f55 oss://bucket-test-hangzhou/user \u7684\u6743\u9650\u4e3a Execute\u3002 \u00a7 8. \u8bbf\u95eeOSS\u3002 \u00a7 \u82e5\u7528\u6237\u8bbf\u95ee Ranger \u6ca1\u6709\u6388\u6743\u7684\u8def\u5f84\uff0c\u5219\u4f1a\u62a5\u5982\u4e0b\u9519\u8bef\uff1a org.apache.hadoop.security.AccessControlException: Permission denied: user=test, access=READ_EXECUTE, resourcePath=\"bucket-test-hangzhou/\"","title":"EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#emr-oss-ranger","text":"\u672c\u6587\u4ecb\u7ecd\u5728 EMR \u96c6\u7fa4\u914d\u7f6e\u963f\u91cc\u4e91 OSS \u548c OSS-HDFS \u670d\u52a1\u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\u3002","title":"EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#_1","text":"Apache Ranger \u63d0\u4f9b\u96c6\u4e2d\u5f0f\u7684\u6743\u9650\u7ba1\u7406\u6846\u67b6\uff0c\u53ef\u4ee5\u5bf9 Hadoop \u751f\u6001\u4e2d\u7684\u591a\u4e2a\u7ec4\u4ef6\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u8bbf\u95ee\u63a7\u5236\u3002\u5f53\u7528\u6237\u5c06\u6570\u636e\u5b58\u653e\u5728\u963f\u91cc\u4e91 OSS \u65f6\uff0c\u5219\u662f\u901a\u8fc7\u963f\u91cc\u4e91 RAM \u4ea7\u54c1\u521b\u5efa\u6216\u7ba1\u7406 RAM \u7528\u6237\uff0c\u5bf9RAM\u7528\u6237\u5b9e\u73b0\u5bf9 OSS \u8d44\u6e90\u7684\u8bbf\u95ee\u63a7\u5236\u3002 \u4e3a\u7ef4\u6301\u5927\u6570\u636e\u5ba2\u6237\u7684\u4f7f\u7528\u4e60\u60ef\uff0c\u901a\u8fc7 JindoFSx Namespace \u63a5\u5165 Ranger \u7684\u5ba2\u6237\u7aef\uff0c\u65b9\u4fbf\u7528\u6237\u7edf\u4e00\u7ba1\u7406\u5927\u6570\u636e\u7ec4\u4ef6\u6743\u9650\u3002","title":"\u80cc\u666f"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#oss","text":"OSS \u7684\u8bbf\u95ee\u5bc6\u94a5 AccessKey\uff08AK\uff09\u7edf\u4e00\u5728 JindoFSx Namespace \u4e2d\u8bbe\u7f6e\uff0c\u907f\u514d\u7528\u6237\u5728\u5ba2\u6237\u7aef\u914d\u7f6e\u660e\u6587\u5bc6\u94a5\uff0c\u5efa\u8bae\u53ea\u5141\u8bb8\u7ba1\u7406\u5458\u64cd\u4f5c\u548c\u7ba1\u7406 JindoFSx Namespace \u670d\u52a1\uff1b JindoSDK \u63d0\u4f9b\u6807\u51c6 Hadoop Filesystem \u5ba2\u6237\u7aef\uff0c\u5c06\u8bbf\u95ee OSS \u7684\u8bf7\u6c42\u4f1a\u53d1\u9001\u81f3 Namespace \u670d\u52a1\uff1b JindoFSx Namespace Service \u8d1f\u8d23\u96c6\u6210 Ranger \u5ba2\u6237\u7aef\uff0c\u5468\u671f\u6027\u5c06\u6743\u9650\u7b56\u7565\u4ece Ranger \u670d\u52a1\u7aef\u540c\u6b65\u5230\u672c\u5730\uff1b Namespace \u670d\u52a1\u5728\u6536\u5230 JindoSDK \u7684\u9274\u6743\u8bf7\u6c42\u540e\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u6821\u9a8c\uff1b \u901a\u8fc7\u6743\u9650\u6821\u9a8c\u540e JindoSDK \u5219\u53ef\u4ee5\u4f7f\u7528 JindoFSx Namespace \u670d\u52a1\u9881\u53d1\u7684\u4e34\u65f6 AK \u8bbf\u95ee OSS\u3002 \u6ce8\uff1a\u82e5 Namespace \u90e8\u7f72\u7684\u8282\u70b9\u4e3a\u963f\u91cc\u4e91 ECS \u8282\u70b9\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u5b89\u5168\u7ec4\uff0c\u9650\u5236\u8bbf\u95ee Namespace \u7684\u5ba2\u6237\u7aef\u3002","title":"\u8bbf\u95ee OSS \u9274\u6743\u6d41\u7a0b"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#_2","text":"\u5df2\u521b\u5efa E-MapReduce EMR-5.6.0/EMR-3.40.0 \u6216\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\uff0c\u5e76\u4e14\u9009\u62e9\u4e86 Ranger \u670d\u52a1\u3002 \u542f\u7528 Kerberos \u8ba4\u8bc1\u6ee1\u8db3\u8ba4\u8bc1\u9700\u6c42\u3002 \u82e5\u4f7f\u7528\u96c6\u7fa4\u90fd\u662f\u53ef\u4fe1\u7528\u6237\uff0c\u5219\u53ef\u4e0d\u542f\u7528 Kerberos \u8ba4\u8bc1\u3002 \u6ce8\u610f\uff1aEMR \u96c6\u7fa4\u5f00\u542f Kerberos \u540e\uff0c\u5c06\u65e0\u6cd5\u5173\u95ed Kerberos \u8ba4\u8bc1\uff0c\u8bf7\u8c28\u614e\u9009\u62e9\u3002\u5f00\u542f Kerberos \u540e\uff0c\u96c6\u7fa4\u6240\u6709\u5927\u6570\u636e\u7ec4\u4ef6\u7684\u670d\u52a1\u5747\u9700\u8981\u7ecf\u8fc7 Kerberos \u8ba4\u8bc1\uff0c\u5927\u6570\u636e\u4f5c\u4e1a\u63d0\u4ea4\u5230\u96c6\u7fa4\u4f1a\u5148\u7ecf\u8fc7\u8eab\u4efd\u8ba4\u8bc1\u3002Kerberos \u8be6\u60c5\u4fe1\u606f\uff0c\u8bf7\u53c2\u89c1 \u9ad8\u5b89\u5168\u7c7b\u578b\u96c6\u7fa4","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#_3","text":"\u521b\u5efa\u96c6\u7fa4\u65f6\uff0c\u5728\u8f6f\u4ef6\u914d\u7f6e\u9875\u9762\u7684\u9ad8\u7ea7\u8bbe\u7f6e\u533a\u57df\u4e2d\uff0c\u6253\u5f00Kerberos\u96c6\u7fa4\u6a21\u5f0f\u5f00\u5173\u3002","title":"\u521b\u5efa\u9ad8\u5b89\u5168\u96c6\u7fa4"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#1-ranger-oss","text":"","title":"1. Ranger \u542f\u7528 OSS\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#2","text":"\u5728 HDFS \u670d\u52a1\u9875\u9762\uff0c\u5355\u51fb\u53f3\u4e0a\u89d2\u7684\u90e8\u7f72\u5ba2\u6237\u7aef\u914d\u7f6e\u3002","title":"2. \u90e8\u7f72\u5ba2\u6237\u7aef\u914d\u7f6e\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#3-jindofsx-namespace","text":"","title":"3. \u91cd\u542f JindoFSx Namespace \u670d\u52a1\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#4-hiveserver2","text":"","title":"4. \u91cd\u542fHiveServer2\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#5-principal","text":"","title":"5. \u521b\u5efa\u7528\u6237 Principal\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#a-sshemr-header-1","text":"","title":"a. \u901a\u8fc7SSH\u65b9\u5f0f\u8fde\u63a5\u96c6\u7fa4\u7684emr-header-1\u8282\u70b9\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#b-kerberosadmin","text":"sh /usr/lib/has-current/bin/admin-local.sh /etc/ecm/has-conf -k /etc/ecm/has-conf/admin.keytab \u672c\u793a\u4f8b\u5bc6\u7801\u8bbe\u7f6e\u4e3a 123456\u3002 addprinc -pw 123456 test","title":"b. \u6267\u884c\u5982\u4e0b\u547d\u4ee4\uff0c\u8fdb\u5165Kerberos\u7684admin\u5de5\u5177\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#tgtprincipalkeytab","text":"","title":"\u8bf4\u660e: \u9700\u8981\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5728\u521b\u5efaTGT\u65f6\u4f1a\u7528\u5230\u3002\u5982\u679c\u60a8\u4e0d\u60f3\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5219\u53ef\u4ee5\u6267\u884c\u4e0b\u4e00\u6b65\uff0c\u628aPrincipal\u7684\u7528\u6237\u540d\u548c\u5bc6\u7801\u5bfc\u5165\u5230keytab\u6587\u4ef6\u4e2d\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#c-keytab","text":"ktadd -k /root/test.keytab test \u6267\u884c quit \u547d\u4ee4\uff0c\u53ef\u4ee5\u9000\u51fa Kerberos \u7684 admin \u5de5\u5177\u3002","title":"c. \u53ef\u9009\uff1a\u6267\u884c\u5982\u4e0b\u547d\u4ee4\uff0c\u751f\u6210keytab\u6587\u4ef6\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#6-tgt","text":"\u521b\u5efa TGT \u7684\u673a\u5668\uff0c\u53ef\u4ee5\u662f\u4efb\u610f\u4e00\u53f0\u9700\u8981\u8bbf\u95ee OSS \u7684\u673a\u5668\u3002","title":"6. \u521b\u5efa TGT\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#a-root-test","text":"useradd test","title":"a. \u4f7f\u7528 root \u7528\u6237\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u521b\u5efa test \u7528\u6237\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#b-test","text":"su test","title":"b. \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u5207\u6362\u4e3a test \u7528\u6237\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#c-tgt","text":"\u65b9\u5f0f\u4e00\uff1a\u4f7f\u7528\u7528\u6237\u540d\u548c\u5bc6\u7801\u65b9\u5f0f\uff0c\u521b\u5efa TGT\u3002 \u6267\u884c kinit \u547d\u4ee4\uff0c\u56de\u8f66\u540e\u8f93\u5165 test \u7684\u5bc6\u7801 123456\u3002 \u65b9\u5f0f\u4e8c\uff1a\u4f7f\u7528 keytab \u6587\u4ef6\uff0c\u521b\u5efa TGT\u3002 \u5728\u6b65\u9aa4 4 \u4e2d\u7684 test.keytab \u6587\u4ef6\uff0c\u5df2\u7ecf\u4fdd\u5b58\u5728 emr-header-1 \u673a\u5668\u7684 /root/ \u76ee\u5f55\u4e0b\uff0c\u9700\u8981\u4f7f\u7528 scp \u547d\u4ee4\u62f7\u8d1d\u5230\u5f53\u524d\u673a\u5668\u7684 /home/test/ \u76ee\u5f55\u4e0b\u3002 kinit -kt /home/test/test.keytab test","title":"c. \u751f\u6210TGT\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#d-tgt","text":"\u4f7f\u7528 klist \u547d\u4ee4\uff0c\u5982\u679c\u51fa\u73b0\u5982\u4e0b\u4fe1\u606f\uff0c\u5219\u8bf4\u660eTGT\u521b\u5efa\u6210\u529f\uff0c\u5373\u53ef\u4ee5\u8bbf\u95eeOSS\u4e86\u3002 Ticket cache: FILE:/tmp/krb5cc_1012 Default principal: test@EMR.23****.COM Valid starting Expires Service principal 03/27/2022 23:20:44 03/28/2022 23:20:44 krbtgt/EMR.238075.COM@EMR.238075.COM renew until 03/28/2022 23:20:44 \u6ce8\u610f: \u9700\u8981\u8bb0\u5f55\u4e0b\u56de\u663e\u4fe1\u606f EMR.23****.COM \u4e2d\u7684\u6570\u5b57 23**** \uff0c\u5373\u4e3a cluster_id \u7684\u503c\uff0c\u540e\u9762\u8bbf\u95ee OSS \u65f6\u9700\u8981\u3002","title":"d. \u67e5\u770bTGT\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#7-ranger-webui-oss","text":"","title":"7. \u5728 Ranger WebUI \u914d\u7f6e OSS \u6743\u9650"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#ranger","text":"\u4f8b\uff1a\u914d\u7f6e\u7528\u6237test\u62e5\u6709\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u6240\u4ee5\u6743\u9650\u7684\u6b65\u9aa4\uff1a","title":"Ranger \u89c4\u5219\u793a\u4f8b"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#a-test-ossbucket-test-hangzhouusertest-all","text":"","title":"a. \u914d\u7f6e test \u7528\u6237\u8bbf\u95eeoss://bucket-test-hangzhou/user/test\u76ee\u5f55\u7684\u8bbf\u95ee\u6743\u9650\u4e3a ALL\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#_4","text":"\u89c4\u5219\u914d\u7f6e\u9875\u9762\u4e2d\uff0c\u914d\u7f6e\u7684 path \u6ca1\u6709 oss:// \u7684\u524d\u7f00\u3002 recursive\u6309\u94ae\u4e0d\u53ef\u5173\u95ed\u3002 \u8def\u5f84\u672b\u5c3e\u65e0\u9700\u5e26\u6b63\u659c\u7ebf\uff08/\uff09\u3002","title":"\u8bf4\u660e\uff1a"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#b-ossbucket-test-hangzhouuser-execute","text":"","title":"b. \u9700\u8981\u914d\u7f6e\u8bbf\u95ee\u8def\u5f84\u7684\u7236\u76ee\u5f55oss://bucket-test-hangzhou/user\u7684\u6743\u9650\u4e3a Execute\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#8-oss","text":"\u82e5\u7528\u6237\u8bbf\u95ee Ranger \u6ca1\u6709\u6388\u6743\u7684\u8def\u5f84\uff0c\u5219\u4f1a\u62a5\u5982\u4e0b\u9519\u8bef\uff1a org.apache.hadoop.security.AccessControlException: Permission denied: user=test, access=READ_EXECUTE, resourcePath=\"bucket-test-hangzhou/\"","title":"8. \u8bbf\u95eeOSS\u3002"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_dls/","text":"Presto \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002 \u6b65\u9aa4 \u00a7 1. \u5b89\u88c5 jar \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/ 2. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e Access Key \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 3. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 \u4f7f\u7528\u793a\u4f8b \u00a7 \u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236\u53f0 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='oss://<Bucket>.<Endpoint>/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"Presto \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_dls/#presto-oss-hdfs-jindofs-jindofsx","text":"Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002","title":"Presto \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_dls/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_dls/#1-jar","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/","title":"1. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_dls/#2-jindosdk","text":"\u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e Access Key \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"2. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_dls/#3-presto","text":"","title":"3. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_dls/#_2","text":"\u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236\u53f0 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='oss://<Bucket>.<Endpoint>/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"\u4f7f\u7528\u793a\u4f8b"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_jindo/","text":"Presto \u67e5\u8be2 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e \u00a7 Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002 \u6b65\u9aa4 \u00a7 1. \u5b89\u88c5 jar \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/ 2. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e Access Key \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 3. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 4. \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55 \u00a7 \u6302\u8f7d\u547d\u4ee4 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/ \u4f7f\u7528\u793a\u4f8b \u00a7 \u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236\u53f0 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='jindo://emr-header-1:8101/jindooss/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"Presto \u67e5\u8be2 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_jindo/#presto-jindofsx","text":"Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002","title":"Presto \u67e5\u8be2 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_jindo/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_jindo/#1-jar","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/","title":"1. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_jindo/#2-jindosdk","text":"\u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e Access Key \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"2. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_jindo/#3-presto","text":"","title":"3. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_jindo/#4-oss-oss-hdfs","text":"\u6302\u8f7d\u547d\u4ee4 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/","title":"4. \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_jindo/#_2","text":"\u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236\u53f0 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='jindo://emr-header-1:8101/jindooss/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"\u4f7f\u7528\u793a\u4f8b"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_oss/","text":"Presto \u67e5\u8be2\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002 \u6b65\u9aa4 \u00a7 1. \u5b89\u88c5 jar \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/ 2. \u914d\u7f6e JindoSDK \u00a7 *\u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key \u5c06 OSS \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 3. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 \u4f7f\u7528\u793a\u4f8b \u00a7 \u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236\u53f0 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='oss://<bucket>/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"Presto \u67e5\u8be2\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_oss/#presto-oss-jindofsx","text":"Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002","title":"Presto \u67e5\u8be2\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_oss/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_oss/#1-jar","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/","title":"1. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_oss/#2-jindosdk","text":"*\u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key \u5c06 OSS \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"2. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_oss/#3-presto","text":"","title":"3. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_oss/#_2","text":"\u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236\u53f0 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='oss://<bucket>/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"\u4f7f\u7528\u793a\u4f8b"},{"location":"backup/jindofsx/security/jindofsx_credential_provider/","text":"JindoSDK OSS Credential Provider \u914d\u7f6e \u00a7 \u57fa\u672c\u914d\u7f6e\u65b9\u5f0f \u00a7 \u5728\u7f13\u5b58\u6570\u636e\u65f6\uff0cJindoFSx \u670d\u52a1\u7aef\u9700\u8981\u80fd\u8bbf\u95ee\u88ab\u7f13\u5b58\u7684\u540e\u7aef\u5b58\u50a8\u7cfb\u7edf\uff0c\u83b7\u53d6\u6240\u9700\u6570\u636e\u3002\u6839\u636e\u60a8\u53ef\u4ee5\u6839\u636e\u8bbf\u95ee\u7684\u5b58\u50a8\u7c7b\u578b\uff0c\u60a8\u53ef\u4ee5\u53c2\u8003\u76f8\u5e94\u7684\u6587\u6863\u5bf9 JindoFSx \u670d\u52a1\u8bbf\u95ee\u540e\u7aef\u5b58\u50a8\u7684 Credential \u548c Endpoint \u8fdb\u884c\u914d\u7f6e\uff0c\u914d\u7f6e\u652f\u6301\u660e\u786e\u5230 bucket \u7ea7\u522b\u3002 \u670d\u52a1\u7aef\u8bbf\u95ee\u963f\u91cc\u4e91 OSS \u00a7 JindoFSx \u670d\u52a1\u7aef\u652f\u6301\u5bf9 OSS \u534f\u8bae\u7684\u6240\u6709\u8bbf\u95ee\u914d\u7f6e\u7edf\u4e00\u7684 Credential \u4fe1\u606f\uff0c\u5982\u4e0b\u6240\u793a\uff1a [jindofsx-common] jindofsx.oss.accessKeyId = xxx jindofsx.oss.accessKeySecret = xxx jindofsx.oss.endpoint = oss-cn-xxx-internal.aliyuncs.com \u60a8\u4e5f\u53ef\u4ee5\u4e3a\u6bcf\u4e2a bucket \u914d\u7f6e\u72ec\u7acb\u7684 Credential \u4fe1\u606f\u548c Endpoint\uff1a jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = oss-cn-xxx-internal.aliyuncs.com \u670d\u52a1\u7aef\u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 JindoFSx \u670d\u52a1\u7aef\u652f\u6301\u5bf9 OSS \u534f\u8bae\u7684\u6240\u6709\u8bbf\u95ee\u914d\u7f6e\u7edf\u4e00\u7684 Credential \u4fe1\u606f\uff0c\u5982\u4e0b\u6240\u793a\uff1a [jindofsx-common] jindofsx.oss.accessKeyId = xxx jindofsx.oss.accessKeySecret = xxx jindofsx.oss.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.user = user #Storage Service \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u4f7f\u7528\u7684\u7528\u6237\u540d \u60a8\u4e5f\u53ef\u4ee5\u4e3a\u6bcf\u4e2a bucket \u914d\u7f6e\u72ec\u7acb\u7684 Credential \u4fe1\u606f\u548c Endpoint\uff0c\u5c24\u5176\u662f\u5728\u540c\u65f6\u4f7f\u7528 OSS \u670d\u52a1\u4e0e OSS-HDFS \u670d\u52a1\u65f6\uff0c\u9700\u8981\u5206\u522b\u914d\u7f6e\u4e0d\u540c\u7684 bucket\uff1a [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = cn-xxx.oss-dls.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = cn-xxx.oss-dls.aliyuncs.com jindofsx.oss.bucket.YYY.user = user #Storage Service \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u4f7f\u7528\u7684\u7528\u6237\u540d \u670d\u52a1\u7aef\u8bbf\u95ee Apache HDFS \u00a7 JindoFSx \u670d\u52a1\u7aef\u9700\u8981\u4e0e HDFS \u96c6\u7fa4\u8fdb\u884c\u7f51\u7edc\u901a\u4fe1\uff0c\u5e76\u6839\u636e\u76f8\u5e94 HDFS \u96c6\u7fa4\u7684\u914d\u7f6e\u8bbe\u7f6e HDFS \u96c6\u7fa4\u7684\u8bbf\u95ee\u4fe1\u606f\u3002\u5982\u679c\u8bbf\u95ee\u5355 master \u7684 HDFS \u96c6\u7fa4\uff0c\u9700\u8981\u914d\u7f6e\u8bbf\u95ee\u7528\u6237\u540d\uff1a [jindofsx-common] jindofsx.hdfs.user = user \u5982\u9700\u8bbf\u95ee nameservice \u914d\u7f6e\u4e3a mycluster \u7684 HDFS HA \u96c6\u7fa4\uff0c\u5219\u914d\u7f6e\u5982\u4e0b\u6240\u793a\uff1a [jindofsx-common] jindofsx.hdfs.mycluster.dfs.ha.namenodes = nn1,nn2 jindofsx.hdfs.mycluster.dfs.namenode.rpc-address.nn1 = <nn1-host>:8998 jindofsx.hdfs.mycluster.dfs.namenode.rpc-address.nn2 = <nn2-host>:8999 jindofsx.hdfs.user = user \u670d\u52a1\u7aef\u8bbf\u95ee NAS \u00a7 JindoFSx \u4e0e JindoSDK \u8981\u6c42 NAS \u670d\u52a1\u5df2\u7ecf mount \u5230\u6240\u6709\u8282\u70b9\u7684\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\uff0c\u5e76\u4fdd\u8bc1\u670d\u52a1\u548c\u5ba2\u6237\u7aef\u90fd\u5bf9\u6302\u8f7d\u76ee\u5f55\u6709\u8bfb\u6743\u9650\u3002 \u670d\u52a1\u7aef\u8bbf\u95ee\u5176\u4ed6\u4e91\u5b58\u50a8\uff08S3/COS/OBS\u7b49\uff09 \u00a7 JindoFSx \u670d\u52a1\u7aef\u652f\u6301\u5bf9\u67d0\u79cd\u534f\u8bae\u7684\u6240\u6709\u8bbf\u95ee\u914d\u7f6e\u7edf\u4e00\u7684 Credential \u4fe1\u606f\uff0c\u5982\u4e0b\u6240\u793a\uff1a [jindofsx-common] jindofsx.s3.accessKeyId = xxx jindofsx.s3.accessKeySecret = xxx jindofsx.s3.endpoint = s3.us-west-1.amazonaws.com jindofsx.cos.accessKeyId = xxx jindofsx.cos.accessKeySecret = xxx jindofsx.cos.endpoint = cos.ap-shanghai.myqcloud.com jindofsx.obs.accessKeyId = xxx jindofsx.obs.accessKeySecret = xxx jindofsx.obs.endpoint = obs.cn-east-3.myhuaweicloud.com \u60a8\u4e5f\u53ef\u4ee5\u4e3a\u6bcf\u4e2a bucket \u914d\u7f6e\u72ec\u7acb\u7684 Credential \u4fe1\u606f\u548c Endpoint\uff0c\u4ee5\u4e0b\u4ee5 S3 \u4e3a\u4f8b\uff1a jindofsx.s3.bucket.XXX.accessKeyId = xxx jindofsx.s3.bucket.XXX.accessKeySecret = xxx jindofsx.s3.bucket.XXX.endpoint = s3.us-west-1.amazonaws.com jindofsx.s3.bucket.YYY.accessKeyId = xxx jindofsx.s3.bucket.YYY.accessKeySecret = xxx jindofsx.s3.bucket.YYY.endpoint = s3.ap-east-1.amazonaws.com \u670d\u52a1\u7aef\u8bbf\u95ee\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u4e0b\u7684\u5b58\u50a8 \u00a7 \u6839\u636e\u5bf9\u5e94\u7684\u5b9e\u9645\u5b58\u50a8\u7cfb\u7edf\uff0c\u8fdb\u884c\u914d\u7f6e\u5373\u53ef\u3002 \u670d\u52a1\u7aef\u4f7f\u7528 JindoSDK OSS Credential Provider \u00a7 \u5f53\u524d\u7248\u672c\u6682\u4e0d\u652f\u6301\u5728 JindoFSx \u670d\u52a1\u7aef\u4f7f\u7528 JindoSDK OSS Credential Provider\u3002","title":"JindoSDK OSS Credential Provider \u914d\u7f6e"},{"location":"backup/jindofsx/security/jindofsx_credential_provider/#jindosdk-oss-credential-provider","text":"","title":"JindoSDK OSS Credential Provider \u914d\u7f6e"},{"location":"backup/jindofsx/security/jindofsx_credential_provider/#_1","text":"\u5728\u7f13\u5b58\u6570\u636e\u65f6\uff0cJindoFSx \u670d\u52a1\u7aef\u9700\u8981\u80fd\u8bbf\u95ee\u88ab\u7f13\u5b58\u7684\u540e\u7aef\u5b58\u50a8\u7cfb\u7edf\uff0c\u83b7\u53d6\u6240\u9700\u6570\u636e\u3002\u6839\u636e\u60a8\u53ef\u4ee5\u6839\u636e\u8bbf\u95ee\u7684\u5b58\u50a8\u7c7b\u578b\uff0c\u60a8\u53ef\u4ee5\u53c2\u8003\u76f8\u5e94\u7684\u6587\u6863\u5bf9 JindoFSx \u670d\u52a1\u8bbf\u95ee\u540e\u7aef\u5b58\u50a8\u7684 Credential \u548c Endpoint \u8fdb\u884c\u914d\u7f6e\uff0c\u914d\u7f6e\u652f\u6301\u660e\u786e\u5230 bucket \u7ea7\u522b\u3002","title":"\u57fa\u672c\u914d\u7f6e\u65b9\u5f0f"},{"location":"backup/jindofsx/security/jindofsx_credential_provider/#oss","text":"JindoFSx \u670d\u52a1\u7aef\u652f\u6301\u5bf9 OSS \u534f\u8bae\u7684\u6240\u6709\u8bbf\u95ee\u914d\u7f6e\u7edf\u4e00\u7684 Credential \u4fe1\u606f\uff0c\u5982\u4e0b\u6240\u793a\uff1a [jindofsx-common] jindofsx.oss.accessKeyId = xxx jindofsx.oss.accessKeySecret = xxx jindofsx.oss.endpoint = oss-cn-xxx-internal.aliyuncs.com \u60a8\u4e5f\u53ef\u4ee5\u4e3a\u6bcf\u4e2a bucket \u914d\u7f6e\u72ec\u7acb\u7684 Credential \u4fe1\u606f\u548c Endpoint\uff1a jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = oss-cn-xxx-internal.aliyuncs.com","title":"\u670d\u52a1\u7aef\u8bbf\u95ee\u963f\u91cc\u4e91 OSS"},{"location":"backup/jindofsx/security/jindofsx_credential_provider/#oss-hdfs-jindofs","text":"JindoFSx \u670d\u52a1\u7aef\u652f\u6301\u5bf9 OSS \u534f\u8bae\u7684\u6240\u6709\u8bbf\u95ee\u914d\u7f6e\u7edf\u4e00\u7684 Credential \u4fe1\u606f\uff0c\u5982\u4e0b\u6240\u793a\uff1a [jindofsx-common] jindofsx.oss.accessKeyId = xxx jindofsx.oss.accessKeySecret = xxx jindofsx.oss.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.user = user #Storage Service \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u4f7f\u7528\u7684\u7528\u6237\u540d \u60a8\u4e5f\u53ef\u4ee5\u4e3a\u6bcf\u4e2a bucket \u914d\u7f6e\u72ec\u7acb\u7684 Credential \u4fe1\u606f\u548c Endpoint\uff0c\u5c24\u5176\u662f\u5728\u540c\u65f6\u4f7f\u7528 OSS \u670d\u52a1\u4e0e OSS-HDFS \u670d\u52a1\u65f6\uff0c\u9700\u8981\u5206\u522b\u914d\u7f6e\u4e0d\u540c\u7684 bucket\uff1a [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = cn-xxx.oss-dls.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = cn-xxx.oss-dls.aliyuncs.com jindofsx.oss.bucket.YYY.user = user #Storage Service \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u4f7f\u7528\u7684\u7528\u6237\u540d","title":"\u670d\u52a1\u7aef\u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofsx/security/jindofsx_credential_provider/#apache-hdfs","text":"JindoFSx \u670d\u52a1\u7aef\u9700\u8981\u4e0e HDFS \u96c6\u7fa4\u8fdb\u884c\u7f51\u7edc\u901a\u4fe1\uff0c\u5e76\u6839\u636e\u76f8\u5e94 HDFS \u96c6\u7fa4\u7684\u914d\u7f6e\u8bbe\u7f6e HDFS \u96c6\u7fa4\u7684\u8bbf\u95ee\u4fe1\u606f\u3002\u5982\u679c\u8bbf\u95ee\u5355 master \u7684 HDFS \u96c6\u7fa4\uff0c\u9700\u8981\u914d\u7f6e\u8bbf\u95ee\u7528\u6237\u540d\uff1a [jindofsx-common] jindofsx.hdfs.user = user \u5982\u9700\u8bbf\u95ee nameservice \u914d\u7f6e\u4e3a mycluster \u7684 HDFS HA \u96c6\u7fa4\uff0c\u5219\u914d\u7f6e\u5982\u4e0b\u6240\u793a\uff1a [jindofsx-common] jindofsx.hdfs.mycluster.dfs.ha.namenodes = nn1,nn2 jindofsx.hdfs.mycluster.dfs.namenode.rpc-address.nn1 = <nn1-host>:8998 jindofsx.hdfs.mycluster.dfs.namenode.rpc-address.nn2 = <nn2-host>:8999 jindofsx.hdfs.user = user","title":"\u670d\u52a1\u7aef\u8bbf\u95ee Apache HDFS"},{"location":"backup/jindofsx/security/jindofsx_credential_provider/#nas","text":"JindoFSx \u4e0e JindoSDK \u8981\u6c42 NAS \u670d\u52a1\u5df2\u7ecf mount \u5230\u6240\u6709\u8282\u70b9\u7684\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\uff0c\u5e76\u4fdd\u8bc1\u670d\u52a1\u548c\u5ba2\u6237\u7aef\u90fd\u5bf9\u6302\u8f7d\u76ee\u5f55\u6709\u8bfb\u6743\u9650\u3002","title":"\u670d\u52a1\u7aef\u8bbf\u95ee NAS"},{"location":"backup/jindofsx/security/jindofsx_credential_provider/#s3cosobs","text":"JindoFSx \u670d\u52a1\u7aef\u652f\u6301\u5bf9\u67d0\u79cd\u534f\u8bae\u7684\u6240\u6709\u8bbf\u95ee\u914d\u7f6e\u7edf\u4e00\u7684 Credential \u4fe1\u606f\uff0c\u5982\u4e0b\u6240\u793a\uff1a [jindofsx-common] jindofsx.s3.accessKeyId = xxx jindofsx.s3.accessKeySecret = xxx jindofsx.s3.endpoint = s3.us-west-1.amazonaws.com jindofsx.cos.accessKeyId = xxx jindofsx.cos.accessKeySecret = xxx jindofsx.cos.endpoint = cos.ap-shanghai.myqcloud.com jindofsx.obs.accessKeyId = xxx jindofsx.obs.accessKeySecret = xxx jindofsx.obs.endpoint = obs.cn-east-3.myhuaweicloud.com \u60a8\u4e5f\u53ef\u4ee5\u4e3a\u6bcf\u4e2a bucket \u914d\u7f6e\u72ec\u7acb\u7684 Credential \u4fe1\u606f\u548c Endpoint\uff0c\u4ee5\u4e0b\u4ee5 S3 \u4e3a\u4f8b\uff1a jindofsx.s3.bucket.XXX.accessKeyId = xxx jindofsx.s3.bucket.XXX.accessKeySecret = xxx jindofsx.s3.bucket.XXX.endpoint = s3.us-west-1.amazonaws.com jindofsx.s3.bucket.YYY.accessKeyId = xxx jindofsx.s3.bucket.YYY.accessKeySecret = xxx jindofsx.s3.bucket.YYY.endpoint = s3.ap-east-1.amazonaws.com","title":"\u670d\u52a1\u7aef\u8bbf\u95ee\u5176\u4ed6\u4e91\u5b58\u50a8\uff08S3/COS/OBS\u7b49\uff09"},{"location":"backup/jindofsx/security/jindofsx_credential_provider/#_2","text":"\u6839\u636e\u5bf9\u5e94\u7684\u5b9e\u9645\u5b58\u50a8\u7cfb\u7edf\uff0c\u8fdb\u884c\u914d\u7f6e\u5373\u53ef\u3002","title":"\u670d\u52a1\u7aef\u8bbf\u95ee\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u4e0b\u7684\u5b58\u50a8"},{"location":"backup/jindofsx/security/jindofsx_credential_provider/#jindosdk-oss-credential-provider_1","text":"\u5f53\u524d\u7248\u672c\u6682\u4e0d\u652f\u6301\u5728 JindoFSx \u670d\u52a1\u7aef\u4f7f\u7528 JindoSDK OSS Credential Provider\u3002","title":"\u670d\u52a1\u7aef\u4f7f\u7528 JindoSDK OSS Credential Provider"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_dls/","text":"Spark \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\uff0c\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fdd Spark \u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/ 3. \u914d\u7f6e JindoSDK \u00a7 \u5168\u5c40\u914d\u7f6e \u00a7 \u4fee\u6539 Spark \u4f7f\u7528\u7684 core-site.xml \u3002 * \u914d\u7f6e JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u4efb\u52a1\u7ea7\u522b\u914d\u7f6e \u00a7 \u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx --conf spark.hadoop.fs.jindofsx.namespace.rpc.address=hostname:port --conf spark.hadoop.fs.jindofsx.data.cache.enable=true 4. \u4f7f\u7528 Spark \u8bbf\u95ee OSS \u00a7 \u521b\u5efa\u8868 create table test_oss (c1 string) location \"oss://test-bucket.<Endpoint>/<path>\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss; 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Spark \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_dls/#spark-oss-hdfs-jindofs-jindofsx","text":"JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\uff0c\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Spark \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_dls/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_dls/#1-spark-hadoop","text":"\u786e\u4fdd Spark \u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_dls/#2-spark-classpath-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/","title":"2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_dls/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_dls/#_2","text":"\u4fee\u6539 Spark \u4f7f\u7528\u7684 core-site.xml \u3002 * \u914d\u7f6e JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"\u5168\u5c40\u914d\u7f6e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_dls/#_3","text":"\u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx --conf spark.hadoop.fs.jindofsx.namespace.rpc.address=hostname:port --conf spark.hadoop.fs.jindofsx.data.cache.enable=true","title":"\u4efb\u52a1\u7ea7\u522b\u914d\u7f6e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_dls/#4-spark-oss","text":"\u521b\u5efa\u8868 create table test_oss (c1 string) location \"oss://test-bucket.<Endpoint>/<path>\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss;","title":"4. \u4f7f\u7528 Spark \u8bbf\u95ee OSS"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_dls/#5","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/","text":"Spark \u5904\u7406 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e \u00a7 JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411Hadoop/Spark\u751f\u6001\u7684OSS\u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684Hadoop FileSystem\u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fddSpark\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/ 3. \u914d\u7f6e JindoSDK \u00a7 \u5168\u5c40\u914d\u7f6e \u00a7 \u4fee\u6539 Spark \u4f7f\u7528\u7684 core-site.xml \u3002 \u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06 OSS \u6216 OSS-HDFS \u670d\u52a1 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 <configuration> <property> <!-- \u5185\u5b58\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u4efb\u52a1\u7ea7\u522b\u914d\u7f6e \u00a7 \u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx --conf spark.hadoop.fs.oss.endpoint=oss-cn-xxx-internal.aliyuncs.com --conf spark.hadoop.fs.jindofsx.namespace.rpc.address=hostname:port --conf spark.hadoop.fs.jindofsx.data.cache.enable=true 4. \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55 \u00a7 \u6302\u8f7d\u547d\u4ee4 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/ 5. \u4f7f\u7528 Spark \u8bbf\u95ee OSS \u00a7 \u521b\u5efa\u8868 create table test_oss (c1 string) location \"jindo://emr-header-1:8101/jindooss/<path>\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss; \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Spark \u5904\u7406 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/#spark-jindofsx","text":"JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411Hadoop/Spark\u751f\u6001\u7684OSS\u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684Hadoop FileSystem\u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Spark \u5904\u7406 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/#1-spark-hadoop","text":"\u786e\u4fddSpark\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/#2-spark-classpath-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/","title":"2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/#_2","text":"\u4fee\u6539 Spark \u4f7f\u7528\u7684 core-site.xml \u3002 \u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06 OSS \u6216 OSS-HDFS \u670d\u52a1 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 <configuration> <property> <!-- \u5185\u5b58\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"\u5168\u5c40\u914d\u7f6e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/#_3","text":"\u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx --conf spark.hadoop.fs.oss.endpoint=oss-cn-xxx-internal.aliyuncs.com --conf spark.hadoop.fs.jindofsx.namespace.rpc.address=hostname:port --conf spark.hadoop.fs.jindofsx.data.cache.enable=true","title":"\u4efb\u52a1\u7ea7\u522b\u914d\u7f6e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/#4-oss-oss-hdfs","text":"\u6302\u8f7d\u547d\u4ee4 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/","title":"4. \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/#5-spark-oss","text":"\u521b\u5efa\u8868 create table test_oss (c1 string) location \"jindo://emr-header-1:8101/jindooss/<path>\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss;","title":"5. \u4f7f\u7528 Spark \u8bbf\u95ee OSS"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/#_4","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_oss/","text":"Spark \u5904\u7406\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411Hadoop/Spark\u751f\u6001\u7684OSS\u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684Hadoop FileSystem\u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fddSpark\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/ 3. \u914d\u7f6e JindoSDK \u00a7 \u5168\u5c40\u914d\u7f6e \u00a7 \u4fee\u6539 Spark \u4f7f\u7528\u7684 core-site.xml \u3002 \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 <configuration> <property> <!-- \u5185\u5b58\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u4efb\u52a1\u7ea7\u522b\u914d\u7f6e \u00a7 \u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx --conf spark.hadoop.fs.oss.endpoint=oss-cn-xxx-internal.aliyuncs.com --conf spark.hadoop.fs.jindofsx.namespace.rpc.address=hostname:port --conf spark.hadoop.fs.jindofsx.data.cache.enable=true 4. \u4f7f\u7528 Spark \u8bbf\u95ee OSS \u00a7 \u521b\u5efa\u8868 create table test_oss (c1 string) location \"oss://<Bucket>/<path>\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss; 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Spark \u5904\u7406\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_oss/#spark-oss-jindofsx","text":"JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411Hadoop/Spark\u751f\u6001\u7684OSS\u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684Hadoop FileSystem\u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Spark \u5904\u7406\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_oss/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_oss/#1-spark-hadoop","text":"\u786e\u4fddSpark\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_oss/#2-spark-classpath-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/","title":"2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_oss/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_oss/#_2","text":"\u4fee\u6539 Spark \u4f7f\u7528\u7684 core-site.xml \u3002 \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 <configuration> <property> <!-- \u5185\u5b58\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"\u5168\u5c40\u914d\u7f6e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_oss/#_3","text":"\u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx --conf spark.hadoop.fs.oss.endpoint=oss-cn-xxx-internal.aliyuncs.com --conf spark.hadoop.fs.jindofsx.namespace.rpc.address=hostname:port --conf spark.hadoop.fs.jindofsx.data.cache.enable=true","title":"\u4efb\u52a1\u7ea7\u522b\u914d\u7f6e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_oss/#4-spark-oss","text":"\u521b\u5efa\u8868 create table test_oss (c1 string) location \"oss://<Bucket>/<path>\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss;","title":"4. \u4f7f\u7528 Spark \u8bbf\u95ee OSS"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_oss/#5","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindosdk/jindosdk_memory_best_practice/","text":"JindoSDk \u901a\u8fc7 jemalloc \u63a7\u5236\u5185\u5b58\u4e0a\u6da8\u6700\u4f73\u5b9e\u8df5 \u00a7 \u80cc\u666f \u00a7 jindosdk hadoop sdk \u4f7f\u7528 jni \u8c03\u7528 native \u5b9e\u73b0\uff0c\u56e0\u6b64 jvm \u76f8\u5173 gc \u53c2\u6570\u65e0\u6cd5\u6709\u6548\u63a7\u5236 native \u5185\u5b58\u4e0a\u6da8\uff0c\u53ef\u4ee5\u901a\u8fc7 jemalloc \u914d\u7f6e\u6765\u8fdb\u884c\u63a7\u5236\u3002 \u90e8\u5c5e\u65b9\u6cd5 \u00a7 jindosdk 4.6.8 \u53ca\u4ee5\u4e0a\u7248\u672c\u5728 tar \u5305\u4e2d\u9ed8\u8ba4\u643a\u5e26\u4e86 jemalloc\uff1a . \u251c\u2500\u2500 bin \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 conf \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 lib \u2502 \u251c\u2500\u2500 jindo-core-4.6.12.jar \u2502 \u251c\u2500\u2500 jindo-sdk-4.6.12.jar \u2502 \u2514\u2500\u2500 native \u2502 \u251c\u2500\u2500 libjemalloc.so \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 plugins \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 tools \u2502 \u251c\u2500\u2500 ... \u2514\u2500\u2500 versions \u2514\u2500\u2500 JINDOSDK_VERSION \u5047\u8bbe\u5b89\u88c5\u76ee\u5f55\u4e3a/opt/apps/JINDOSDK/jindosdk-current/\uff0c\u8981\u4fdd\u8bc1\u6240\u6709\u8c03\u7528jindosdk\u7684\u73af\u5883\uff0c\u90fd\u914d\u7f6e\u4ee5\u4e0b\u7684\u73af\u5883\u53d8\u91cf\uff1a LD_PRELOAD=/opt/apps/JINDOSDK/jindosdk-current/lib/native/libjemalloc.so MALLOC_CONF=narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0,background_thread:true,abort_conf:true \u6548\u679c \u00a7 wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/memcheck/jindo-sdk-memleak-check-1.0.jar \u6267\u884c\u4ee5\u4e0bmemleak-check\u6d4b\u8bd5 HADOOP_HEAPSIZE=2560 HADOOP_ROOT_LOGGER=ERROR,console hadoop jar jindo-sdk-memleak-check-1.0.jar MemLeakTestProgram -baseDir oss://yanbin-hd2-test/ -duration 8899 \u4f7f\u7528\u9ed8\u8ba4glibc\uff0c\u6700\u7ec8\u5360\u75283627M \u4f7f\u7528jemalloc\u4e0d\u8bbe\u7f6eMALLOC_CONF\uff0c\u6700\u7ec8\u5360\u75282929M \u4f7f\u7528jemalloc\u5e76\u8bbe\u7f6eMALLOC_CONF\uff0c\u6700\u7ec8\u5360\u75282289M","title":"JindoSDk \u901a\u8fc7 jemalloc \u63a7\u5236\u5185\u5b58\u4e0a\u6da8\u6700\u4f73\u5b9e\u8df5"},{"location":"backup/jindosdk/jindosdk_memory_best_practice/#jindosdk-jemalloc","text":"","title":"JindoSDk \u901a\u8fc7 jemalloc \u63a7\u5236\u5185\u5b58\u4e0a\u6da8\u6700\u4f73\u5b9e\u8df5"},{"location":"backup/jindosdk/jindosdk_memory_best_practice/#_1","text":"jindosdk hadoop sdk \u4f7f\u7528 jni \u8c03\u7528 native \u5b9e\u73b0\uff0c\u56e0\u6b64 jvm \u76f8\u5173 gc \u53c2\u6570\u65e0\u6cd5\u6709\u6548\u63a7\u5236 native \u5185\u5b58\u4e0a\u6da8\uff0c\u53ef\u4ee5\u901a\u8fc7 jemalloc \u914d\u7f6e\u6765\u8fdb\u884c\u63a7\u5236\u3002","title":"\u80cc\u666f"},{"location":"backup/jindosdk/jindosdk_memory_best_practice/#_2","text":"jindosdk 4.6.8 \u53ca\u4ee5\u4e0a\u7248\u672c\u5728 tar \u5305\u4e2d\u9ed8\u8ba4\u643a\u5e26\u4e86 jemalloc\uff1a . \u251c\u2500\u2500 bin \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 conf \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 lib \u2502 \u251c\u2500\u2500 jindo-core-4.6.12.jar \u2502 \u251c\u2500\u2500 jindo-sdk-4.6.12.jar \u2502 \u2514\u2500\u2500 native \u2502 \u251c\u2500\u2500 libjemalloc.so \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 plugins \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 tools \u2502 \u251c\u2500\u2500 ... \u2514\u2500\u2500 versions \u2514\u2500\u2500 JINDOSDK_VERSION \u5047\u8bbe\u5b89\u88c5\u76ee\u5f55\u4e3a/opt/apps/JINDOSDK/jindosdk-current/\uff0c\u8981\u4fdd\u8bc1\u6240\u6709\u8c03\u7528jindosdk\u7684\u73af\u5883\uff0c\u90fd\u914d\u7f6e\u4ee5\u4e0b\u7684\u73af\u5883\u53d8\u91cf\uff1a LD_PRELOAD=/opt/apps/JINDOSDK/jindosdk-current/lib/native/libjemalloc.so MALLOC_CONF=narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0,background_thread:true,abort_conf:true","title":"\u90e8\u5c5e\u65b9\u6cd5"},{"location":"backup/jindosdk/jindosdk_memory_best_practice/#_3","text":"wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/memcheck/jindo-sdk-memleak-check-1.0.jar \u6267\u884c\u4ee5\u4e0bmemleak-check\u6d4b\u8bd5 HADOOP_HEAPSIZE=2560 HADOOP_ROOT_LOGGER=ERROR,console hadoop jar jindo-sdk-memleak-check-1.0.jar MemLeakTestProgram -baseDir oss://yanbin-hd2-test/ -duration 8899 \u4f7f\u7528\u9ed8\u8ba4glibc\uff0c\u6700\u7ec8\u5360\u75283627M \u4f7f\u7528jemalloc\u4e0d\u8bbe\u7f6eMALLOC_CONF\uff0c\u6700\u7ec8\u5360\u75282929M \u4f7f\u7528jemalloc\u5e76\u8bbe\u7f6eMALLOC_CONF\uff0c\u6700\u7ec8\u5360\u75282289M","title":"\u6548\u679c"},{"location":"backup/jindosdk/outline/","text":"JindoSDK \u591a\u4e91\u5b58\u50a8 \u00a7 JindoSDK \u652f\u6301 S3\u3001COS\u3001OBS \u591a\u4e91\u5b58\u50a8\u3002 \u5feb\u901f\u5165\u95e8 \u00a7 JindoSDK + S3 \u5feb\u901f\u5165\u95e8 JindoSDK + COS \u5feb\u901f\u5165\u95e8 JindoSDK + OBS \u5feb\u901f\u5165\u95e8 \u5386\u53f2\u7248\u672c \u00a7 JindoSDK \u591a\u4e91\u5b58\u50a8 4.6.0 \u7528\u6237\u6587\u6863 JindoSDK \u591a\u4e91\u5b58\u50a8 4.5.1 \u7528\u6237\u6587\u6863 JindoSDK \u591a\u4e91\u5b58\u50a8 4.5.0 \u7528\u6237\u6587\u6863 JindoSDK \u591a\u4e91\u5b58\u50a8 4.4.0 \u7528\u6237\u6587\u6863 JindoSDK \u591a\u4e91\u5b58\u50a8 4.3.0 \u7528\u6237\u6587\u6863","title":"JindoSDK \u591a\u4e91\u5b58\u50a8"},{"location":"backup/jindosdk/outline/#jindosdk","text":"JindoSDK \u652f\u6301 S3\u3001COS\u3001OBS \u591a\u4e91\u5b58\u50a8\u3002","title":"JindoSDK \u591a\u4e91\u5b58\u50a8"},{"location":"backup/jindosdk/outline/#_1","text":"JindoSDK + S3 \u5feb\u901f\u5165\u95e8 JindoSDK + COS \u5feb\u901f\u5165\u95e8 JindoSDK + OBS \u5feb\u901f\u5165\u95e8","title":"\u5feb\u901f\u5165\u95e8"},{"location":"backup/jindosdk/outline/#_2","text":"JindoSDK \u591a\u4e91\u5b58\u50a8 4.6.0 \u7528\u6237\u6587\u6863 JindoSDK \u591a\u4e91\u5b58\u50a8 4.5.1 \u7528\u6237\u6587\u6863 JindoSDK \u591a\u4e91\u5b58\u50a8 4.5.0 \u7528\u6237\u6587\u6863 JindoSDK \u591a\u4e91\u5b58\u50a8 4.4.0 \u7528\u6237\u6587\u6863 JindoSDK \u591a\u4e91\u5b58\u50a8 4.3.0 \u7528\u6237\u6587\u6863","title":"\u5386\u53f2\u7248\u672c"},{"location":"backup/jindosdk/cos/jindosdk_quickstart/","text":"JindoSDK + COS \u5feb\u901f\u5165\u95e8 \u00a7 JindoSDK \u5168\u9762\u517c\u5bb9 Hadoop FileSystem \u63a5\u53e3\uff0c\u63d0\u4f9b\u4e86\u66f4\u597d\u7684\u517c\u5bb9\u6027\u548c\u6613\u7528\u6027\u3002 \u76f8\u5bf9\u4e8e Hadoop \u793e\u533a COS \u5ba2\u6237\u7aef\u5b9e\u73b0\uff0c\u60a8\u4ecd\u7136\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e JindoSDK COS \u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u5c06 JindoSDK COS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.cos.impl</name> <value>com.aliyun.jindodata.cos.COS</value> </property> <property> <name>fs.cos.impl</name> <value>com.aliyun.jindodata.cos.JindoCosFileSystem</value> </property> </configuration> 4. \u914d\u7f6e COS Access Key \u00a7 \u5c06 Access Key \u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 \u914d\u7f6e COS Access Key \u00a7 \u5c06 COS bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.cos.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.cos.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.cos.endpoint</name> <value>xxx</value> </property> </configuration> \u6309\u7167 bucket \u914d\u7f6e Access Key \u00a7 JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 COS bucket \u914d\u7f6e\u4e0d\u540c\u7684 Access Key\u3002 <configuration> <property> <name>fs.cos.bucket.XXX.accessKeyId</name> <value>COS bucket \u7684 AccessKey Id</value> </property> <property> <name>fs.cos.bucket.XXX.accessKeySecret</name> <value>COS bucket \u7684A ccessKey Secret</value> </property> </configuration> \u8bf4\u660e XXX \u4e3a COS bucket \u540d\u79f0\u3002 5. \u4f7f\u7528 JindoSDK \u8bbf\u95ee COS \u00a7 \u7528 Hadoop Shell \u8bbf\u95ee COS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> cos://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls cos://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir cos://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm cos://<bucket>/<path>","title":"JindoSDK + COS \u5feb\u901f\u5165\u95e8"},{"location":"backup/jindosdk/cos/jindosdk_quickstart/#jindosdk-cos","text":"JindoSDK \u5168\u9762\u517c\u5bb9 Hadoop FileSystem \u63a5\u53e3\uff0c\u63d0\u4f9b\u4e86\u66f4\u597d\u7684\u517c\u5bb9\u6027\u548c\u6613\u7528\u6027\u3002 \u76f8\u5bf9\u4e8e Hadoop \u793e\u533a COS \u5ba2\u6237\u7aef\u5b9e\u73b0\uff0c\u60a8\u4ecd\u7136\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"JindoSDK + COS \u5feb\u901f\u5165\u95e8"},{"location":"backup/jindosdk/cos/jindosdk_quickstart/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindosdk/cos/jindosdk_quickstart/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindosdk/cos/jindosdk_quickstart/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindosdk/cos/jindosdk_quickstart/#3-jindosdk-cos-access-key","text":"\u5c06 JindoSDK COS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.cos.impl</name> <value>com.aliyun.jindodata.cos.COS</value> </property> <property> <name>fs.cos.impl</name> <value>com.aliyun.jindodata.cos.JindoCosFileSystem</value> </property> </configuration>","title":"3. \u914d\u7f6e JindoSDK COS \u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"backup/jindosdk/cos/jindosdk_quickstart/#4-cos-access-key","text":"\u5c06 Access Key \u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002","title":"4. \u914d\u7f6e COS Access Key"},{"location":"backup/jindosdk/cos/jindosdk_quickstart/#cos-access-key","text":"\u5c06 COS bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.cos.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.cos.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.cos.endpoint</name> <value>xxx</value> </property> </configuration>","title":"\u914d\u7f6e COS Access Key"},{"location":"backup/jindosdk/cos/jindosdk_quickstart/#bucket-access-key","text":"JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 COS bucket \u914d\u7f6e\u4e0d\u540c\u7684 Access Key\u3002 <configuration> <property> <name>fs.cos.bucket.XXX.accessKeyId</name> <value>COS bucket \u7684 AccessKey Id</value> </property> <property> <name>fs.cos.bucket.XXX.accessKeySecret</name> <value>COS bucket \u7684A ccessKey Secret</value> </property> </configuration> \u8bf4\u660e XXX \u4e3a COS bucket \u540d\u79f0\u3002","title":"\u6309\u7167 bucket \u914d\u7f6e Access Key"},{"location":"backup/jindosdk/cos/jindosdk_quickstart/#5-jindosdk-cos","text":"\u7528 Hadoop Shell \u8bbf\u95ee COS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> cos://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls cos://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir cos://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm cos://<bucket>/<path>","title":"5. \u4f7f\u7528 JindoSDK \u8bbf\u95ee COS"},{"location":"backup/jindosdk/obs/jindosdk_quickstart/","text":"JindoSDK + \u963f\u91cc\u4e91 OBS \u5feb\u901f\u5165\u95e8 \u00a7 JindoSDK \u5168\u9762\u517c\u5bb9 Hadoop FileSystem \u63a5\u53e3\uff0c\u63d0\u4f9b\u4e86\u66f4\u597d\u7684\u517c\u5bb9\u6027\u548c\u6613\u7528\u6027\u3002 \u76f8\u5bf9\u4e8e Hadoop \u793e\u533a OBS \u5ba2\u6237\u7aef\u5b9e\u73b0\uff0c\u60a8\u4ecd\u7136\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e JindoSDK OBS \u5b9e\u73b0\u7c7b \u00a7 \u5c06 JindoSDK OBS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.obs.impl</name> <value>com.aliyun.jindodata.obs.OBS</value> </property> <property> <name>fs.obs.impl</name> <value>com.aliyun.jindodata.obs.JindoObsFileSystem</value> </property> </configuration> 4. \u914d\u7f6e OBS Access Key \u00a7 \u5c06 Access Key \u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 \u914d\u7f6e OBS Access Key \u00a7 \u5c06 OBS bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.obs.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.obs.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.obs.endpoint</name> <value>xxx</value> </property> </configuration> \u6309\u7167 bucket \u914d\u7f6e Access Key \u00a7 JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 OBS bucket \u914d\u7f6e\u4e0d\u540c\u7684 Access Key\u3002 <configuration> <property> <name>fs.obs.bucket.XXX.accessKeyId</name> <value>OBS bucket \u7684 AccessKey Id</value> </property> <property> <name>fs.obs.bucket.XXX.accessKeySecret</name> <value>OBS bucket \u7684 AccessKey Secret</value> </property> </configuration> \u8bf4\u660e XXX \u4e3a OBS bucket \u540d\u79f0\u3002 5. \u4f7f\u7528 JindoSDK \u8bbf\u95ee OBS \u00a7 \u7528 Hadoop Shell \u8bbf\u95ee OBS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> obs://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls obs://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir obs://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm obs://<bucket>/<path>","title":"JindoSDK + \u963f\u91cc\u4e91 OBS \u5feb\u901f\u5165\u95e8"},{"location":"backup/jindosdk/obs/jindosdk_quickstart/#jindosdk-obs","text":"JindoSDK \u5168\u9762\u517c\u5bb9 Hadoop FileSystem \u63a5\u53e3\uff0c\u63d0\u4f9b\u4e86\u66f4\u597d\u7684\u517c\u5bb9\u6027\u548c\u6613\u7528\u6027\u3002 \u76f8\u5bf9\u4e8e Hadoop \u793e\u533a OBS \u5ba2\u6237\u7aef\u5b9e\u73b0\uff0c\u60a8\u4ecd\u7136\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"JindoSDK + \u963f\u91cc\u4e91 OBS \u5feb\u901f\u5165\u95e8"},{"location":"backup/jindosdk/obs/jindosdk_quickstart/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindosdk/obs/jindosdk_quickstart/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindosdk/obs/jindosdk_quickstart/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindosdk/obs/jindosdk_quickstart/#3-jindosdk-obs","text":"\u5c06 JindoSDK OBS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.obs.impl</name> <value>com.aliyun.jindodata.obs.OBS</value> </property> <property> <name>fs.obs.impl</name> <value>com.aliyun.jindodata.obs.JindoObsFileSystem</value> </property> </configuration>","title":"3. \u914d\u7f6e JindoSDK OBS \u5b9e\u73b0\u7c7b"},{"location":"backup/jindosdk/obs/jindosdk_quickstart/#4-obs-access-key","text":"\u5c06 Access Key \u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002","title":"4. \u914d\u7f6e OBS Access Key"},{"location":"backup/jindosdk/obs/jindosdk_quickstart/#obs-access-key","text":"\u5c06 OBS bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.obs.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.obs.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.obs.endpoint</name> <value>xxx</value> </property> </configuration>","title":"\u914d\u7f6e OBS Access Key"},{"location":"backup/jindosdk/obs/jindosdk_quickstart/#bucket-access-key","text":"JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 OBS bucket \u914d\u7f6e\u4e0d\u540c\u7684 Access Key\u3002 <configuration> <property> <name>fs.obs.bucket.XXX.accessKeyId</name> <value>OBS bucket \u7684 AccessKey Id</value> </property> <property> <name>fs.obs.bucket.XXX.accessKeySecret</name> <value>OBS bucket \u7684 AccessKey Secret</value> </property> </configuration> \u8bf4\u660e XXX \u4e3a OBS bucket \u540d\u79f0\u3002","title":"\u6309\u7167 bucket \u914d\u7f6e Access Key"},{"location":"backup/jindosdk/obs/jindosdk_quickstart/#5-jindosdk-obs","text":"\u7528 Hadoop Shell \u8bbf\u95ee OBS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> obs://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls obs://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir obs://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm obs://<bucket>/<path>","title":"5. \u4f7f\u7528 JindoSDK \u8bbf\u95ee OBS"},{"location":"backup/jindosdk/s3/jindosdk_quickstart/","text":"JindoSDK + S3 \u5feb\u901f\u5165\u95e8 \u00a7 JindoSDK \u5168\u9762\u517c\u5bb9 Hadoop FileSystem \u63a5\u53e3\uff0c\u63d0\u4f9b\u4e86\u66f4\u597d\u7684\u517c\u5bb9\u6027\u548c\u6613\u7528\u6027\u3002 \u76f8\u5bf9\u4e8e Hadoop \u793e\u533a S3 \u5ba2\u6237\u7aef\u5b9e\u73b0\uff0c\u60a8\u4ecd\u7136\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e JindoSDK S3 \u5b9e\u73b0\u7c7b \u00a7 \u5c06 JindoSDK S3 \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.s3.impl</name> <value>com.aliyun.jindodata.s3.S3</value> </property> <property> <name>fs.s3.impl</name> <value>com.aliyun.jindodata.s3.JindoS3FileSystem</value> </property> </configuration> 4. \u914d\u7f6e S3 Access Key \u00a7 \u5c06 Access Key \u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 \u914d\u7f6e S3 Access Key \u00a7 \u5c06 S3 bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.s3.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.s3.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.s3.endpoint</name> <value>xxx</value> </property> </configuration> \u6309\u7167 bucket \u914d\u7f6e Access Key \u00a7 JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 S3 bucket \u914d\u7f6e\u4e0d\u540c\u7684 Access Key\u3002 <configuration> <property> <name>fs.s3.bucket.XXX.accessKeyId</name> <value>S3 bucket \u7684 AccessKey Id</value> </property> <property> <name>fs.s3.bucket.XXX.accessKeySecret</name> <value>S3 bucket \u7684 AccessKey Secret</value> </property> </configuration> \u8bf4\u660e XXX \u4e3a S3 bucket \u540d\u79f0\u3002 5. \u4f7f\u7528 JindoSDK \u8bbf\u95ee S3 \u00a7 \u7528 Hadoop Shell \u8bbf\u95ee S3\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> s3://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls s3://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir s3://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm s3://<bucket>/<path>","title":"JindoSDK + S3 \u5feb\u901f\u5165\u95e8"},{"location":"backup/jindosdk/s3/jindosdk_quickstart/#jindosdk-s3","text":"JindoSDK \u5168\u9762\u517c\u5bb9 Hadoop FileSystem \u63a5\u53e3\uff0c\u63d0\u4f9b\u4e86\u66f4\u597d\u7684\u517c\u5bb9\u6027\u548c\u6613\u7528\u6027\u3002 \u76f8\u5bf9\u4e8e Hadoop \u793e\u533a S3 \u5ba2\u6237\u7aef\u5b9e\u73b0\uff0c\u60a8\u4ecd\u7136\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"JindoSDK + S3 \u5feb\u901f\u5165\u95e8"},{"location":"backup/jindosdk/s3/jindosdk_quickstart/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindosdk/s3/jindosdk_quickstart/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindosdk/s3/jindosdk_quickstart/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindosdk/s3/jindosdk_quickstart/#3-jindosdk-s3","text":"\u5c06 JindoSDK S3 \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.s3.impl</name> <value>com.aliyun.jindodata.s3.S3</value> </property> <property> <name>fs.s3.impl</name> <value>com.aliyun.jindodata.s3.JindoS3FileSystem</value> </property> </configuration>","title":"3. \u914d\u7f6e JindoSDK S3 \u5b9e\u73b0\u7c7b"},{"location":"backup/jindosdk/s3/jindosdk_quickstart/#4-s3-access-key","text":"\u5c06 Access Key \u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002","title":"4. \u914d\u7f6e S3 Access Key"},{"location":"backup/jindosdk/s3/jindosdk_quickstart/#s3-access-key","text":"\u5c06 S3 bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.s3.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.s3.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.s3.endpoint</name> <value>xxx</value> </property> </configuration>","title":"\u914d\u7f6e S3 Access Key"},{"location":"backup/jindosdk/s3/jindosdk_quickstart/#bucket-access-key","text":"JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 S3 bucket \u914d\u7f6e\u4e0d\u540c\u7684 Access Key\u3002 <configuration> <property> <name>fs.s3.bucket.XXX.accessKeyId</name> <value>S3 bucket \u7684 AccessKey Id</value> </property> <property> <name>fs.s3.bucket.XXX.accessKeySecret</name> <value>S3 bucket \u7684 AccessKey Secret</value> </property> </configuration> \u8bf4\u660e XXX \u4e3a S3 bucket \u540d\u79f0\u3002","title":"\u6309\u7167 bucket \u914d\u7f6e Access Key"},{"location":"backup/jindosdk/s3/jindosdk_quickstart/#5-jindosdk-s3","text":"\u7528 Hadoop Shell \u8bbf\u95ee S3\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> s3://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls s3://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir s3://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm s3://<bucket>/<path>","title":"5. \u4f7f\u7528 JindoSDK \u8bbf\u95ee S3"},{"location":"backup/oss/jindosdk_quickstart/","text":"JindoSDK + \u963f\u91cc\u4e91 OSS \u5feb\u901f\u5165\u95e8 \u00a7 JindoSDK \u5168\u9762\u517c\u5bb9 Hadoop FileSystem \u63a5\u53e3\uff0c\u63d0\u4f9b\u4e86\u66f4\u597d\u7684\u517c\u5bb9\u6027\u548c\u6613\u7528\u6027\uff0c\u53c8\u80fd\u517c\u5177OSS\u7684\u6027\u80fd\u548c\u6210\u672c\u4f18\u52bf\u3002 \u76f8\u5bf9\u4e8e Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\u5b9e\u73b0\uff0c\u60a8\u4ecd\u7136\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u76ee\u524d\u652f\u6301\u5e02\u9762\u4e0a\u5927\u90e8\u5206 Hadoop \u7248\u672c\uff0c\u5728 Hadoop 2.3 \u53ca\u4ee5\u4e0a\u7684\u7248\u672c\u4e0a\u9a8c\u8bc1\u901a\u8fc7\uff082.3 \u4ee5\u524d\u7248\u672c\u6682\u672a\u6d4b\u8bd5\uff0c\u5982\u6709\u95ee\u9898\u8bf7 \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988\uff09\u3002 \u5173\u4e8e JindoSDK \u548c Hadoop \u793e\u533a OSS connector \u7684\u6027\u80fd\u5bf9\u6bd4\uff0c\u8bf7\u53c2\u8003\u6587\u6863 JindoSDK\u548cHadoop-OSS-SDK\u6027\u80fd\u5bf9\u6bd4\u6d4b\u8bd5 \u3002 \u6b65\u9aa4 \u00a7 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06 OSS bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 4. \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u00a7 \u7528 Hadoop Shell \u8bbf\u95ee OSS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls oss://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<bucket>/<path> 5. \u6e05\u7406\u56de\u6536\u7ad9 \u00a7 Hadoop \u901a\u8fc7\u5c06\u5220\u9664\u7684\u6587\u4ef6\u6216\u76ee\u5f55\u653e\u5165\u56de\u6536\u7ad9\u6765\u9632\u6b62\u8bef\u5220\u6587\u4ef6\u6216\u6587\u4ef6\u5939\u3002\u5f53\u4f7f\u7528 Hadoop Shell \u5220\u9664 OSS \u7684\u6587\u4ef6\u6216\u76ee\u5f55\u65f6\uff0c \u82e5\u5f00\u542f\u4e86\u56de\u6536\u7ad9\u529f\u80fd\uff0c\u4f1a\u5728 OSS \u7684 bucket \u4e0a\u5b58\u50a8\u88ab\u5220\u9664\u7684\u6587\u4ef6\u6216\u76ee\u5f55\uff0c\u56de\u6536\u7ad9\u7684\u76ee\u5f55\u4e3a oss://buckect-name/user/<username>/.Trash \u3002\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e OSS \u6240\u4f7f\u7528\u7684\u56de\u6536\u7ad9\u76ee\u5f55\u7684\u751f\u547d\u5468\u671f\u6765\u6e05\u7406\u56de\u6536\u7ad9\u3002 \u5173\u4e8e\u5982\u4f55\u8bbe\u7f6e OSS \u751f\u547d\u5468\u671f\uff0c\u8bf7\u53c2\u8003 \u8bbe\u7f6e\u751f\u547d\u5468\u671f\u89c4\u5219 \u82e5\u8981\u8df3\u8fc7\u56de\u6536\u7ad9\uff0c\u76f4\u63a5\u5220\u9664\uff0c\u5219\u53ef\u4ee5\u6307\u5b9a -skipTrash \u53c2\u6570\uff08\u614e\u7528\uff09\u3002 hadoop fs -rm -skipTrash oss://<bucket>/<path> 6. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 7. \u6743\u9650\u7ba1\u7406 \u00a7 \u81ea\u5efa\u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u963f\u91cc\u4e91 E-MapReduce \u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848","title":"JindoSDK + \u963f\u91cc\u4e91 OSS \u5feb\u901f\u5165\u95e8"},{"location":"backup/oss/jindosdk_quickstart/#jindosdk-oss","text":"JindoSDK \u5168\u9762\u517c\u5bb9 Hadoop FileSystem \u63a5\u53e3\uff0c\u63d0\u4f9b\u4e86\u66f4\u597d\u7684\u517c\u5bb9\u6027\u548c\u6613\u7528\u6027\uff0c\u53c8\u80fd\u517c\u5177OSS\u7684\u6027\u80fd\u548c\u6210\u672c\u4f18\u52bf\u3002 \u76f8\u5bf9\u4e8e Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\u5b9e\u73b0\uff0c\u60a8\u4ecd\u7136\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u76ee\u524d\u652f\u6301\u5e02\u9762\u4e0a\u5927\u90e8\u5206 Hadoop \u7248\u672c\uff0c\u5728 Hadoop 2.3 \u53ca\u4ee5\u4e0a\u7684\u7248\u672c\u4e0a\u9a8c\u8bc1\u901a\u8fc7\uff082.3 \u4ee5\u524d\u7248\u672c\u6682\u672a\u6d4b\u8bd5\uff0c\u5982\u6709\u95ee\u9898\u8bf7 \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988\uff09\u3002 \u5173\u4e8e JindoSDK \u548c Hadoop \u793e\u533a OSS connector \u7684\u6027\u80fd\u5bf9\u6bd4\uff0c\u8bf7\u53c2\u8003\u6587\u6863 JindoSDK\u548cHadoop-OSS-SDK\u6027\u80fd\u5bf9\u6bd4\u6d4b\u8bd5 \u3002","title":"JindoSDK + \u963f\u91cc\u4e91 OSS \u5feb\u901f\u5165\u95e8"},{"location":"backup/oss/jindosdk_quickstart/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/oss/jindosdk_quickstart/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/oss/jindosdk_quickstart/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/oss/jindosdk_quickstart/#3-oss-access-key","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06 OSS bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002","title":"3. \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"backup/oss/jindosdk_quickstart/#4-jindosdk-oss","text":"\u7528 Hadoop Shell \u8bbf\u95ee OSS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls oss://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<bucket>/<path>","title":"4. \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/oss/jindosdk_quickstart/#5","text":"Hadoop \u901a\u8fc7\u5c06\u5220\u9664\u7684\u6587\u4ef6\u6216\u76ee\u5f55\u653e\u5165\u56de\u6536\u7ad9\u6765\u9632\u6b62\u8bef\u5220\u6587\u4ef6\u6216\u6587\u4ef6\u5939\u3002\u5f53\u4f7f\u7528 Hadoop Shell \u5220\u9664 OSS \u7684\u6587\u4ef6\u6216\u76ee\u5f55\u65f6\uff0c \u82e5\u5f00\u542f\u4e86\u56de\u6536\u7ad9\u529f\u80fd\uff0c\u4f1a\u5728 OSS \u7684 bucket \u4e0a\u5b58\u50a8\u88ab\u5220\u9664\u7684\u6587\u4ef6\u6216\u76ee\u5f55\uff0c\u56de\u6536\u7ad9\u7684\u76ee\u5f55\u4e3a oss://buckect-name/user/<username>/.Trash \u3002\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e OSS \u6240\u4f7f\u7528\u7684\u56de\u6536\u7ad9\u76ee\u5f55\u7684\u751f\u547d\u5468\u671f\u6765\u6e05\u7406\u56de\u6536\u7ad9\u3002 \u5173\u4e8e\u5982\u4f55\u8bbe\u7f6e OSS \u751f\u547d\u5468\u671f\uff0c\u8bf7\u53c2\u8003 \u8bbe\u7f6e\u751f\u547d\u5468\u671f\u89c4\u5219 \u82e5\u8981\u8df3\u8fc7\u56de\u6536\u7ad9\uff0c\u76f4\u63a5\u5220\u9664\uff0c\u5219\u53ef\u4ee5\u6307\u5b9a -skipTrash \u53c2\u6570\uff08\u614e\u7528\uff09\u3002 hadoop fs -rm -skipTrash oss://<bucket>/<path>","title":"5. \u6e05\u7406\u56de\u6536\u7ad9"},{"location":"backup/oss/jindosdk_quickstart/#6","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"6. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/oss/jindosdk_quickstart/#7","text":"\u81ea\u5efa\u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u963f\u91cc\u4e91 E-MapReduce \u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848","title":"7. \u6743\u9650\u7ba1\u7406"},{"location":"backup/oss/jindoshell_oss_howto/","text":"JindoShell CLI \u652f\u6301\u963f\u91cc\u4e91 OSS \u4f7f\u7528\u8bf4\u660e \u00a7 JindoShell CLI \u652f\u6301\u963f\u91cc\u4e91 OSS archive, unarchive, restore \u7b49\u547d\u4ee4\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ Archive \u547d\u4ee4 \u00a7 Archive\u547d\u4ee4\u53ef\u4ee5\u5f52\u6863OSS\u4e0a\u7684\u6570\u636e\u81f3\u4f4e\u9891\u8bbf\u95ee\u5b58\u50a8\u6216\u8005\u5f52\u6863\u5b58\u50a8\u4e0a\u3002 jindo oss -archive [-i|c] <path> -i\u53c2\u6570\u53ef\u4ee5\u5f52\u6863\u6570\u636e\u81f3OSS\u4f4e\u9891\u5b58\u50a8\u7c7b\u578b\u3002-c\u53c2\u6570\u53ef\u4ee5\u5f52\u6863\u6570\u636e\u81f3OSS\u51b7\u5f52\u6863\u5b58\u50a8\u7c7b\u578b\u3002\u9ed8\u8ba4\u53ef\u4ee5\u5f52\u6863\u6570\u636e\u81f3OSS\u5f52\u6863\u5b58\u50a8\u7c7b\u578b\u3002 Unarchive \u547d\u4ee4 \u00a7 Unarchive\u547d\u4ee4\u53ef\u4ee5\u5c06\u6570\u636e\u4ece\u5f52\u6863\u5b58\u50a8\u7c7b\u578b\u6062\u590d\u5230\u4f4e\u9891\u5b58\u50a8\u6216\u8005\u6807\u51c6\u5b58\u3002 jindo oss -unarchive [-i|a] <path> -i\u53c2\u6570\u53ef\u4ee5\u6062\u590d\u6570\u636e\u81f3OSS\u4f4e\u9891\u5b58\u50a8\u7c7b\u578b\u3002-a\u53c2\u6570\u53ef\u4ee5\u6062\u590d\u6570\u636e\u81f3OSS\u5f52\u6863\u5b58\u50a8\u7c7b\u578b\u3002\u9ed8\u8ba4\u53ef\u4ee5\u6062\u590d\u6570\u636e\u81f3OSS\u6807\u51c6\u5b58\u50a8\u3002 Restore \u547d\u4ee4 \u00a7 Restore \u547d\u4ee4\u53ef\u4ee5\u89e3\u51bb\u5f52\u6863\u7c7b\u578b\uff08Archive\uff09\u6216\u51b7\u5f52\u6863\uff08Cold Archive\uff09\u7684\u76ee\u5f55\u3002 jindo oss -restore [-days <days>] <path>","title":"JindoShell CLI \u652f\u6301\u963f\u91cc\u4e91 OSS \u4f7f\u7528\u8bf4\u660e"},{"location":"backup/oss/jindoshell_oss_howto/#jindoshell-cli-oss","text":"JindoShell CLI \u652f\u6301\u963f\u91cc\u4e91 OSS archive, unarchive, restore \u7b49\u547d\u4ee4\u3002","title":"JindoShell CLI \u652f\u6301\u963f\u91cc\u4e91 OSS \u4f7f\u7528\u8bf4\u660e"},{"location":"backup/oss/jindoshell_oss_howto/#_1","text":"","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/oss/jindoshell_oss_howto/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/oss/jindoshell_oss_howto/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/oss/jindoshell_oss_howto/#archive","text":"Archive\u547d\u4ee4\u53ef\u4ee5\u5f52\u6863OSS\u4e0a\u7684\u6570\u636e\u81f3\u4f4e\u9891\u8bbf\u95ee\u5b58\u50a8\u6216\u8005\u5f52\u6863\u5b58\u50a8\u4e0a\u3002 jindo oss -archive [-i|c] <path> -i\u53c2\u6570\u53ef\u4ee5\u5f52\u6863\u6570\u636e\u81f3OSS\u4f4e\u9891\u5b58\u50a8\u7c7b\u578b\u3002-c\u53c2\u6570\u53ef\u4ee5\u5f52\u6863\u6570\u636e\u81f3OSS\u51b7\u5f52\u6863\u5b58\u50a8\u7c7b\u578b\u3002\u9ed8\u8ba4\u53ef\u4ee5\u5f52\u6863\u6570\u636e\u81f3OSS\u5f52\u6863\u5b58\u50a8\u7c7b\u578b\u3002","title":"Archive \u547d\u4ee4"},{"location":"backup/oss/jindoshell_oss_howto/#unarchive","text":"Unarchive\u547d\u4ee4\u53ef\u4ee5\u5c06\u6570\u636e\u4ece\u5f52\u6863\u5b58\u50a8\u7c7b\u578b\u6062\u590d\u5230\u4f4e\u9891\u5b58\u50a8\u6216\u8005\u6807\u51c6\u5b58\u3002 jindo oss -unarchive [-i|a] <path> -i\u53c2\u6570\u53ef\u4ee5\u6062\u590d\u6570\u636e\u81f3OSS\u4f4e\u9891\u5b58\u50a8\u7c7b\u578b\u3002-a\u53c2\u6570\u53ef\u4ee5\u6062\u590d\u6570\u636e\u81f3OSS\u5f52\u6863\u5b58\u50a8\u7c7b\u578b\u3002\u9ed8\u8ba4\u53ef\u4ee5\u6062\u590d\u6570\u636e\u81f3OSS\u6807\u51c6\u5b58\u50a8\u3002","title":"Unarchive \u547d\u4ee4"},{"location":"backup/oss/jindoshell_oss_howto/#restore","text":"Restore \u547d\u4ee4\u53ef\u4ee5\u89e3\u51bb\u5f52\u6863\u7c7b\u578b\uff08Archive\uff09\u6216\u51b7\u5f52\u6863\uff08Cold Archive\uff09\u7684\u76ee\u5f55\u3002 jindo oss -restore [-days <days>] <path>","title":"Restore \u547d\u4ee4"},{"location":"backup/oss/outline/","text":"JindoSDK + \u963f\u91cc\u4e91 OSS \u7528\u6237\u6587\u6863 \u00a7 \u5feb\u901f\u5165\u95e8 \u00a7 JindoSDK + \u963f\u91cc\u4e91 OSS \u5feb\u901f\u5165\u95e8 Hadoop \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 \u57fa\u672c\u529f\u80fd \u00a7 JindoShell CLI \u652f\u6301\u963f\u91cc\u4e91 OSS \u4f7f\u7528\u8bf4\u660e \u5927\u6570\u636e\u751f\u6001 \u00a7 Spark \u7ec4\u4ef6 \u00a7 Spark \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e Hive \u7ec4\u4ef6 \u00a7 Hive \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e Presto \u7ec4\u4ef6 \u00a7 Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS \u4e0a\u7684\u6570\u636e Impala \u7ec4\u4ef6 \u00a7 Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS \u4e0a\u7684\u6570\u636e HBase \u7ec4\u4ef6 \u00a7 HBase \u4f7f\u7528 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 Flink \u7ec4\u4ef6 \u00a7 Flink \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e Flume \u7ec4\u4ef6 \u00a7 Flume \u4f7f\u7528 JindoSDK \u5199\u5165 OSS Sqoop \u7ec4\u4ef6 \u00a7 Sqoop \u4f7f\u7528 Kite SDK \u8bfb\u5199 OSS AI \u751f\u6001 \u00a7 JindoFuse \u4f7f\u7528 \u00a7 \u4f7f\u7528 JindoFuse \u8bbf\u95ee OSS \u6700\u4f73\u5b9e\u8df5 \u00a7 JindoDistCp \u6570\u636e\u8fc1\u79fb \u00a7 JindoDistCp \u6570\u636e\u8fc1\u79fb JindoTable \u6570\u4ed3\u8fc1\u79fb \u00a7 \u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a \u00a7 JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a \u793e\u533a\u6700\u4f73\u5b9e\u8df5 \u00a7 Flink-table \u4f7f\u7528 JindoSDK + OSS \u4ee3\u66ff HDFS \u5b58\u50a8\u5b9e\u8df5\uff08\u4f5c\u8005\uff1a\u674e\u4f1f\uff09 \u00a7 Flink-table \u4f7f\u7528 JindoSDK + OSS \u4ee3\u66ff HDFS \u5b58\u50a8\u5b9e\u8df5 \u76f8\u5173\u6587\u7ae0 \u00a7 \u4f7f\u7528 JindoSDK \u5927\u5e45\u63d0\u5347 OSS \u6587\u4ef6\u5404\u9879\u64cd\u4f5c\u6027\u80fd \u5e38\u89c1\u95ee\u9898 \u00a7 \u5e38\u89c1\u95ee\u9898 \u5386\u53f2\u7248\u672c \u00a7 JindoSDK 4.6.0 \u7528\u6237\u6587\u6863 \u00a7 JindoSDK 4.5.1 \u7528\u6237\u6587\u6863 \u00a7 JindoSDK 4.5.0 \u7528\u6237\u6587\u6863 \u00a7 JindoSDK 4.4.0 \u7528\u6237\u6587\u6863 \u00a7 JindoSDK 4.3.0 \u7528\u6237\u6587\u6863 \u00a7 JindoSDK 4.2.0 \u7528\u6237\u6587\u6863 \u00a7 JindoSDK 4.1.0 \u7528\u6237\u6587\u6863 \u00a7 JindoSDK 4.0.0 \u7528\u6237\u6587\u6863 \u00a7 JindoSDK 3.x + OSS \u7528\u6237\u6587\u6863 \u00a7","title":"JindoSDK + \u963f\u91cc\u4e91 OSS \u7528\u6237\u6587\u6863"},{"location":"backup/oss/outline/#jindosdk-oss","text":"","title":"JindoSDK + \u963f\u91cc\u4e91 OSS \u7528\u6237\u6587\u6863"},{"location":"backup/oss/outline/#_1","text":"JindoSDK + \u963f\u91cc\u4e91 OSS \u5feb\u901f\u5165\u95e8 Hadoop \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5","title":"\u5feb\u901f\u5165\u95e8"},{"location":"backup/oss/outline/#_2","text":"JindoShell CLI \u652f\u6301\u963f\u91cc\u4e91 OSS \u4f7f\u7528\u8bf4\u660e","title":"\u57fa\u672c\u529f\u80fd"},{"location":"backup/oss/outline/#_3","text":"","title":"\u5927\u6570\u636e\u751f\u6001"},{"location":"backup/oss/outline/#spark","text":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e","title":"Spark \u7ec4\u4ef6"},{"location":"backup/oss/outline/#hive","text":"Hive \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e","title":"Hive \u7ec4\u4ef6"},{"location":"backup/oss/outline/#presto","text":"Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS \u4e0a\u7684\u6570\u636e","title":"Presto \u7ec4\u4ef6"},{"location":"backup/oss/outline/#impala","text":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS \u4e0a\u7684\u6570\u636e","title":"Impala \u7ec4\u4ef6"},{"location":"backup/oss/outline/#hbase","text":"HBase \u4f7f\u7528 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8","title":"HBase \u7ec4\u4ef6"},{"location":"backup/oss/outline/#flink","text":"Flink \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e","title":"Flink \u7ec4\u4ef6"},{"location":"backup/oss/outline/#flume","text":"Flume \u4f7f\u7528 JindoSDK \u5199\u5165 OSS","title":"Flume \u7ec4\u4ef6"},{"location":"backup/oss/outline/#sqoop","text":"Sqoop \u4f7f\u7528 Kite SDK \u8bfb\u5199 OSS","title":"Sqoop \u7ec4\u4ef6"},{"location":"backup/oss/outline/#ai","text":"","title":"AI \u751f\u6001"},{"location":"backup/oss/outline/#jindofuse","text":"\u4f7f\u7528 JindoFuse \u8bbf\u95ee OSS","title":"JindoFuse \u4f7f\u7528"},{"location":"backup/oss/outline/#_4","text":"","title":"\u6700\u4f73\u5b9e\u8df5"},{"location":"backup/oss/outline/#jindodistcp","text":"JindoDistCp \u6570\u636e\u8fc1\u79fb","title":"JindoDistCp \u6570\u636e\u8fc1\u79fb"},{"location":"backup/oss/outline/#jindotable","text":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS","title":"JindoTable \u6570\u4ed3\u8fc1\u79fb"},{"location":"backup/oss/outline/#jindotable-oss","text":"JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a","title":"JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a"},{"location":"backup/oss/outline/#_5","text":"","title":"\u793e\u533a\u6700\u4f73\u5b9e\u8df5"},{"location":"backup/oss/outline/#flink-table-jindosdk-oss-hdfs","text":"Flink-table \u4f7f\u7528 JindoSDK + OSS \u4ee3\u66ff HDFS \u5b58\u50a8\u5b9e\u8df5","title":"Flink-table \u4f7f\u7528 JindoSDK + OSS \u4ee3\u66ff HDFS \u5b58\u50a8\u5b9e\u8df5\uff08\u4f5c\u8005\uff1a\u674e\u4f1f\uff09"},{"location":"backup/oss/outline/#_6","text":"\u4f7f\u7528 JindoSDK \u5927\u5e45\u63d0\u5347 OSS \u6587\u4ef6\u5404\u9879\u64cd\u4f5c\u6027\u80fd","title":"\u76f8\u5173\u6587\u7ae0"},{"location":"backup/oss/outline/#_7","text":"\u5e38\u89c1\u95ee\u9898","title":"\u5e38\u89c1\u95ee\u9898"},{"location":"backup/oss/outline/#_8","text":"","title":"\u5386\u53f2\u7248\u672c"},{"location":"backup/oss/outline/#jindosdk-460","text":"","title":"JindoSDK 4.6.0 \u7528\u6237\u6587\u6863"},{"location":"backup/oss/outline/#jindosdk-451","text":"","title":"JindoSDK 4.5.1 \u7528\u6237\u6587\u6863"},{"location":"backup/oss/outline/#jindosdk-450","text":"","title":"JindoSDK 4.5.0 \u7528\u6237\u6587\u6863"},{"location":"backup/oss/outline/#jindosdk-440","text":"","title":"JindoSDK 4.4.0 \u7528\u6237\u6587\u6863"},{"location":"backup/oss/outline/#jindosdk-430","text":"","title":"JindoSDK 4.3.0 \u7528\u6237\u6587\u6863"},{"location":"backup/oss/outline/#jindosdk-420","text":"","title":"JindoSDK 4.2.0 \u7528\u6237\u6587\u6863"},{"location":"backup/oss/outline/#jindosdk-410","text":"","title":"JindoSDK 4.1.0 \u7528\u6237\u6587\u6863"},{"location":"backup/oss/outline/#jindosdk-400","text":"","title":"JindoSDK 4.0.0 \u7528\u6237\u6587\u6863"},{"location":"backup/oss/outline/#jindosdk-3x-oss","text":"","title":"JindoSDK 3.x + OSS \u7528\u6237\u6587\u6863"},{"location":"backup/oss/configuration/jindosdk_configuration_list/","text":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e \u00a7 SDK\u914d\u7f6e\u9879\u5217\u8868 \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u5ba2\u6237\u7aef\u5199\u5165\u65f6\u7684\u4e34\u65f6\u6587\u4ef6\u76ee\u5f55\uff0c\u53ef\u914d\u7f6e\u591a\u4e2a\uff08\u9017\u53f7\u9694\u5f00\uff09\uff0c\u4f1a\u8f6e\u6d41\u5199\u5165\uff0c\u591a\u7528\u6237\u73af\u5883\u9700\u914d\u7f6e\u53ef\u8bfb\u5199\u6743\u9650 --> <name>fs.oss.tmp.data.dirs</name> <value>/tmp/</value> </property> <property> <!-- \u4e34\u65f6\u6587\u4ef6\u81ea\u6e05\u7406\u670d\u52a1 --> <name>fs.oss.tmp.data.cleaner.enable</name> <value>true</value> </property> <property> <!-- \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u6b21\u6570 --> <name>fs.oss.retry.count</name> <value>5</value> </property> <property> <!-- \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u95f4\u9694\uff08\u6beb\u79d2\uff09 --> <name>fs.oss.retry.interval.millisecond</name> <value>500</value> </property> <property> <!-- \u8bf7\u6c42 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 --> <name>fs.oss.timeout.millisecond</name> <value>30000</value> </property> <property> <!-- \u8fde\u63a5 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 --> <name>fs.oss.connection.timeout.millisecond</name> <value>3000</value> </property> <property> <!-- \u8fde\u63a5 oss \u7684\u8fde\u63a5\u6c60\u5bf9\u6bcf\u4e2ahost\u7684\u6700\u5927\u8fde\u63a5\u6570\uff08\u8d85\u8fc7\u9608\u503c\u5916\u7684\u8fde\u63a5\u4f1a\u4f7f\u7528\u77ed\u8fde\u63a5\uff09 --> <name>fs.oss.max.connections.per.host</name> <value>100</value> </property> <property> <!-- \u5355\u4e2a\u6587\u4ef6 oss \u5e76\u53d1\u4e0a\u4f20\u7ebf\u7a0b\u6570 --> <name>fs.oss.upload.thread.concurrency</name> <value>5</value> </property> <property> <!-- oss \u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u961f\u5217\u5927\u5c0f --> <name>fs.oss.upload.queue.size</name> <value>5</value> </property> <property> <!-- \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u6570 --> <name>fs.oss.upload.max.pending.tasks.per.stream</name> <value>16</value> </property> <property> <!-- \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u6570 --> <name>fs.oss.download.thread.concurrency</name> <value>16</value> </property> <property> <!-- \u6700\u5927\u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 --> <name>fs.oss.read.readahead.max.buffer.count</name> <value>48</value> </property> <property> <!-- oss \u8bfb\u53d6\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 --> <name>fs.oss.read.buffer.size</name> <value>10485764</value> </property> <property> <!-- oss \u5199\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 --> <name>fs.oss.write.buffer.size</name> <value>1048576</value> </property> <property> <!--oss \u5237\u65b0\u7f13\u51b2\u533a\u95f4\u9694\uff08\u6beb\u79d2\uff09\uff0c\u5c0f\u4e8e 0 \u65f6\u4e0d\u751f\u6548 --> <name>fs.oss.flush.interval.millisecond</name> <value>-1</value> </property> <property> <!-- \u6587\u4ef6\u7ea7\u522b crc64 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u6253\u5f00 --> <name>fs.oss.checksum.crc64.enable</name> <value>true</value> </property> <property> <!--\u8bf7\u6c42\u7ea7\u522b md5 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u5173\u95ed --> <name>fs.oss.checksum.md5.enable</name> <value>false</value> </property> <property> <!-- oss \u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u961f\u5217\u5927\u5c0f (\u5f03\u7528) --> <name>fs.oss.download.queue.size</name> <value>5</value> </property> <property> <!-- \u9884\u8bfb oss \u7684 buffer \u5927\u5c0f (\u5f03\u7528) --> <name>fs.oss.read.readahead.buffer.size</name> <value>1048576</value> </property> <property> <!-- \u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 (\u5f03\u7528) --> <name>fs.oss.read.readahead.buffer.count</name> <value>4</value> </property> </configuration> \u3010\u6ce8\uff1a4.5\u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u65b9\u5f0f\u3011","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/oss/configuration/jindosdk_configuration_list/#jindosdk","text":"","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/oss/configuration/jindosdk_configuration_list/#sdk","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u5ba2\u6237\u7aef\u5199\u5165\u65f6\u7684\u4e34\u65f6\u6587\u4ef6\u76ee\u5f55\uff0c\u53ef\u914d\u7f6e\u591a\u4e2a\uff08\u9017\u53f7\u9694\u5f00\uff09\uff0c\u4f1a\u8f6e\u6d41\u5199\u5165\uff0c\u591a\u7528\u6237\u73af\u5883\u9700\u914d\u7f6e\u53ef\u8bfb\u5199\u6743\u9650 --> <name>fs.oss.tmp.data.dirs</name> <value>/tmp/</value> </property> <property> <!-- \u4e34\u65f6\u6587\u4ef6\u81ea\u6e05\u7406\u670d\u52a1 --> <name>fs.oss.tmp.data.cleaner.enable</name> <value>true</value> </property> <property> <!-- \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u6b21\u6570 --> <name>fs.oss.retry.count</name> <value>5</value> </property> <property> <!-- \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u95f4\u9694\uff08\u6beb\u79d2\uff09 --> <name>fs.oss.retry.interval.millisecond</name> <value>500</value> </property> <property> <!-- \u8bf7\u6c42 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 --> <name>fs.oss.timeout.millisecond</name> <value>30000</value> </property> <property> <!-- \u8fde\u63a5 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 --> <name>fs.oss.connection.timeout.millisecond</name> <value>3000</value> </property> <property> <!-- \u8fde\u63a5 oss \u7684\u8fde\u63a5\u6c60\u5bf9\u6bcf\u4e2ahost\u7684\u6700\u5927\u8fde\u63a5\u6570\uff08\u8d85\u8fc7\u9608\u503c\u5916\u7684\u8fde\u63a5\u4f1a\u4f7f\u7528\u77ed\u8fde\u63a5\uff09 --> <name>fs.oss.max.connections.per.host</name> <value>100</value> </property> <property> <!-- \u5355\u4e2a\u6587\u4ef6 oss \u5e76\u53d1\u4e0a\u4f20\u7ebf\u7a0b\u6570 --> <name>fs.oss.upload.thread.concurrency</name> <value>5</value> </property> <property> <!-- oss \u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u961f\u5217\u5927\u5c0f --> <name>fs.oss.upload.queue.size</name> <value>5</value> </property> <property> <!-- \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u6570 --> <name>fs.oss.upload.max.pending.tasks.per.stream</name> <value>16</value> </property> <property> <!-- \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u6570 --> <name>fs.oss.download.thread.concurrency</name> <value>16</value> </property> <property> <!-- \u6700\u5927\u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 --> <name>fs.oss.read.readahead.max.buffer.count</name> <value>48</value> </property> <property> <!-- oss \u8bfb\u53d6\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 --> <name>fs.oss.read.buffer.size</name> <value>10485764</value> </property> <property> <!-- oss \u5199\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 --> <name>fs.oss.write.buffer.size</name> <value>1048576</value> </property> <property> <!--oss \u5237\u65b0\u7f13\u51b2\u533a\u95f4\u9694\uff08\u6beb\u79d2\uff09\uff0c\u5c0f\u4e8e 0 \u65f6\u4e0d\u751f\u6548 --> <name>fs.oss.flush.interval.millisecond</name> <value>-1</value> </property> <property> <!-- \u6587\u4ef6\u7ea7\u522b crc64 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u6253\u5f00 --> <name>fs.oss.checksum.crc64.enable</name> <value>true</value> </property> <property> <!--\u8bf7\u6c42\u7ea7\u522b md5 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u5173\u95ed --> <name>fs.oss.checksum.md5.enable</name> <value>false</value> </property> <property> <!-- oss \u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u961f\u5217\u5927\u5c0f (\u5f03\u7528) --> <name>fs.oss.download.queue.size</name> <value>5</value> </property> <property> <!-- \u9884\u8bfb oss \u7684 buffer \u5927\u5c0f (\u5f03\u7528) --> <name>fs.oss.read.readahead.buffer.size</name> <value>1048576</value> </property> <property> <!-- \u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 (\u5f03\u7528) --> <name>fs.oss.read.readahead.buffer.count</name> <value>4</value> </property> </configuration> \u3010\u6ce8\uff1a4.5\u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u65b9\u5f0f\u3011","title":"SDK\u914d\u7f6e\u9879\u5217\u8868"},{"location":"backup/oss/configuration/jindosdk_configuration_list_ini/","text":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e \u00a7 SDK\u914d\u7f6e\u9879\u5217\u8868 \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 jindosdk.cfg \u4e2d\u3002 [jindosdk] # \u5ba2\u6237\u7aef\u5199\u5165\u65f6\u7684\u4e34\u65f6\u6587\u4ef6\u76ee\u5f55\uff0c\u53ef\u914d\u7f6e\u591a\u4e2a\uff08\u9017\u53f7\u9694\u5f00\uff09\uff0c\u4f1a\u8f6e\u6d41\u5199\u5165\uff0c\u591a\u7528\u6237\u73af\u5883\u9700\u914d\u7f6e\u53ef\u8bfb\u5199\u6743\u9650 fs.oss.tmp.data.dirs=/tmp/ # \u4e34\u65f6\u6587\u4ef6\u81ea\u6e05\u7406\u670d\u52a1 fs.oss.tmp.data.cleaner.enable=true # \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u6b21\u6570 fs.oss.retry.count=5 # \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u95f4\u9694\uff08\u6beb\u79d2\uff09 fs.oss.retry.interval.millisecond=500 # \u8bf7\u6c42 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 fs.oss.timeout.millisecond=30000 # \u8fde\u63a5 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 fs.oss.connection.timeout.millisecond=3000 # \u8fde\u63a5 oss \u7684\u8fde\u63a5\u6c60\u5bf9\u6bcf\u4e2ahost\u7684\u6700\u5927\u8fde\u63a5\u6570\uff08\u8d85\u8fc7\u9608\u503c\u5916\u7684\u8fde\u63a5\u4f1a\u4f7f\u7528\u77ed\u8fde\u63a5\uff09 fs.oss.max.connections.per.host=100 # \u5355\u4e2a\u6587\u4ef6 oss \u5e76\u53d1\u4e0a\u4f20\u7ebf\u7a0b\u6570 fs.oss.upload.thread.concurrency=5 # oss \u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u961f\u5217\u5927\u5c0f fs.oss.upload.queue.size=5 # \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u6570 fs.oss.upload.max.pending.tasks.per.stream=16 # \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u6570 fs.oss.download.thread.concurrency=16 # \u6700\u5927\u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 fs.oss.read.readahead.max.buffer.count=48 # \u4f7f\u7528ECS\u514d\u5bc6\u670d\u52a1\uff0c\u907f\u514d\u914d\u7f6eAK\uff08\u4e0d\u63a8\u8350\uff0c\u5efa\u8bae\u4f7f\u7528\u56fa\u5b9aAK\u65b9\u5f0f\u8bbf\u95ee\uff09 fs.oss.provider.endpoint=ECS_ROLE # oss \u8bfb\u53d6\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 fs.oss.read.buffer.size=1048576 # oss \u5199\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 fs.oss.write.buffer.size=1048576 # oss \u5237\u65b0\u7f13\u51b2\u533a\u95f4\u9694\uff08\u6beb\u79d2\uff09\uff0c\u5c0f\u4e8e 0 \u65f6\u4e0d\u751f\u6548 fs.oss.flush.interval.millisecond=-1 # \u6587\u4ef6\u7ea7\u522b crc64 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u6253\u5f00 fs.oss.checksum.crc64.enable=true # \u8bf7\u6c42\u7ea7\u522b md5 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u5173\u95ed fs.oss.checksum.md5.enable=false # oss \u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u961f\u5217\u5927\u5c0f (\u5f03\u7528) fs.oss.download.queue.size=5 # \u9884\u8bfb oss \u7684 buffer \u5927\u5c0f (\u5f03\u7528) fs.oss.read.readahead.buffer.size=1048576 # \u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 (\u5f03\u7528) fs.oss.read.readahead.buffer.count=4 \u3010\u6ce8\uff1a4.5\u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u65b9\u5f0f\u3011","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/oss/configuration/jindosdk_configuration_list_ini/#jindosdk","text":"","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/oss/configuration/jindosdk_configuration_list_ini/#sdk","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 jindosdk.cfg \u4e2d\u3002 [jindosdk] # \u5ba2\u6237\u7aef\u5199\u5165\u65f6\u7684\u4e34\u65f6\u6587\u4ef6\u76ee\u5f55\uff0c\u53ef\u914d\u7f6e\u591a\u4e2a\uff08\u9017\u53f7\u9694\u5f00\uff09\uff0c\u4f1a\u8f6e\u6d41\u5199\u5165\uff0c\u591a\u7528\u6237\u73af\u5883\u9700\u914d\u7f6e\u53ef\u8bfb\u5199\u6743\u9650 fs.oss.tmp.data.dirs=/tmp/ # \u4e34\u65f6\u6587\u4ef6\u81ea\u6e05\u7406\u670d\u52a1 fs.oss.tmp.data.cleaner.enable=true # \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u6b21\u6570 fs.oss.retry.count=5 # \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u95f4\u9694\uff08\u6beb\u79d2\uff09 fs.oss.retry.interval.millisecond=500 # \u8bf7\u6c42 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 fs.oss.timeout.millisecond=30000 # \u8fde\u63a5 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 fs.oss.connection.timeout.millisecond=3000 # \u8fde\u63a5 oss \u7684\u8fde\u63a5\u6c60\u5bf9\u6bcf\u4e2ahost\u7684\u6700\u5927\u8fde\u63a5\u6570\uff08\u8d85\u8fc7\u9608\u503c\u5916\u7684\u8fde\u63a5\u4f1a\u4f7f\u7528\u77ed\u8fde\u63a5\uff09 fs.oss.max.connections.per.host=100 # \u5355\u4e2a\u6587\u4ef6 oss \u5e76\u53d1\u4e0a\u4f20\u7ebf\u7a0b\u6570 fs.oss.upload.thread.concurrency=5 # oss \u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u961f\u5217\u5927\u5c0f fs.oss.upload.queue.size=5 # \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u6570 fs.oss.upload.max.pending.tasks.per.stream=16 # \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u6570 fs.oss.download.thread.concurrency=16 # \u6700\u5927\u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 fs.oss.read.readahead.max.buffer.count=48 # \u4f7f\u7528ECS\u514d\u5bc6\u670d\u52a1\uff0c\u907f\u514d\u914d\u7f6eAK\uff08\u4e0d\u63a8\u8350\uff0c\u5efa\u8bae\u4f7f\u7528\u56fa\u5b9aAK\u65b9\u5f0f\u8bbf\u95ee\uff09 fs.oss.provider.endpoint=ECS_ROLE # oss \u8bfb\u53d6\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 fs.oss.read.buffer.size=1048576 # oss \u5199\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 fs.oss.write.buffer.size=1048576 # oss \u5237\u65b0\u7f13\u51b2\u533a\u95f4\u9694\uff08\u6beb\u79d2\uff09\uff0c\u5c0f\u4e8e 0 \u65f6\u4e0d\u751f\u6548 fs.oss.flush.interval.millisecond=-1 # \u6587\u4ef6\u7ea7\u522b crc64 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u6253\u5f00 fs.oss.checksum.crc64.enable=true # \u8bf7\u6c42\u7ea7\u522b md5 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u5173\u95ed fs.oss.checksum.md5.enable=false # oss \u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u961f\u5217\u5927\u5c0f (\u5f03\u7528) fs.oss.download.queue.size=5 # \u9884\u8bfb oss \u7684 buffer \u5927\u5c0f (\u5f03\u7528) fs.oss.read.readahead.buffer.size=1048576 # \u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 (\u5f03\u7528) fs.oss.read.readahead.buffer.count=4 \u3010\u6ce8\uff1a4.5\u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u65b9\u5f0f\u3011","title":"SDK\u914d\u7f6e\u9879\u5217\u8868"},{"location":"backup/oss/flink/jindosdk_on_flink/","text":"Flink \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 \u5728\u96c6\u7fa4\u4e0a\u6709\u5f00\u6e90\u7248\u672c Flink \u8f6f\u4ef6\uff0c\u7248\u672c\u4e0d\u4f4e\u4e8e 1.10.1\u3002 \u6ce8\uff1aFlink 1.16.0 \u53ca\u66f4\u9ad8\u7248\u672c\u7684\u53ef\u7528\u6027\u5c1a\u672a\u5f97\u5230\u9a8c\u8bc1\u3002 \u4e3a\u4ec0\u4e48 Flink \u9700\u8981\u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u00a7 \u82e5\u9700\u8981 Flink \u4fdd\u6301 EXACTLY_ONCE \u8bed\u4e49\u6d41\u5f0f\u5199\u5165 Aliyun OSS\uff0c\u5219\u9700\u8981\u4f7f\u7528 JindoSDK\u3002 Apache Flink \u662f\u4e00\u79cd\u5f53\u524d\u4e1a\u754c\u6d41\u884c\u7684\u5f00\u6e90\u5927\u6570\u636e\u6d41\u5f0f\u8ba1\u7b97\u5f15\u64ce\uff0c\u652f\u6301 \u201c\u4e25\u683c\u4e00\u6b21\u201d\uff08EXACTLY_ONCE\uff09\u8bed\u4e49\u5199\u5165\u90e8\u5206\u5b58\u50a8\u4ecb\u8d28\u3002Aliyun Object Storage Service (Aliyun OSS) \u5219\u662f\u963f\u91cc\u4e91\u65d7\u4e0b\u7684\u6838\u5fc3\u4ea7\u54c1\u4e4b\u4e00\uff0c\u63d0\u4f9b\u4e91\u539f\u751f\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1\uff0c\u51ed\u501f\u4f18\u5f02\u7684\u6027\u80fd\u548c\u5b89\u5168\u6027\u5df2\u670d\u52a1\u4e8e\u6d77\u91cf\u7528\u6237\u3002 \u76ee\u524d\uff0c\u5f00\u6e90\u7248\u672c Flink \u5bf9\u6d41\u5f0f\u5199\u5165 Aliyun OSS \u5c1a\u4e0d\u80fd\u652f\u6301 EXACTLY_ONCE \u8bed\u4e49\uff0c\u5982\u6709\u8be5\u9700\u6c42\u5219\u9700\u8981\u4f7f\u7528 JindoSDK\u3002 \u6ce8\uff1aApache Flink \u81ea 1.16 \u7248\u672c\u8d77\u539f\u751f\u63d0\u4f9b\u4e86\u6d41\u5f0f\u5199\u5165 Aliyun OSS \u5bf9\u8c61\u5b58\u50a8\u7684\u652f\u6301\u3002JindoSDK \u63d0\u4f9b\u7684\u5b9e\u73b0\u4e0e Apache Flink \u7684\u81ea\u6709\u5b9e\u73b0\u6709\u4e24\u4e2a\u4e3b\u8981\u533a\u522b\uff1a - JindoSDK \u7684\u5b9e\u73b0\u80fd\u591f\u540c\u65f6\u652f\u6301 Aliyun OSS \u5bf9\u8c61\u5b58\u50a8\u548c Aliyun OSS-HDFS \u6587\u4ef6\u5b58\u50a8\u3002\u5bf9 OSS-HDFS \u7684\u4ecb\u7ecd\u53c2\u8003 OSS-HDFS\u670d\u52a1\u6982\u8ff0 - JindoSDK \u7684\u5b9e\u73b0\u5e95\u5c42\u91c7\u7528\u4e86 Native \u65b9\u6848\u3002 JindoSDK \u914d\u7f6e \u00a7 \u9700\u8981\u5728\u6240\u6709 Flink \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flink \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e .jar \u6587\u4ef6\uff1a * jindo-flink-${version}-full.jar \u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 plugins/flink/ \u76ee\u5f55\u4e0b\u627e\u5230\u3002 \u53e6\u5916\uff0c\u5982\u679c\u96c6\u7fa4\u5185\u5b58\u5728 Apache Flink \u63d0\u4f9b\u7684\u81ea\u5e26 Flink OSS Connector\uff0c\u9700\u79fb\u9664\u3002\u79fb\u9664\u65b9\u6cd5\u4e3a\uff0c\u4ece Flink \u96c6\u7fa4\u7684 lib \u76ee\u5f55\u6216 plugins/oss-fs-hadoop \u79fb\u8d70\u5982\u4e0b jar \u5305\uff1a * flink-oss-fs-hadoop-${flink-version}.jar \u6ce8\uff1aJindoSDK \u76ee\u524d\u63d0\u4f9b\u591a\u5e73\u53f0\u652f\u6301\uff0c\u5f53\u524d\u652f\u6301\u7684\u5e73\u53f0\u4ee5\u53ca\u9700\u8981\u989d\u5916\u653e\u7f6e\u7684\u8d44\u6e90\uff0c\u53c2\u8003 JindoData 4.6.4 \u7248\u672c\u8bf4\u660e \u5982\u4f55\u4f7f\u7528 \u00a7 \u914d\u7f6e\u597d SDK \u540e\uff0c\u65e0\u9700\u989d\u5916\u914d\u7f6e\uff0c\u4ee5\u5e38\u89c4 Flink \u6d41\u5f0f\u4f5c\u4e1a\u65b9\u6cd5\u4f7f\u7528\u5373\u53ef\uff0c\u6ce8\u610f\u4f7f\u7528\u6b63\u786e\u7684\u8def\u5f84\u3002\u5199\u5165 OSS (Aliyun Object Storage Service) \u987b\u4ee5 oss:// \u4e3a\u524d\u7f00\u3002 \u793a\u4f8b \u00a7 \u4e0b\u9762\u662f\u4e00\u4e2a\u7b80\u5355\u7684\u793a\u4f8b\u7a0b\u5e8f\uff0c\u5982\u679c\u5df2\u7ecf\u6309\u7167\u4e0a\u4e00\u8282\u914d\u7f6e\u597d SDK\uff0c\u5219\u4e0b\u5217\u7a0b\u5e8f\u5e94\u80fd\u6b63\u786e\u8fd0\u884c\uff1a (1) \u901a\u7528\u914d\u7f6e \u6253\u5f00 Flink \u7684 \u201c\u68c0\u67e5\u70b9\u201d\uff08Checkpoint\uff09\u3002\u4f8b\u5982\uff0cenv \u662f\u4e00\u4e2a\u5df2\u7ecf\u5efa\u7acb\u7684 StreamExecutionEnvironment\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u65b9\u5f0f\u5efa\u7acb\uff1a StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); \u5219\u53ef\u4ee5\u8fd9\u6837\u542f\u7528 checkpoint\uff1a env.enableCheckpointing(<userDefinedCheckpointInterval>, CheckpointingMode.EXACTLY_ONCE); \u4f7f\u7528\u53ef\u91cd\u53d1\u7684\u6570\u636e\u6e90\uff0c\u4f8b\u5982 Kafka (2) \u793a\u4f8b\u7a0b\u5e8f \u4e0b\u6587\u4e2d\uff0coutputStream \u662f\u4e00\u4e2a\u9884\u5148\u5f62\u6210\u7684 DataStream<String> \u5bf9\u8c61\uff0c\u5e76\u671f\u671b\u5c06\u5176\u5199\u5165 OSS\uff0c\u90a3\u4e48\u53ef\u4ee5\u8fd9\u6837\u6dfb\u52a0 sink\uff1a String outputPath = \"oss://<user-defined-bucket>/<user-defined-dir>\" StreamingFileSink<String> sink = StreamingFileSink.forRowFormat( new Path(outputPath), new SimpleStringEncoder<String>(\"UTF-8\") ).build(); outputStream.addSink(sink); \u6700\u540e\u7528 env.execute() \u6267\u884c Flink \u4f5c\u4e1a\u5373\u53ef\u3002 \u9ad8\u7ea7\u4f7f\u7528\uff1a\u81ea\u5b9a\u4e49\u53c2\u6570 \u00a7 \u7528\u6237\u5728\u63d0\u4ea4 Flink \u4f5c\u4e1a\u65f6\uff0c\u53ef\u4ee5\u81ea\u5b9a\u4e49\u914d\u7f6e\u4e00\u4e9b\u53c2\u6570\uff0c\u4ee5\u5f00\u542f\u6216\u63a7\u5236\u7279\u5b9a\u529f\u80fd\u3002\u4f8b\u5982\uff0c\u4ee5 yarn-cluster \u6a21\u5f0f\u63d0\u4ea4\u65f6\uff0c\u53ef\u4ee5\u901a\u8fc7 -yD \u8fdb\u884c\u914d\u7f6e\uff0c\u65b9\u5f0f\u7c7b\u4f3c\u4e8e\uff1a <flink_home>/bin/flink run -m yarn-cluster -yD key1=value1 -yD key2=value2 ... \u76ee\u524d\u652f\u6301\u901a\u8fc7\u914d\u7f6e\u5f00\u542f \u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\uff0c\u6216\u8005\u63a7\u5236 \u201c\u5206\u7247\u4e0a\u4f20\u201d \u7684\u5e76\u884c\u5ea6\u3002 \u71b5\u6ce8\u5165 \u00a7 \u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\u7684\u76ee\u7684\u662f\uff0c\u5c06\u5199\u5165\u8def\u5f84\u7684\u4e00\u6bb5\u7279\u5b9a\u5b57\u7b26\u4e32\u5339\u914d\u51fa\u6765\uff0c\u7528\u4e00\u6bb5\u968f\u673a\u7684\u5b57\u7b26\u4e32\u8fdb\u884c\u66ff\u6362\uff0c\u4ee5\u524a\u5f31\u6240\u8c13 \u201c\u7247\u533a\u201d (sharding) \u6548\u5e94\uff0c\u63d0\u9ad8\u5199\u5165\u6548\u7387\u3002\u914d\u7f6e\u53c2\u6570\uff1a oss.entropy.key=<user-defined-key> oss.entropy.length=<user-defined-length> \u5982\u6b64\uff0c\u6bcf\u6b21\u6253\u5f00\u5199\u4e00\u4e2a\u65b0\u6587\u4ef6\u65f6\uff0c\u8def\u5f84\u4e2d\u4e0e <user-defined-key> \u76f8\u540c\u7684\u5b57\u7b26\u4e32\u4f1a\u88ab\u66ff\u6362\u4e3a\u4e00\u4e2a\u968f\u673a\u5b57\u7b26\u4e32\uff0c\u968f\u673a\u4e32\u7684\u957f\u5ea6\u4e3a <user-defined-length> \u3002\u914d\u7f6e\u8be5\u53c2\u6570\u65f6\uff0c <user-defined-length> \u5fc5\u987b\u5927\u4e8e\u96f6\u3002 \u5206\u7247\u4e0a\u4f20\u5e76\u884c\u5ea6 \u00a7 \u5199\u5165 OSS \u4f1a\u81ea\u52a8\u8c03\u7528\u9ad8\u6548\u7684 \u201c\u5206\u7247\u4e0a\u4f20\u201d (Multipart Upload) \u673a\u5236\uff0c\u5c06\u5f85\u4e0a\u4f20\u7684\u6587\u4ef6\u5206\u4e3a\u591a\u4e2a\u6570\u636e\u5757 (part) \u5206\u522b\u4e0a\u4f20\uff0c\u6700\u540e\u7ec4\u5408\u3002\u76ee\u524d\u652f\u6301\u914d\u7f6e\u53c2\u6570 oss.upload.max.concurrent.uploads \uff0c\u7528\u6765\u63a7\u5236\u4e0a\u4f20\u6570\u636e\u5757 (part) \u7684\u5e76\u884c\u5ea6\uff0c\u5982\u679c\u8bbe\u7f6e\u8f83\u9ad8\u7684\u6570\u503c\u5219\u53ef\u80fd\u4f1a\u63d0\u9ad8\u5199\u5165\u6548\u7387\uff08\u4f46\u4e5f\u4f1a\u5360\u7528\u66f4\u591a\u8d44\u6e90\uff09\u3002\u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u8be5\u503c\u4e3a\u5f53\u524d\u53ef\u7528\u7684\u5904\u7406\u5668\u6570\u91cf\u3002","title":"Flink \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e"},{"location":"backup/oss/flink/jindosdk_on_flink/#flink-jindosdk-oss","text":"","title":"Flink \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e"},{"location":"backup/oss/flink/jindosdk_on_flink/#_1","text":"\u5728\u96c6\u7fa4\u4e0a\u6709\u5f00\u6e90\u7248\u672c Flink \u8f6f\u4ef6\uff0c\u7248\u672c\u4e0d\u4f4e\u4e8e 1.10.1\u3002 \u6ce8\uff1aFlink 1.16.0 \u53ca\u66f4\u9ad8\u7248\u672c\u7684\u53ef\u7528\u6027\u5c1a\u672a\u5f97\u5230\u9a8c\u8bc1\u3002","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/oss/flink/jindosdk_on_flink/#flink-jindosdk-oss_1","text":"\u82e5\u9700\u8981 Flink \u4fdd\u6301 EXACTLY_ONCE \u8bed\u4e49\u6d41\u5f0f\u5199\u5165 Aliyun OSS\uff0c\u5219\u9700\u8981\u4f7f\u7528 JindoSDK\u3002 Apache Flink \u662f\u4e00\u79cd\u5f53\u524d\u4e1a\u754c\u6d41\u884c\u7684\u5f00\u6e90\u5927\u6570\u636e\u6d41\u5f0f\u8ba1\u7b97\u5f15\u64ce\uff0c\u652f\u6301 \u201c\u4e25\u683c\u4e00\u6b21\u201d\uff08EXACTLY_ONCE\uff09\u8bed\u4e49\u5199\u5165\u90e8\u5206\u5b58\u50a8\u4ecb\u8d28\u3002Aliyun Object Storage Service (Aliyun OSS) \u5219\u662f\u963f\u91cc\u4e91\u65d7\u4e0b\u7684\u6838\u5fc3\u4ea7\u54c1\u4e4b\u4e00\uff0c\u63d0\u4f9b\u4e91\u539f\u751f\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1\uff0c\u51ed\u501f\u4f18\u5f02\u7684\u6027\u80fd\u548c\u5b89\u5168\u6027\u5df2\u670d\u52a1\u4e8e\u6d77\u91cf\u7528\u6237\u3002 \u76ee\u524d\uff0c\u5f00\u6e90\u7248\u672c Flink \u5bf9\u6d41\u5f0f\u5199\u5165 Aliyun OSS \u5c1a\u4e0d\u80fd\u652f\u6301 EXACTLY_ONCE \u8bed\u4e49\uff0c\u5982\u6709\u8be5\u9700\u6c42\u5219\u9700\u8981\u4f7f\u7528 JindoSDK\u3002 \u6ce8\uff1aApache Flink \u81ea 1.16 \u7248\u672c\u8d77\u539f\u751f\u63d0\u4f9b\u4e86\u6d41\u5f0f\u5199\u5165 Aliyun OSS \u5bf9\u8c61\u5b58\u50a8\u7684\u652f\u6301\u3002JindoSDK \u63d0\u4f9b\u7684\u5b9e\u73b0\u4e0e Apache Flink \u7684\u81ea\u6709\u5b9e\u73b0\u6709\u4e24\u4e2a\u4e3b\u8981\u533a\u522b\uff1a - JindoSDK \u7684\u5b9e\u73b0\u80fd\u591f\u540c\u65f6\u652f\u6301 Aliyun OSS \u5bf9\u8c61\u5b58\u50a8\u548c Aliyun OSS-HDFS \u6587\u4ef6\u5b58\u50a8\u3002\u5bf9 OSS-HDFS \u7684\u4ecb\u7ecd\u53c2\u8003 OSS-HDFS\u670d\u52a1\u6982\u8ff0 - JindoSDK \u7684\u5b9e\u73b0\u5e95\u5c42\u91c7\u7528\u4e86 Native \u65b9\u6848\u3002","title":"\u4e3a\u4ec0\u4e48 Flink \u9700\u8981\u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/oss/flink/jindosdk_on_flink/#jindosdk","text":"\u9700\u8981\u5728\u6240\u6709 Flink \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flink \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e .jar \u6587\u4ef6\uff1a * jindo-flink-${version}-full.jar \u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 plugins/flink/ \u76ee\u5f55\u4e0b\u627e\u5230\u3002 \u53e6\u5916\uff0c\u5982\u679c\u96c6\u7fa4\u5185\u5b58\u5728 Apache Flink \u63d0\u4f9b\u7684\u81ea\u5e26 Flink OSS Connector\uff0c\u9700\u79fb\u9664\u3002\u79fb\u9664\u65b9\u6cd5\u4e3a\uff0c\u4ece Flink \u96c6\u7fa4\u7684 lib \u76ee\u5f55\u6216 plugins/oss-fs-hadoop \u79fb\u8d70\u5982\u4e0b jar \u5305\uff1a * flink-oss-fs-hadoop-${flink-version}.jar \u6ce8\uff1aJindoSDK \u76ee\u524d\u63d0\u4f9b\u591a\u5e73\u53f0\u652f\u6301\uff0c\u5f53\u524d\u652f\u6301\u7684\u5e73\u53f0\u4ee5\u53ca\u9700\u8981\u989d\u5916\u653e\u7f6e\u7684\u8d44\u6e90\uff0c\u53c2\u8003 JindoData 4.6.4 \u7248\u672c\u8bf4\u660e","title":"JindoSDK \u914d\u7f6e"},{"location":"backup/oss/flink/jindosdk_on_flink/#_2","text":"\u914d\u7f6e\u597d SDK \u540e\uff0c\u65e0\u9700\u989d\u5916\u914d\u7f6e\uff0c\u4ee5\u5e38\u89c4 Flink \u6d41\u5f0f\u4f5c\u4e1a\u65b9\u6cd5\u4f7f\u7528\u5373\u53ef\uff0c\u6ce8\u610f\u4f7f\u7528\u6b63\u786e\u7684\u8def\u5f84\u3002\u5199\u5165 OSS (Aliyun Object Storage Service) \u987b\u4ee5 oss:// \u4e3a\u524d\u7f00\u3002","title":"\u5982\u4f55\u4f7f\u7528"},{"location":"backup/oss/flink/jindosdk_on_flink/#_3","text":"\u4e0b\u9762\u662f\u4e00\u4e2a\u7b80\u5355\u7684\u793a\u4f8b\u7a0b\u5e8f\uff0c\u5982\u679c\u5df2\u7ecf\u6309\u7167\u4e0a\u4e00\u8282\u914d\u7f6e\u597d SDK\uff0c\u5219\u4e0b\u5217\u7a0b\u5e8f\u5e94\u80fd\u6b63\u786e\u8fd0\u884c\uff1a (1) \u901a\u7528\u914d\u7f6e \u6253\u5f00 Flink \u7684 \u201c\u68c0\u67e5\u70b9\u201d\uff08Checkpoint\uff09\u3002\u4f8b\u5982\uff0cenv \u662f\u4e00\u4e2a\u5df2\u7ecf\u5efa\u7acb\u7684 StreamExecutionEnvironment\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u65b9\u5f0f\u5efa\u7acb\uff1a StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); \u5219\u53ef\u4ee5\u8fd9\u6837\u542f\u7528 checkpoint\uff1a env.enableCheckpointing(<userDefinedCheckpointInterval>, CheckpointingMode.EXACTLY_ONCE); \u4f7f\u7528\u53ef\u91cd\u53d1\u7684\u6570\u636e\u6e90\uff0c\u4f8b\u5982 Kafka (2) \u793a\u4f8b\u7a0b\u5e8f \u4e0b\u6587\u4e2d\uff0coutputStream \u662f\u4e00\u4e2a\u9884\u5148\u5f62\u6210\u7684 DataStream<String> \u5bf9\u8c61\uff0c\u5e76\u671f\u671b\u5c06\u5176\u5199\u5165 OSS\uff0c\u90a3\u4e48\u53ef\u4ee5\u8fd9\u6837\u6dfb\u52a0 sink\uff1a String outputPath = \"oss://<user-defined-bucket>/<user-defined-dir>\" StreamingFileSink<String> sink = StreamingFileSink.forRowFormat( new Path(outputPath), new SimpleStringEncoder<String>(\"UTF-8\") ).build(); outputStream.addSink(sink); \u6700\u540e\u7528 env.execute() \u6267\u884c Flink \u4f5c\u4e1a\u5373\u53ef\u3002","title":"\u793a\u4f8b"},{"location":"backup/oss/flink/jindosdk_on_flink/#_4","text":"\u7528\u6237\u5728\u63d0\u4ea4 Flink \u4f5c\u4e1a\u65f6\uff0c\u53ef\u4ee5\u81ea\u5b9a\u4e49\u914d\u7f6e\u4e00\u4e9b\u53c2\u6570\uff0c\u4ee5\u5f00\u542f\u6216\u63a7\u5236\u7279\u5b9a\u529f\u80fd\u3002\u4f8b\u5982\uff0c\u4ee5 yarn-cluster \u6a21\u5f0f\u63d0\u4ea4\u65f6\uff0c\u53ef\u4ee5\u901a\u8fc7 -yD \u8fdb\u884c\u914d\u7f6e\uff0c\u65b9\u5f0f\u7c7b\u4f3c\u4e8e\uff1a <flink_home>/bin/flink run -m yarn-cluster -yD key1=value1 -yD key2=value2 ... \u76ee\u524d\u652f\u6301\u901a\u8fc7\u914d\u7f6e\u5f00\u542f \u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\uff0c\u6216\u8005\u63a7\u5236 \u201c\u5206\u7247\u4e0a\u4f20\u201d \u7684\u5e76\u884c\u5ea6\u3002","title":"\u9ad8\u7ea7\u4f7f\u7528\uff1a\u81ea\u5b9a\u4e49\u53c2\u6570"},{"location":"backup/oss/flink/jindosdk_on_flink/#_5","text":"\u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\u7684\u76ee\u7684\u662f\uff0c\u5c06\u5199\u5165\u8def\u5f84\u7684\u4e00\u6bb5\u7279\u5b9a\u5b57\u7b26\u4e32\u5339\u914d\u51fa\u6765\uff0c\u7528\u4e00\u6bb5\u968f\u673a\u7684\u5b57\u7b26\u4e32\u8fdb\u884c\u66ff\u6362\uff0c\u4ee5\u524a\u5f31\u6240\u8c13 \u201c\u7247\u533a\u201d (sharding) \u6548\u5e94\uff0c\u63d0\u9ad8\u5199\u5165\u6548\u7387\u3002\u914d\u7f6e\u53c2\u6570\uff1a oss.entropy.key=<user-defined-key> oss.entropy.length=<user-defined-length> \u5982\u6b64\uff0c\u6bcf\u6b21\u6253\u5f00\u5199\u4e00\u4e2a\u65b0\u6587\u4ef6\u65f6\uff0c\u8def\u5f84\u4e2d\u4e0e <user-defined-key> \u76f8\u540c\u7684\u5b57\u7b26\u4e32\u4f1a\u88ab\u66ff\u6362\u4e3a\u4e00\u4e2a\u968f\u673a\u5b57\u7b26\u4e32\uff0c\u968f\u673a\u4e32\u7684\u957f\u5ea6\u4e3a <user-defined-length> \u3002\u914d\u7f6e\u8be5\u53c2\u6570\u65f6\uff0c <user-defined-length> \u5fc5\u987b\u5927\u4e8e\u96f6\u3002","title":"\u71b5\u6ce8\u5165"},{"location":"backup/oss/flink/jindosdk_on_flink/#_6","text":"\u5199\u5165 OSS \u4f1a\u81ea\u52a8\u8c03\u7528\u9ad8\u6548\u7684 \u201c\u5206\u7247\u4e0a\u4f20\u201d (Multipart Upload) \u673a\u5236\uff0c\u5c06\u5f85\u4e0a\u4f20\u7684\u6587\u4ef6\u5206\u4e3a\u591a\u4e2a\u6570\u636e\u5757 (part) \u5206\u522b\u4e0a\u4f20\uff0c\u6700\u540e\u7ec4\u5408\u3002\u76ee\u524d\u652f\u6301\u914d\u7f6e\u53c2\u6570 oss.upload.max.concurrent.uploads \uff0c\u7528\u6765\u63a7\u5236\u4e0a\u4f20\u6570\u636e\u5757 (part) \u7684\u5e76\u884c\u5ea6\uff0c\u5982\u679c\u8bbe\u7f6e\u8f83\u9ad8\u7684\u6570\u503c\u5219\u53ef\u80fd\u4f1a\u63d0\u9ad8\u5199\u5165\u6548\u7387\uff08\u4f46\u4e5f\u4f1a\u5360\u7528\u66f4\u591a\u8d44\u6e90\uff09\u3002\u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u8be5\u503c\u4e3a\u5f53\u524d\u53ef\u7528\u7684\u5904\u7406\u5668\u6570\u91cf\u3002","title":"\u5206\u7247\u4e0a\u4f20\u5e76\u884c\u5ea6"},{"location":"backup/oss/flume/jindosdk_on_flume/","text":"Flume \u4f7f\u7528 JindoSDK \u5199\u5165 OSS \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 \u5728\u96c6\u7fa4\u4e0a\u5df2\u7ecf\u90e8\u7f72 Flume\uff0c\u5df2\u90e8\u7f72 JindoSDK 4.0 \u4ee5\u4e0a\u7248\u672c\u3002 SDK \u914d\u7f6e \u00a7 \u9700\u8981\u5728 Flume \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flume \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e JindoSDK\u3002 \u4e0b\u8f7d\u5e76\u89e3\u538b\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 ) cp jindosdk-x.x.x/lib/*.jar $FLUME_HOME/jars/ Sink \u914d\u7f6e\u793a\u4f8b \u00a7 # \u914d\u7f6e OSS Sink xxx.sinks.oss_sink.hdfs.path = oss://${your_bucket}/flume_dir/%Y-%m-%d/%H # Sink\u53c2\u6570\uff0cbatchSize \u9700\u8981\u8bbe\u7f6e\u5927\u4e00\u4e9b\uff0c\u63a8\u8350\u6bcf\u6b21 Flush \u7684\u91cf\u5728 32MB\u4ee5\u4e0a\uff0c\u5426\u5219\u4f1a\u5f71\u54cd\u6027\u80fd xxx.sinks.oss_sink.hdfs.batchSize = 100000 ... xxx.sinks.oss_sink.hdfs.round = true xxx.sinks.oss_sink.hdfs.roundValue = 15 xxx.sinks.oss_sink.hdfs.Unit = minute xxx.sinks.oss_sink.hdfs.filePrefix = your_topic xxx.sinks.oss_sink.rollSize = 3600 xxx.sinks.oss_sink.threadsPoolSize = 30 ...","title":"Flume \u4f7f\u7528 JindoSDK \u5199\u5165 OSS"},{"location":"backup/oss/flume/jindosdk_on_flume/#flume-jindosdk-oss","text":"","title":"Flume \u4f7f\u7528 JindoSDK \u5199\u5165 OSS"},{"location":"backup/oss/flume/jindosdk_on_flume/#_1","text":"\u5728\u96c6\u7fa4\u4e0a\u5df2\u7ecf\u90e8\u7f72 Flume\uff0c\u5df2\u90e8\u7f72 JindoSDK 4.0 \u4ee5\u4e0a\u7248\u672c\u3002","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/oss/flume/jindosdk_on_flume/#sdk","text":"\u9700\u8981\u5728 Flume \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flume \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e JindoSDK\u3002 \u4e0b\u8f7d\u5e76\u89e3\u538b\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 ) cp jindosdk-x.x.x/lib/*.jar $FLUME_HOME/jars/","title":"SDK \u914d\u7f6e"},{"location":"backup/oss/flume/jindosdk_on_flume/#sink","text":"# \u914d\u7f6e OSS Sink xxx.sinks.oss_sink.hdfs.path = oss://${your_bucket}/flume_dir/%Y-%m-%d/%H # Sink\u53c2\u6570\uff0cbatchSize \u9700\u8981\u8bbe\u7f6e\u5927\u4e00\u4e9b\uff0c\u63a8\u8350\u6bcf\u6b21 Flush \u7684\u91cf\u5728 32MB\u4ee5\u4e0a\uff0c\u5426\u5219\u4f1a\u5f71\u54cd\u6027\u80fd xxx.sinks.oss_sink.hdfs.batchSize = 100000 ... xxx.sinks.oss_sink.hdfs.round = true xxx.sinks.oss_sink.hdfs.roundValue = 15 xxx.sinks.oss_sink.hdfs.Unit = minute xxx.sinks.oss_sink.hdfs.filePrefix = your_topic xxx.sinks.oss_sink.rollSize = 3600 xxx.sinks.oss_sink.threadsPoolSize = 30 ...","title":"Sink \u914d\u7f6e\u793a\u4f8b"},{"location":"backup/oss/hadoop/jindosdk_ide_hadoop/","text":"Hadoop \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 \u00a7 \u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindosdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.net.URI; public class TestJindoSDK { public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. FileSystem fs = FileSystem.get(URI.create(\"oss://<bucket>/\"), conf); FSDataInputStream in = fs.open(new Path(\"/uttest/file1\")); in.read(); in.close(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency>","title":"Hadoop \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5"},{"location":"backup/oss/hadoop/jindosdk_ide_hadoop/#hadoop-jindosdk-ide","text":"\u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindosdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.net.URI; public class TestJindoSDK { public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. FileSystem fs = FileSystem.get(URI.create(\"oss://<bucket>/\"), conf); FSDataInputStream in = fs.open(new Path(\"/uttest/file1\")); in.read(); in.close(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency>","title":"Hadoop \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5"},{"location":"backup/oss/hadoop/jindosdk_on_hadoop/","text":"Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u00a7 JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0\u3002 \u5373\u4f7f\u60a8\u4f7f\u7528 JindoSDK \u4ec5\u4ec5\u4f5c\u4e3a OSS \u5ba2\u6237\u7aef\uff0c\u76f8\u5bf9\u4e8e Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\u5b9e\u73b0\uff0c\u60a8\u8fd8\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u76ee\u524d\u652f\u6301\u5e02\u9762\u4e0a\u5927\u90e8\u5206 Hadoop \u7248\u672c\uff0c\u5728 Hadoop 2.3 \u53ca\u4ee5\u4e0a\u7684\u7248\u672c\u4e0a\u9a8c\u8bc1\u901a\u8fc7\uff082.3 \u4ee5\u524d\u7248\u672c\u6682\u672a\u6d4b\u8bd5\uff0c\u5982\u6709\u95ee\u9898\u8bf7 \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988\uff09\u3002 \u5173\u4e8e JindoSDK \u548c Hadoop \u793e\u533a OSS connector \u7684\u6027\u80fd\u5bf9\u6bd4\uff0c\u8bf7\u53c2\u8003\u6587\u6863 JindoSDK\u548cHadoop-OSS-SDK\u6027\u80fd\u5bf9\u6bd4\u6d4b\u8bd5 \u3002 \u6b65\u9aa4 \u00a7 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06 OSS bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 4. \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u00a7 \u7528Hadoop Shell\u8bbf\u95eeOSS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls oss://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<bucket>/<path> 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 6. \u6743\u9650\u7ba1\u7406 \u00a7 \u81ea\u5efa\u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u963f\u91cc\u4e91 E-MapReduce \u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848","title":"Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/oss/hadoop/jindosdk_on_hadoop/#hadoop-jindosdk-oss","text":"JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0\u3002 \u5373\u4f7f\u60a8\u4f7f\u7528 JindoSDK \u4ec5\u4ec5\u4f5c\u4e3a OSS \u5ba2\u6237\u7aef\uff0c\u76f8\u5bf9\u4e8e Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\u5b9e\u73b0\uff0c\u60a8\u8fd8\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u76ee\u524d\u652f\u6301\u5e02\u9762\u4e0a\u5927\u90e8\u5206 Hadoop \u7248\u672c\uff0c\u5728 Hadoop 2.3 \u53ca\u4ee5\u4e0a\u7684\u7248\u672c\u4e0a\u9a8c\u8bc1\u901a\u8fc7\uff082.3 \u4ee5\u524d\u7248\u672c\u6682\u672a\u6d4b\u8bd5\uff0c\u5982\u6709\u95ee\u9898\u8bf7 \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988\uff09\u3002 \u5173\u4e8e JindoSDK \u548c Hadoop \u793e\u533a OSS connector \u7684\u6027\u80fd\u5bf9\u6bd4\uff0c\u8bf7\u53c2\u8003\u6587\u6863 JindoSDK\u548cHadoop-OSS-SDK\u6027\u80fd\u5bf9\u6bd4\u6d4b\u8bd5 \u3002","title":"Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/oss/hadoop/jindosdk_on_hadoop/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/oss/hadoop/jindosdk_on_hadoop/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/oss/hadoop/jindosdk_on_hadoop/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/oss/hadoop/jindosdk_on_hadoop/#3-oss-access-key","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06 OSS bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002","title":"3. \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"backup/oss/hadoop/jindosdk_on_hadoop/#4-jindosdk-oss","text":"\u7528Hadoop Shell\u8bbf\u95eeOSS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls oss://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<bucket>/<path>","title":"4. \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/oss/hadoop/jindosdk_on_hadoop/#5","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/oss/hadoop/jindosdk_on_hadoop/#6","text":"\u81ea\u5efa\u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u963f\u91cc\u4e91 E-MapReduce \u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848","title":"6. \u6743\u9650\u7ba1\u7406"},{"location":"backup/oss/hbase/jindosdk_on_hbase/","text":"HBase \u4f7f\u7528 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 \u00a7 HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\uff0c\u4f46\u9700\u901a\u8fc7\u672c\u5730 HDFS \u96c6\u7fa4\u5b58\u50a8 WAL \u6587\u4ef6\u3002 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06 OSS bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84 \u00a7 \u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84\uff0c\u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u7684\u503c\u4e3a OSS \u5730\u5740\uff0c\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84\u548c WAL \u6587\u4ef6\u7684\u5b58\u50a8\u8def\u5f84\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 OSS\u3002\u53c2\u6570\u503c\u4e3a oss://bucket/hbase-root-dir","title":"HBase \u4f7f\u7528 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8"},{"location":"backup/oss/hbase/jindosdk_on_hbase/#hbase-oss","text":"HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\uff0c\u4f46\u9700\u901a\u8fc7\u672c\u5730 HDFS \u96c6\u7fa4\u5b58\u50a8 WAL \u6587\u4ef6\u3002","title":"HBase \u4f7f\u7528 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8"},{"location":"backup/oss/hbase/jindosdk_on_hbase/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/oss/hbase/jindosdk_on_hbase/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/oss/hbase/jindosdk_on_hbase/#3-oss-access-key","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06 OSS bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002","title":"3. \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"backup/oss/hbase/jindosdk_on_hbase/#hbase","text":"\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84\uff0c\u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u7684\u503c\u4e3a OSS \u5730\u5740\uff0c\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84\u548c WAL \u6587\u4ef6\u7684\u5b58\u50a8\u8def\u5f84\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 OSS\u3002\u53c2\u6570\u503c\u4e3a oss://bucket/hbase-root-dir","title":"\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84"},{"location":"backup/oss/hive/jindosdk_on_hive/","text":"Hive \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e \u00a7 Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS \u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002 \u5b89\u88c5\u914d\u7f6e \u00a7 1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/ 2. \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> 3. \u914d\u7f6e OSS Access Key \u00a7 \u5c06 OSS \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u6700\u540e\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 4. Hive on MR \u00a7 \u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002 5. Hive on Tez \u00a7 \u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002 6. Hive on Spark \u00a7 \u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002 OSS \u7528\u4e8e\u8868\u7684\u5b58\u50a8 \u00a7 \u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS \u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS \u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'oss://bucketname/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'oss://bucketname/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS \u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>oss://bucketname/path/to/warehouse</value> </property> </configuration> \u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a \u00a7 ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'oss://bucketname/path/to/us/part210301cn'; \u6e05\u7406\u56de\u6536\u7ad9 \u00a7 \u5f53\u4f7f\u7528 Hive \u5220\u9664 OSS \u7684\u8868\u65f6\uff0c\u82e5\u5f00\u542f\u4e86\u56de\u6536\u7ad9\u529f\u80fd\uff0c\u4f1a\u5728 OSS \u7684 bucket \u4e0a\u5b58\u50a8\u88ab\u5220\u9664\u7684\u8868\u7684\u6570\u636e\uff0c\u56de\u6536\u7ad9\u7684\u76ee\u5f55\u4e3a oss://buckect-name/user/<username>/.Trash \u3002\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e OSS \u6240\u4f7f\u7528\u7684\u56de\u6536\u7ad9\u76ee\u5f55\u7684\u751f\u547d\u5468\u671f\u6765\u6e05\u7406\u56de\u6536\u7ad9\u3002 \u5173\u4e8e\u5982\u4f55\u8bbe\u7f6e OSS \u751f\u547d\u5468\u671f\uff0c\u8bf7\u53c2\u8003 \u8bbe\u7f6e\u751f\u547d\u5468\u671f\u89c4\u5219 \u82e5\u8981\u8df3\u8fc7\u56de\u6536\u7ad9\uff0c\u76f4\u63a5\u5220\u9664\u8868\uff0c\u5219\u53ef\u4ee5\u6307\u5b9a PURGE \u53c2\u6570\uff08\u614e\u7528\uff09\u3002 DROP TABLE [IF EXISTS] table_name [PURGE] \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Hive \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e"},{"location":"backup/oss/hive/jindosdk_on_hive/#hive-jindosdk-oss","text":"Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS \u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002","title":"Hive \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e"},{"location":"backup/oss/hive/jindosdk_on_hive/#_1","text":"","title":"\u5b89\u88c5\u914d\u7f6e"},{"location":"backup/oss/hive/jindosdk_on_hive/#1-hive-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/","title":"1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002"},{"location":"backup/oss/hive/jindosdk_on_hive/#2-jindosdk-oss","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration>","title":"2. \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b"},{"location":"backup/oss/hive/jindosdk_on_hive/#3-oss-access-key","text":"\u5c06 OSS \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u6700\u540e\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002","title":"3. \u914d\u7f6e OSS Access Key"},{"location":"backup/oss/hive/jindosdk_on_hive/#4-hive-on-mr","text":"\u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002","title":"4. Hive on MR"},{"location":"backup/oss/hive/jindosdk_on_hive/#5-hive-on-tez","text":"\u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002","title":"5. Hive on Tez"},{"location":"backup/oss/hive/jindosdk_on_hive/#6-hive-on-spark","text":"\u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002","title":"6. Hive on Spark"},{"location":"backup/oss/hive/jindosdk_on_hive/#oss","text":"\u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS \u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS \u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'oss://bucketname/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'oss://bucketname/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS \u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>oss://bucketname/path/to/warehouse</value> </property> </configuration>","title":"OSS \u7528\u4e8e\u8868\u7684\u5b58\u50a8"},{"location":"backup/oss/hive/jindosdk_on_hive/#oss_1","text":"ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'oss://bucketname/path/to/us/part210301cn';","title":"\u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a"},{"location":"backup/oss/hive/jindosdk_on_hive/#_2","text":"\u5f53\u4f7f\u7528 Hive \u5220\u9664 OSS \u7684\u8868\u65f6\uff0c\u82e5\u5f00\u542f\u4e86\u56de\u6536\u7ad9\u529f\u80fd\uff0c\u4f1a\u5728 OSS \u7684 bucket \u4e0a\u5b58\u50a8\u88ab\u5220\u9664\u7684\u8868\u7684\u6570\u636e\uff0c\u56de\u6536\u7ad9\u7684\u76ee\u5f55\u4e3a oss://buckect-name/user/<username>/.Trash \u3002\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e OSS \u6240\u4f7f\u7528\u7684\u56de\u6536\u7ad9\u76ee\u5f55\u7684\u751f\u547d\u5468\u671f\u6765\u6e05\u7406\u56de\u6536\u7ad9\u3002 \u5173\u4e8e\u5982\u4f55\u8bbe\u7f6e OSS \u751f\u547d\u5468\u671f\uff0c\u8bf7\u53c2\u8003 \u8bbe\u7f6e\u751f\u547d\u5468\u671f\u89c4\u5219 \u82e5\u8981\u8df3\u8fc7\u56de\u6536\u7ad9\uff0c\u76f4\u63a5\u5220\u9664\u8868\uff0c\u5219\u53ef\u4ee5\u6307\u5b9a PURGE \u53c2\u6570\uff08\u614e\u7528\uff09\u3002 DROP TABLE [IF EXISTS] table_name [PURGE]","title":"\u6e05\u7406\u56de\u6536\u7ad9"},{"location":"backup/oss/hive/jindosdk_on_hive/#_3","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/oss/impala/jindosdk_on_impala/","text":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS \u4e0a\u7684\u6570\u636e \u00a7 JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Impala \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Impala \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fddImpala\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728\u6240\u6709 Impala \u8282\u70b9\u5b89\u88c5 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Impala \u7684 classpath \u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $IMPALA_HOME/lib/ 3. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e IMPALA \u4f7f\u7528\u7684 core-site.xml \u914d\u7f6e JindoSDK \u8bbf\u95ee OSS \u00a7 \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 4. \u4f7f\u7528 Impala \u8bbf\u95ee OSS\u3002 \u00a7 \u4e0b\u8f7d\u6d4b\u8bd5\u6570\u636e\u96c6 \u94fe\u63a5 \u4e0a\u4f20\u6570\u636e bash hadoop fs -put test_data oss://bucket/dir \u521b\u5efa\u8868 sql CREATE EXTERNAL TABLE customer_demographics ( `cd_demo_sk` INT, `cd_gender` STRING, `cd_marital_status` STRING, `cd_education_status` STRING, `cd_purchase_estimate` INT, `cd_credit_rating` STRING, `cd_dep_count` INT, `cd_dep_employed_count` INT, `cd_dep_college_count` INT) STORED AS PARQUET LOCATION 'oss://bucket/dir'; \u67e5\u8be2 OSS \u8868 sql select * from customer_demographics; 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS \u4e0a\u7684\u6570\u636e"},{"location":"backup/oss/impala/jindosdk_on_impala/#impala-jindosdk-oss","text":"JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Impala \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS \u4e0a\u7684\u6570\u636e"},{"location":"backup/oss/impala/jindosdk_on_impala/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/oss/impala/jindosdk_on_impala/#1-impala-hadoop","text":"\u786e\u4fddImpala\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Impala \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"backup/oss/impala/jindosdk_on_impala/#2-impala-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Impala \u7684 classpath \u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $IMPALA_HOME/lib/","title":"2. \u5728\u6240\u6709 Impala \u8282\u70b9\u5b89\u88c5 JindoSDK"},{"location":"backup/oss/impala/jindosdk_on_impala/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/oss/impala/jindosdk_on_impala/#impala-core-sitexml-jindosdk-oss","text":"\u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002","title":"\u914d\u7f6e IMPALA \u4f7f\u7528\u7684 core-site.xml \u914d\u7f6e JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/oss/impala/jindosdk_on_impala/#4-impala-oss","text":"\u4e0b\u8f7d\u6d4b\u8bd5\u6570\u636e\u96c6 \u94fe\u63a5 \u4e0a\u4f20\u6570\u636e bash hadoop fs -put test_data oss://bucket/dir \u521b\u5efa\u8868 sql CREATE EXTERNAL TABLE customer_demographics ( `cd_demo_sk` INT, `cd_gender` STRING, `cd_marital_status` STRING, `cd_education_status` STRING, `cd_purchase_estimate` INT, `cd_credit_rating` STRING, `cd_dep_count` INT, `cd_dep_employed_count` INT, `cd_dep_college_count` INT) STORED AS PARQUET LOCATION 'oss://bucket/dir'; \u67e5\u8be2 OSS \u8868 sql select * from customer_demographics;","title":"4. \u4f7f\u7528 Impala \u8bbf\u95ee OSS\u3002"},{"location":"backup/oss/impala/jindosdk_on_impala/#5","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/oss/jindo_distcp/jindo_distcp_QA/","text":"1\u3001list \u6027\u80fd\u4f4e \u00a7 \u73b0\u8c61 \u00a7 \u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u53d1\u73b0 list \u6027\u80fd\u8f83\u6162\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Successfully list objects with prefix xxx/yyy/ in bucket xxx recursive 0 result 315 dur 100036.615031MS \u5176\u4e2d dur 100036.615031MS \u4ee3\u8868\u6b64\u6b21 list \u7528\u65f6\uff0c\u5355\u4f4d\u6beb\u79d2\uff0c\u5982\u4e0a\u8ff0\u4e00\u6b21 list \u8017\u65f6 100 \u79d2\uff0cOSS \u4e0a\u6587\u4ef6\u6b63\u5e38\u7684 list \u901f\u5ea6\u662f 1000 \u6587\u4ef6\u5728 1s \u4ee5\u4e0b\uff0c\u60a8\u53ef\u6839\u636e\u5f53\u524d\u76ee\u5f55\u4e0b\u6587\u4ef6\u6570\u91cf\u6765\u5224\u65ad\u8be5 list \u8017\u65f6\u662f\u5426\u5f02\u5e38\uff0c\u5982\u4e0a\u8ff0\u4fe1\u606f\u663e\u793a list 315 \u4e2a\u6587\u4ef6\u7684\u76ee\u5f55\u9700\u8981 100s \u663e\u7136\u662f\u4e0d\u592a\u6b63\u5e38\u7684\u3002 \u89e3\u51b3\u529e\u6cd5 \u00a7 \u589e\u52a0\u5ba2\u6237\u7aef\u5185\u5b58\uff0c\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u589e\u5927\u5ba2\u6237\u7aef\u5185\u5b58 export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx4096m\"","title":"jindo distcp QA"},{"location":"backup/oss/jindo_distcp/jindo_distcp_QA/#1list","text":"","title":"1\u3001list \u6027\u80fd\u4f4e"},{"location":"backup/oss/jindo_distcp/jindo_distcp_QA/#_1","text":"\u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u53d1\u73b0 list \u6027\u80fd\u8f83\u6162\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Successfully list objects with prefix xxx/yyy/ in bucket xxx recursive 0 result 315 dur 100036.615031MS \u5176\u4e2d dur 100036.615031MS \u4ee3\u8868\u6b64\u6b21 list \u7528\u65f6\uff0c\u5355\u4f4d\u6beb\u79d2\uff0c\u5982\u4e0a\u8ff0\u4e00\u6b21 list \u8017\u65f6 100 \u79d2\uff0cOSS \u4e0a\u6587\u4ef6\u6b63\u5e38\u7684 list \u901f\u5ea6\u662f 1000 \u6587\u4ef6\u5728 1s \u4ee5\u4e0b\uff0c\u60a8\u53ef\u6839\u636e\u5f53\u524d\u76ee\u5f55\u4e0b\u6587\u4ef6\u6570\u91cf\u6765\u5224\u65ad\u8be5 list \u8017\u65f6\u662f\u5426\u5f02\u5e38\uff0c\u5982\u4e0a\u8ff0\u4fe1\u606f\u663e\u793a list 315 \u4e2a\u6587\u4ef6\u7684\u76ee\u5f55\u9700\u8981 100s \u663e\u7136\u662f\u4e0d\u592a\u6b63\u5e38\u7684\u3002","title":"\u73b0\u8c61"},{"location":"backup/oss/jindo_distcp/jindo_distcp_QA/#_2","text":"\u589e\u52a0\u5ba2\u6237\u7aef\u5185\u5b58\uff0c\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u589e\u5927\u5ba2\u6237\u7aef\u5185\u5b58 export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx4096m\"","title":"\u89e3\u51b3\u529e\u6cd5"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/","text":"\u4f7f\u7528 JindoDistCp \u00a7 \u4ecb\u7ecd \u00a7 JindoDistCp\uff08\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u5de5\u5177\uff09\u662f\u7528\u4e8e\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u62f7\u8d1d\u6587\u4ef6\u7684\u5de5\u5177\u3002 \u5b83\u4f7f\u7528 MapReduce \u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3a MapReduce \u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002\u5168\u91cf\u652f\u6301 HDFS->OSS\uff0cHDFS->HDFS\uff0cOSS->HDFS\uff0cOSS->OSS \u7684\u6570\u636e\u62f7\u8d1d\u573a\u666f\uff0c\u63d0\u4f9b\u591a\u79cd\u4e2a\u6027\u5316\u62f7\u8d1d\u53c2\u6570\u548c\u591a\u79cd\u62f7\u8d1d\u7b56\u7565\u3002\u91cd\u70b9\u4f18\u5316 HDFS \u5230 OSS \u7684\u6570\u636e\u62f7\u8d1d\uff0c\u901a\u8fc7\u5b9a\u5236\u5316 CopyCommitter\uff0c\u5b9e\u73b0 No-Rename \u62f7\u8d1d\uff0c\u5e76\u4fdd\u8bc1\u6570\u636e\u62f7\u8d1d\u843d\u5730\u7684\u4e00\u81f4\u6027\u3002\u529f\u80fd\u5168\u91cf\u5bf9\u9f50 S3 DistCp \u548c HDFS DistCp\uff0c\u6027\u80fd\u8f83 HDFS DistCp \u6709\u8f83\u5927\u63d0\u5347\uff0c\u76ee\u6807\u63d0\u4f9b\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177\u3002 \u60a8\u53ef\u4ee5\u4f7f\u7528 JindoDistCp \u8fdb\u884c HDFS \u548c OSS \u4e4b\u95f4\u7684\u6570\u636e\u8fc1\u79fb\uff0c\u76f8\u5bf9\u4e8e Hadoop \u793e\u533a DistCp \u5b9e\u73b0\uff0c\u60a8\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u4f7f\u7528\u6307\u5357 \u00a7 JindoDistCp\u63d0\u4f9bjar\u5305\u5f62\u5f0f\u4f7f\u7528\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528hadoop jar\u547d\u4ee4\u914d\u5408\u4e00\u7cfb\u5217\u53c2\u6570\u6765\u5b8c\u6210distcp\u64cd\u4f5c\u3002 1\u3001\u4f7f\u7528--src\u548c--dest (\u5fc5\u9009) \u00a7 --src \u8868\u793a\u6307\u5b9a\u6e90\u6587\u4ef6\u7684\u8def\u5f84 --dest \u8868\u793a\u76ee\u6807\u6587\u4ef6\u7684\u8def\u5f84 \u5982\u4eceHDFS\u4e0a/opt/tmp\u76ee\u5f55\u62f7\u8d1d\u5230OSS bucket\uff0c\u53ef\u4ee5\u6267\u884c\u5982\u4e0b\u8bed\u53e5 hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht/tmp \u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9adest\u8def\u5f84\u6765\u786e\u5b9a\u62f7\u8d1d\u540e\u7684\u6587\u4ef6\u5c42\u6b21\uff0c\u5982\u60a8\u9700\u8981\u5c06/opt/tmp\u4e0b\u7684\u6587\u4ef6\u62f7\u8d1d\u5230yang-hhht\u8fd9\u4e2abucket\u4e0b\u7684tmp\u76ee\u5f55\u4e0b\uff0c\u5219\u53ef\u4ee5\u4f7f\u7528\u4e0a\u8ff0\u8bed\u53e5\u6765\u5b8c\u6210\u3002\u6b64\u5904\u548cHadoop\u7684distcp\u884c\u4e3a\u6709\u6240\u4e0d\u540c\uff0cJindoDistCp\u4f1a\u9ed8\u8ba4\u5c06src\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6587\u4ef6\u62f7\u8d1d\u5230\u60a8\u6307\u5b9adest\u8def\u5f84\u4e0b\uff0c\u5e76\u4e0d\u5305\u62ec\u5f53\u524d\u7684\u6839\u76ee\u5f55\u540d\u79f0\uff0c\u60a8\u53ef\u4ee5\u5728dest\u4e2d\u6307\u5b9a\u62f7\u8d1d\u8def\u5f84\u7684\u6839\u76ee\u5f55\uff0c\u5982\u679c\u4e0d\u5b58\u5728\u4f1a\u81ea\u52a8\u521b\u5efa\u3002 \u6ce8\u610f\uff1a\u5982\u9700\u62f7\u8d1d\u5355\u4e2a\u6587\u4ef6, dest\u9700\u6307\u5b9a\u4e3a\u76ee\u5f55\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /test.txt --dest oss://destBucket/tmp 2\u3001\u4f7f\u7528--bandWidth \u00a7 \u5728\u60a8\u7684distcp\u8fc7\u7a0b\u4e2d\uff0c\u60a8\u53ef\u4ee5\u6307\u5b9a\u672c\u6b21distcp\u4efb\u52a1\u6240\u7528\u7684\u5355\u673a\u5e26\u5bbd(\u4ee5MB\u4e3a\u5355\u4f4d)\uff0c\u907f\u514d\u5355\u673a\u5360\u7528\u8fc7\u5927\u5e26\u5bbd \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --bandWidth 6 3\u3001\u4f7f\u7528--codec \u00a7 \u539f\u59cb\u6587\u4ef6\u901a\u5e38\u4ee5\u672a\u538b\u7f29\u7684\u6587\u672c\u683c\u5f0f\u8fdb\u5165OSS\u6216\u8005HDFS\u3002\u65e0\u8bba\u662f\u5b58\u50a8\u6210\u672c\u8fd8\u662f\u5bf9\u8be5\u6570\u636e\u8fdb\u884c\u5206\u6790\uff0c\u6b64\u683c\u5f0f\u90fd\u4e0d\u751a\u7406\u60f3\u3002JindoDistCp \u53ef\u4ee5\u4f7f\u7528 --codec \u9009\u9879\u5e2e\u52a9\u60a8\u5728\u7ebf\u9ad8\u6548\u5730\u5b58\u50a8\u6570\u636e\u548c\u538b\u7f29\u6587\u4ef6 \u547d\u4ee4\u793a\u4f8b\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --codec gz --parallelism 20 JindoDistCp \u7684\u5f53\u524d\u7248\u672c\u652f\u6301\u7f16\u89e3\u7801\u5668 gzip\u3001gz\u3001lzo\u3001lzop \u548c snappy \u4ee5\u53ca\u5173\u952e\u5b57 none \u548c keep \uff08\u9ed8\u8ba4\u503c\uff09\u3002\u8fd9\u4e9b\u5173\u952e\u5b57\u542b\u4e49\u5982\u4e0b\uff1a \"none\" \u2013 \u4fdd\u5b58\u4e3a\u672a\u538b\u7f29\u7684\u6587\u4ef6\u3002\u5982\u679c\u6587\u4ef6\u5df2\u538b\u7f29\uff0c\u5219 JindoDistCp \u4f1a\u5c06\u5176\u89e3\u538b\u7f29\u3002 \"keep\" \u2013 \u4e0d\u66f4\u6539\u6587\u4ef6\u538b\u7f29\u5f62\u6001\uff0c\u6309\u539f\u6837\u590d\u5236\u3002 \u6211\u4eec\u68c0\u67e5\u4e00\u4e0b\u76ee\u6807\u6587\u4ef6\u5939\u4e2d\u7684\u6587\u4ef6\uff0c\u8fd9\u4e9b\u6587\u4ef6\u73b0\u5728\u5df2\u7ecf\u7528 gz \u7f16\u89e3\u7801\u5668\u538b\u7f29\u4e86\uff1a [root@emr-header-1 opt]# hdfs dfs -ls oss://yang-hhht/hourly_table/2017-02-01/03 Found 6 items -rw-rw-rw- 1 938 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/000151.sst.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/1.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/2.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/OPTIONS-000109.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/emp01.txt.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/emp06.txt.gz \u5982\u60a8\u5728\u5f00\u6e90Hadoop\u96c6\u7fa4\u73af\u5883\u4e2d\u4f7f\u7528lzo\u538b\u7f29\u529f\u80fd\uff0c\u5219\u60a8\u9700\u8981\u53bb\u5b89\u88c5gplcompression\u7684native\u5e93\u548chadoop-lzo\u5305\uff0c\u5982\u60a8\u7f3a\u5c11\u76f8\u5173\u73af\u5883\uff0c\u5efa\u8bae\u4f7f\u7528\u5176\u4ed6\u538b\u7f29\u65b9\u5f0f\u8fdb\u884c\u538b\u7f29\u3002 4\u3001\u4f7f\u7528\u51b7\u5f52\u6863/\u5f52\u6863/\u4f4e\u9891\u5199\u5165OSS \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--policy\u6765\u6307\u5b9a\u4ee5\u51b7\u5f52\u6863\u3001\u5f52\u6863\u548c\u4f4e\u9891\u7684\u6a21\u5f0f\u5199\u5165OSS\uff0c\u8fdb\u884c\u6570\u636e\u5b58\u50a8\u3002 4.1\u3001\u4f7f\u7528\u51b7\u5f52\u6863\u5199\u5165OSS\uff08coldArchive\uff09 \u00a7 \u76ee\u524d\u53ea\u5728\u90e8\u5206region\u53ef\u7528\uff0c\u5177\u4f53\u53c2\u89c1 OSS\u5b58\u50a8\u7c7b\u578b\u4ecb\u7ecd \uff0c\u4f7f\u7528\u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --policy coldArchive --parallelism 20 4.2\u3001\u4f7f\u7528\u5f52\u6863\u5199\u5165OSS\uff08archive\uff09 \u00a7 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --policy archive --parallelism 20 4.3\u3001\u4f7f\u7528\u4f4e\u9891\u5199\u5165OSS\uff08ia\uff09 \u00a7 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --policy ia --parallelism 20 \u5982\u60a8\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4\u4ee5standard\u53ca\u6807\u51c6\u6a21\u5f0f\u5199\u5165\uff0c\u4e0d\u8fdb\u884c\u51b7\u5f52\u6863\u3001\u5f52\u6863\u548c\u4f4e\u9891\u64cd\u4f5c 5\u3001\u4f7f\u7528--filters \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--filters\u6765\u6307\u5b9a\u6709\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --filters filter.txt \u5047\u8bbe filter.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u4e0d\u4f1a\u5c06\u8def\u5f84\u4e2d\u5e26\u6709 test \u7684\u6587\u4ef6\u62f7\u8d1d\u5230OSS\u3002 6\u3001\u4f7f\u7528--srcPrefixesFile \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--srcPrefixesFile\u6765\u6307\u5b9a\u6709\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --srcPrefixesFile prefixes.txt \u5047\u8bbe prefixes.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u53ea\u6709\u8def\u5f84\u4e2d\u5e26\u6709 test \u7684\u6587\u4ef6\u624d\u4f1a\u62f7\u8d1d\u5230OSS\u3002 7\u3001\u4f7f\u7528--parallelism \u00a7 parallelism \u53c2\u6570\u7528\u6765\u6307\u5b9aMR\u4efb\u52a1\u91cc\u7684 mapreduce.job.maps \u53c2\u6570\uff0c\u60a8\u53ef\u4ee5\u6839\u636e\u96c6\u7fa4\u7684\u8d44\u6e90\u60c5\u51b5\u81ea\u5b9a\u4e49 parallelism \u7684\u5927\u5c0f\u6765\u63a7\u5236 distcp \u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\u3002 \u5982\u4eceHDFS\u4e0a/opt/tmp\u76ee\u5f55\u62f7\u8d1d\u5230OSS bucket\uff0c\u53ef\u4ee5\u6267\u884c hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht/tmp --parallelism 20 8\u3001\u4f7f\u7528--taskBatch \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--taskBatch\u6765\u6307\u5b9a\u6bcf\u4e2adistcp task\u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --taskBatch 10 9\u3001\u4f7f\u7528--tmp \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7tmp\u53c2\u6570\u6307\u5b9a\u4e00\u4e2a\u4e34\u65f6\u76ee\u5f55\u5728hdfs\u4e0a\u4f20\u5b58\u653e\u4e34\u65f6\u6570\u636e\uff0c\u9ed8\u8ba4\u503c\u4e3a/tmp\uff0c\u5373hdfs:///tmp/\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --parallelism 20 --tmp /tmp 10\u3001\u4f7f\u7528OSS AK \u00a7 \u5728 EMR \u5916\u6216\u8005\u514d\u5bc6\u670d\u52a1\u51fa\u73b0\u95ee\u9898\u7684\u60c5\u51b5\u4e0b\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9aAK\u6765\u83b7\u5f97\u8bbf\u95ee OSS \u7684\u6743\u9650\u3002\u60a8\u53ef\u4ee5\u5728\u547d\u4ee4\u4e2d\u4f7f\u7528 --hadoopConf \u9009\u9879\u6765\u6307\u5b9aAK\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --parallelism 20 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) \u60a8\u4e5f\u53ef\u4ee5\u5c06oss\u7684ak\u3001secret\u3001endpoint\u9884\u5148\u914d\u7f6e\u5728 hadoop\u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u907f\u514d\u6bcf\u6b21\u4f7f\u7528\u65f6\u4e34\u65f6\u586b\u5199ak\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> 11\u3001\u4f7f\u7528--disableChecksum \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--disableChecksum\u5173\u95ed\u68c0\u67e5\u6587\u4ef6checksum\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --disableChecksum 12\u3001\u4f7f\u7528--deleteOnSuccess \u00a7 \u6709\u65f6\u5019\u6211\u4eec\u662f\u60f3\u8981\u79fb\u52a8\u6570\u636e\u800c\u4e0d\u662f\u590d\u5236\u6570\u636e\uff0c\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 --deleteOnSuccess \u9009\u9879\u3002\u6b64\u9009\u9879\u7c7b\u4f3c\u4e8emv\u64cd\u4f5c\uff0c\u9996\u5148\u590d\u5236\u6587\u4ef6\uff0c\u7136\u540e\u4ece\u6e90\u4f4d\u7f6e\u5220\u9664\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --deleteOnSuccess --parallelism 20 13\u3001\u4f7f\u7528--enableTransaction \u00a7 JindoDistCp\u9ed8\u8ba4\u4f7f\u7528task\u7ea7\u522b\u5b8c\u6574\u6027\uff0c\u5982\u60a8\u9700\u8981\u4fdd\u8bc1Job\u7ea7\u522b\u7684\u5b8c\u6574\u6027\u4ee5\u53ca\u4fdd\u8bc1Job\u4e4b\u95f4\u7684\u4e8b\u52a1\u652f\u6301\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528--enableTransaction\u53c2\u6570\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --enableTransaction --parallelism 20 14\u3001\u4f7f\u7528--ignore \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7ignore\u53c2\u6570\uff0c\u5ffd\u7565\u6570\u636e\u8fc1\u79fb\u671f\u95f4\u53d1\u751f\u7684\u5f02\u5e38\uff0c\u76f8\u5173\u62a5\u9519\u4e0d\u4f1a\u4e2d\u65ad\u4efb\u52a1\uff0c\u5e76\u6700\u7ec8\u4ee5DistCp Counter\u7684\u5f62\u5f0f\u900f\u51fa\u3002\uff08\u5982\u679c\u5f00\u542fCMS\u7684\u8bdd\uff0c\u4e5f\u4f1a\u4ee5\u6307\u5b9a\u65b9\u5f0f\u8fdb\u884c\u901a\u77e5\u3002\uff09 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --ignore --parallelism 20 15\u3001\u4f7f\u7528--diff \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5b8c\u6210\u540e\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9asrc\u548cdest\u6765\u67e5\u770b\u5f53\u524ddistcp\u7684\u6587\u4ef6\u5dee\u5f02, \u5982\u679csrc\u7684\u6587\u4ef6\u672a\u80fd\u540c\u6b65\u5230dest\u4e0a\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\u3002\u5982\u60a8\u7684distcp\u4efb\u52a1\u5305\u542b\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u5219--diff\u4e0d\u80fd\u663e\u793a\u6b63\u786e\u7684\u6587\u4ef6\u5dee\u5f02\uff0c\u56e0\u4e3a\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u4f1a\u6539\u53d8\u6587\u4ef6\u7684\u5927\u5c0f\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --diff \u5982\u679c\u6587\u4ef6\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter DIFF_FILES=1 \u5982\u679c\u60a8\u7684--dest\u4e3aHDFS\u8def\u5f84\uff0c\u73b0\u652f\u6301 /path \uff0c hdfs://hostname:ip/path \uff0c hdfs://headerIp:ip/path \u7684\u5199\u6cd5\uff0c\u6682\u4e0d\u652f\u6301 hdfs:///path \uff0c hdfs:/path \u548c\u5176\u4ed6\u81ea\u5b9a\u4e49\u5199\u6cd5\u3002 \u5982\u679c\u60f3\u67e5\u770b\u6587\u4ef6\u5143\u6570\u636e\u7684\u5dee\u5f02\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 --diff --preserveMeta \u547d\u4ee4\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --diff --preserveMeta \u5982\u679c\u6587\u4ef6\u5143\u6570\u636e\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5143\u6570\u636e\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter META_FILE_DIFF=1 16\u3001\u4f7f\u7528--update \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7update\u53c2\u6570\u4f7f\u7528\u589e\u91cf\u540c\u6b65\u529f\u80fd\uff0c\u8df3\u8fc7\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u548c\u76ee\u5f55\uff0c\u76f4\u63a5\u5c06src\u4e2d\u65b0\u589e\u6216\u53d1\u751f\u6539\u53d8\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u540c\u6b65\u5230dest\u4e0a\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --update --parallelism 20 17\u3001\u4f7f\u7528--preserveMeta \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht/tmp --preserveMeta 18\u3001\u4f7f\u7528--jobBatch \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--jobBatch\u6765\u6307\u5b9a\u6bcf\u4e2adistcp job\u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10000\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --jobBatch 50000","title":"\u4f7f\u7528 JindoDistCp"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#jindodistcp","text":"","title":"\u4f7f\u7528 JindoDistCp"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#_1","text":"JindoDistCp\uff08\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u5de5\u5177\uff09\u662f\u7528\u4e8e\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u62f7\u8d1d\u6587\u4ef6\u7684\u5de5\u5177\u3002 \u5b83\u4f7f\u7528 MapReduce \u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3a MapReduce \u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002\u5168\u91cf\u652f\u6301 HDFS->OSS\uff0cHDFS->HDFS\uff0cOSS->HDFS\uff0cOSS->OSS \u7684\u6570\u636e\u62f7\u8d1d\u573a\u666f\uff0c\u63d0\u4f9b\u591a\u79cd\u4e2a\u6027\u5316\u62f7\u8d1d\u53c2\u6570\u548c\u591a\u79cd\u62f7\u8d1d\u7b56\u7565\u3002\u91cd\u70b9\u4f18\u5316 HDFS \u5230 OSS \u7684\u6570\u636e\u62f7\u8d1d\uff0c\u901a\u8fc7\u5b9a\u5236\u5316 CopyCommitter\uff0c\u5b9e\u73b0 No-Rename \u62f7\u8d1d\uff0c\u5e76\u4fdd\u8bc1\u6570\u636e\u62f7\u8d1d\u843d\u5730\u7684\u4e00\u81f4\u6027\u3002\u529f\u80fd\u5168\u91cf\u5bf9\u9f50 S3 DistCp \u548c HDFS DistCp\uff0c\u6027\u80fd\u8f83 HDFS DistCp \u6709\u8f83\u5927\u63d0\u5347\uff0c\u76ee\u6807\u63d0\u4f9b\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177\u3002 \u60a8\u53ef\u4ee5\u4f7f\u7528 JindoDistCp \u8fdb\u884c HDFS \u548c OSS \u4e4b\u95f4\u7684\u6570\u636e\u8fc1\u79fb\uff0c\u76f8\u5bf9\u4e8e Hadoop \u793e\u533a DistCp \u5b9e\u73b0\uff0c\u60a8\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"\u4ecb\u7ecd"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#_2","text":"JindoDistCp\u63d0\u4f9bjar\u5305\u5f62\u5f0f\u4f7f\u7528\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528hadoop jar\u547d\u4ee4\u914d\u5408\u4e00\u7cfb\u5217\u53c2\u6570\u6765\u5b8c\u6210distcp\u64cd\u4f5c\u3002","title":"\u4f7f\u7528\u6307\u5357"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#1-src-dest","text":"--src \u8868\u793a\u6307\u5b9a\u6e90\u6587\u4ef6\u7684\u8def\u5f84 --dest \u8868\u793a\u76ee\u6807\u6587\u4ef6\u7684\u8def\u5f84 \u5982\u4eceHDFS\u4e0a/opt/tmp\u76ee\u5f55\u62f7\u8d1d\u5230OSS bucket\uff0c\u53ef\u4ee5\u6267\u884c\u5982\u4e0b\u8bed\u53e5 hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht/tmp \u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9adest\u8def\u5f84\u6765\u786e\u5b9a\u62f7\u8d1d\u540e\u7684\u6587\u4ef6\u5c42\u6b21\uff0c\u5982\u60a8\u9700\u8981\u5c06/opt/tmp\u4e0b\u7684\u6587\u4ef6\u62f7\u8d1d\u5230yang-hhht\u8fd9\u4e2abucket\u4e0b\u7684tmp\u76ee\u5f55\u4e0b\uff0c\u5219\u53ef\u4ee5\u4f7f\u7528\u4e0a\u8ff0\u8bed\u53e5\u6765\u5b8c\u6210\u3002\u6b64\u5904\u548cHadoop\u7684distcp\u884c\u4e3a\u6709\u6240\u4e0d\u540c\uff0cJindoDistCp\u4f1a\u9ed8\u8ba4\u5c06src\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6587\u4ef6\u62f7\u8d1d\u5230\u60a8\u6307\u5b9adest\u8def\u5f84\u4e0b\uff0c\u5e76\u4e0d\u5305\u62ec\u5f53\u524d\u7684\u6839\u76ee\u5f55\u540d\u79f0\uff0c\u60a8\u53ef\u4ee5\u5728dest\u4e2d\u6307\u5b9a\u62f7\u8d1d\u8def\u5f84\u7684\u6839\u76ee\u5f55\uff0c\u5982\u679c\u4e0d\u5b58\u5728\u4f1a\u81ea\u52a8\u521b\u5efa\u3002 \u6ce8\u610f\uff1a\u5982\u9700\u62f7\u8d1d\u5355\u4e2a\u6587\u4ef6, dest\u9700\u6307\u5b9a\u4e3a\u76ee\u5f55\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /test.txt --dest oss://destBucket/tmp","title":"1\u3001\u4f7f\u7528--src\u548c--dest (\u5fc5\u9009)"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#2-bandwidth","text":"\u5728\u60a8\u7684distcp\u8fc7\u7a0b\u4e2d\uff0c\u60a8\u53ef\u4ee5\u6307\u5b9a\u672c\u6b21distcp\u4efb\u52a1\u6240\u7528\u7684\u5355\u673a\u5e26\u5bbd(\u4ee5MB\u4e3a\u5355\u4f4d)\uff0c\u907f\u514d\u5355\u673a\u5360\u7528\u8fc7\u5927\u5e26\u5bbd \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --bandWidth 6","title":"2\u3001\u4f7f\u7528--bandWidth"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#3-codec","text":"\u539f\u59cb\u6587\u4ef6\u901a\u5e38\u4ee5\u672a\u538b\u7f29\u7684\u6587\u672c\u683c\u5f0f\u8fdb\u5165OSS\u6216\u8005HDFS\u3002\u65e0\u8bba\u662f\u5b58\u50a8\u6210\u672c\u8fd8\u662f\u5bf9\u8be5\u6570\u636e\u8fdb\u884c\u5206\u6790\uff0c\u6b64\u683c\u5f0f\u90fd\u4e0d\u751a\u7406\u60f3\u3002JindoDistCp \u53ef\u4ee5\u4f7f\u7528 --codec \u9009\u9879\u5e2e\u52a9\u60a8\u5728\u7ebf\u9ad8\u6548\u5730\u5b58\u50a8\u6570\u636e\u548c\u538b\u7f29\u6587\u4ef6 \u547d\u4ee4\u793a\u4f8b\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --codec gz --parallelism 20 JindoDistCp \u7684\u5f53\u524d\u7248\u672c\u652f\u6301\u7f16\u89e3\u7801\u5668 gzip\u3001gz\u3001lzo\u3001lzop \u548c snappy \u4ee5\u53ca\u5173\u952e\u5b57 none \u548c keep \uff08\u9ed8\u8ba4\u503c\uff09\u3002\u8fd9\u4e9b\u5173\u952e\u5b57\u542b\u4e49\u5982\u4e0b\uff1a \"none\" \u2013 \u4fdd\u5b58\u4e3a\u672a\u538b\u7f29\u7684\u6587\u4ef6\u3002\u5982\u679c\u6587\u4ef6\u5df2\u538b\u7f29\uff0c\u5219 JindoDistCp \u4f1a\u5c06\u5176\u89e3\u538b\u7f29\u3002 \"keep\" \u2013 \u4e0d\u66f4\u6539\u6587\u4ef6\u538b\u7f29\u5f62\u6001\uff0c\u6309\u539f\u6837\u590d\u5236\u3002 \u6211\u4eec\u68c0\u67e5\u4e00\u4e0b\u76ee\u6807\u6587\u4ef6\u5939\u4e2d\u7684\u6587\u4ef6\uff0c\u8fd9\u4e9b\u6587\u4ef6\u73b0\u5728\u5df2\u7ecf\u7528 gz \u7f16\u89e3\u7801\u5668\u538b\u7f29\u4e86\uff1a [root@emr-header-1 opt]# hdfs dfs -ls oss://yang-hhht/hourly_table/2017-02-01/03 Found 6 items -rw-rw-rw- 1 938 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/000151.sst.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/1.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/2.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/OPTIONS-000109.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/emp01.txt.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/emp06.txt.gz \u5982\u60a8\u5728\u5f00\u6e90Hadoop\u96c6\u7fa4\u73af\u5883\u4e2d\u4f7f\u7528lzo\u538b\u7f29\u529f\u80fd\uff0c\u5219\u60a8\u9700\u8981\u53bb\u5b89\u88c5gplcompression\u7684native\u5e93\u548chadoop-lzo\u5305\uff0c\u5982\u60a8\u7f3a\u5c11\u76f8\u5173\u73af\u5883\uff0c\u5efa\u8bae\u4f7f\u7528\u5176\u4ed6\u538b\u7f29\u65b9\u5f0f\u8fdb\u884c\u538b\u7f29\u3002","title":"3\u3001\u4f7f\u7528--codec"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#4oss","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--policy\u6765\u6307\u5b9a\u4ee5\u51b7\u5f52\u6863\u3001\u5f52\u6863\u548c\u4f4e\u9891\u7684\u6a21\u5f0f\u5199\u5165OSS\uff0c\u8fdb\u884c\u6570\u636e\u5b58\u50a8\u3002","title":"4\u3001\u4f7f\u7528\u51b7\u5f52\u6863/\u5f52\u6863/\u4f4e\u9891\u5199\u5165OSS"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#41osscoldarchive","text":"\u76ee\u524d\u53ea\u5728\u90e8\u5206region\u53ef\u7528\uff0c\u5177\u4f53\u53c2\u89c1 OSS\u5b58\u50a8\u7c7b\u578b\u4ecb\u7ecd \uff0c\u4f7f\u7528\u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --policy coldArchive --parallelism 20","title":"4.1\u3001\u4f7f\u7528\u51b7\u5f52\u6863\u5199\u5165OSS\uff08coldArchive\uff09"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#42ossarchive","text":"\u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --policy archive --parallelism 20","title":"4.2\u3001\u4f7f\u7528\u5f52\u6863\u5199\u5165OSS\uff08archive\uff09"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#43ossia","text":"\u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --policy ia --parallelism 20 \u5982\u60a8\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4\u4ee5standard\u53ca\u6807\u51c6\u6a21\u5f0f\u5199\u5165\uff0c\u4e0d\u8fdb\u884c\u51b7\u5f52\u6863\u3001\u5f52\u6863\u548c\u4f4e\u9891\u64cd\u4f5c","title":"4.3\u3001\u4f7f\u7528\u4f4e\u9891\u5199\u5165OSS\uff08ia\uff09"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#5-filters","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--filters\u6765\u6307\u5b9a\u6709\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --filters filter.txt \u5047\u8bbe filter.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u4e0d\u4f1a\u5c06\u8def\u5f84\u4e2d\u5e26\u6709 test \u7684\u6587\u4ef6\u62f7\u8d1d\u5230OSS\u3002","title":"5\u3001\u4f7f\u7528--filters"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#6-srcprefixesfile","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--srcPrefixesFile\u6765\u6307\u5b9a\u6709\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --srcPrefixesFile prefixes.txt \u5047\u8bbe prefixes.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u53ea\u6709\u8def\u5f84\u4e2d\u5e26\u6709 test \u7684\u6587\u4ef6\u624d\u4f1a\u62f7\u8d1d\u5230OSS\u3002","title":"6\u3001\u4f7f\u7528--srcPrefixesFile"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#7-parallelism","text":"parallelism \u53c2\u6570\u7528\u6765\u6307\u5b9aMR\u4efb\u52a1\u91cc\u7684 mapreduce.job.maps \u53c2\u6570\uff0c\u60a8\u53ef\u4ee5\u6839\u636e\u96c6\u7fa4\u7684\u8d44\u6e90\u60c5\u51b5\u81ea\u5b9a\u4e49 parallelism \u7684\u5927\u5c0f\u6765\u63a7\u5236 distcp \u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\u3002 \u5982\u4eceHDFS\u4e0a/opt/tmp\u76ee\u5f55\u62f7\u8d1d\u5230OSS bucket\uff0c\u53ef\u4ee5\u6267\u884c hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht/tmp --parallelism 20","title":"7\u3001\u4f7f\u7528--parallelism"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#8-taskbatch","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--taskBatch\u6765\u6307\u5b9a\u6bcf\u4e2adistcp task\u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --taskBatch 10","title":"8\u3001\u4f7f\u7528--taskBatch"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#9-tmp","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7tmp\u53c2\u6570\u6307\u5b9a\u4e00\u4e2a\u4e34\u65f6\u76ee\u5f55\u5728hdfs\u4e0a\u4f20\u5b58\u653e\u4e34\u65f6\u6570\u636e\uff0c\u9ed8\u8ba4\u503c\u4e3a/tmp\uff0c\u5373hdfs:///tmp/\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --parallelism 20 --tmp /tmp","title":"9\u3001\u4f7f\u7528--tmp"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#10oss-ak","text":"\u5728 EMR \u5916\u6216\u8005\u514d\u5bc6\u670d\u52a1\u51fa\u73b0\u95ee\u9898\u7684\u60c5\u51b5\u4e0b\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9aAK\u6765\u83b7\u5f97\u8bbf\u95ee OSS \u7684\u6743\u9650\u3002\u60a8\u53ef\u4ee5\u5728\u547d\u4ee4\u4e2d\u4f7f\u7528 --hadoopConf \u9009\u9879\u6765\u6307\u5b9aAK\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --parallelism 20 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) \u60a8\u4e5f\u53ef\u4ee5\u5c06oss\u7684ak\u3001secret\u3001endpoint\u9884\u5148\u914d\u7f6e\u5728 hadoop\u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u907f\u514d\u6bcf\u6b21\u4f7f\u7528\u65f6\u4e34\u65f6\u586b\u5199ak\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration>","title":"10\u3001\u4f7f\u7528OSS AK"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#11-disablechecksum","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--disableChecksum\u5173\u95ed\u68c0\u67e5\u6587\u4ef6checksum\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --disableChecksum","title":"11\u3001\u4f7f\u7528--disableChecksum"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#12-deleteonsuccess","text":"\u6709\u65f6\u5019\u6211\u4eec\u662f\u60f3\u8981\u79fb\u52a8\u6570\u636e\u800c\u4e0d\u662f\u590d\u5236\u6570\u636e\uff0c\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 --deleteOnSuccess \u9009\u9879\u3002\u6b64\u9009\u9879\u7c7b\u4f3c\u4e8emv\u64cd\u4f5c\uff0c\u9996\u5148\u590d\u5236\u6587\u4ef6\uff0c\u7136\u540e\u4ece\u6e90\u4f4d\u7f6e\u5220\u9664\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --deleteOnSuccess --parallelism 20","title":"12\u3001\u4f7f\u7528--deleteOnSuccess"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#13-enabletransaction","text":"JindoDistCp\u9ed8\u8ba4\u4f7f\u7528task\u7ea7\u522b\u5b8c\u6574\u6027\uff0c\u5982\u60a8\u9700\u8981\u4fdd\u8bc1Job\u7ea7\u522b\u7684\u5b8c\u6574\u6027\u4ee5\u53ca\u4fdd\u8bc1Job\u4e4b\u95f4\u7684\u4e8b\u52a1\u652f\u6301\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528--enableTransaction\u53c2\u6570\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --enableTransaction --parallelism 20","title":"13\u3001\u4f7f\u7528--enableTransaction"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#14-ignore","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7ignore\u53c2\u6570\uff0c\u5ffd\u7565\u6570\u636e\u8fc1\u79fb\u671f\u95f4\u53d1\u751f\u7684\u5f02\u5e38\uff0c\u76f8\u5173\u62a5\u9519\u4e0d\u4f1a\u4e2d\u65ad\u4efb\u52a1\uff0c\u5e76\u6700\u7ec8\u4ee5DistCp Counter\u7684\u5f62\u5f0f\u900f\u51fa\u3002\uff08\u5982\u679c\u5f00\u542fCMS\u7684\u8bdd\uff0c\u4e5f\u4f1a\u4ee5\u6307\u5b9a\u65b9\u5f0f\u8fdb\u884c\u901a\u77e5\u3002\uff09 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --ignore --parallelism 20","title":"14\u3001\u4f7f\u7528--ignore"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#15-diff","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5b8c\u6210\u540e\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9asrc\u548cdest\u6765\u67e5\u770b\u5f53\u524ddistcp\u7684\u6587\u4ef6\u5dee\u5f02, \u5982\u679csrc\u7684\u6587\u4ef6\u672a\u80fd\u540c\u6b65\u5230dest\u4e0a\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\u3002\u5982\u60a8\u7684distcp\u4efb\u52a1\u5305\u542b\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u5219--diff\u4e0d\u80fd\u663e\u793a\u6b63\u786e\u7684\u6587\u4ef6\u5dee\u5f02\uff0c\u56e0\u4e3a\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u4f1a\u6539\u53d8\u6587\u4ef6\u7684\u5927\u5c0f\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --diff \u5982\u679c\u6587\u4ef6\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter DIFF_FILES=1 \u5982\u679c\u60a8\u7684--dest\u4e3aHDFS\u8def\u5f84\uff0c\u73b0\u652f\u6301 /path \uff0c hdfs://hostname:ip/path \uff0c hdfs://headerIp:ip/path \u7684\u5199\u6cd5\uff0c\u6682\u4e0d\u652f\u6301 hdfs:///path \uff0c hdfs:/path \u548c\u5176\u4ed6\u81ea\u5b9a\u4e49\u5199\u6cd5\u3002 \u5982\u679c\u60f3\u67e5\u770b\u6587\u4ef6\u5143\u6570\u636e\u7684\u5dee\u5f02\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 --diff --preserveMeta \u547d\u4ee4\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --diff --preserveMeta \u5982\u679c\u6587\u4ef6\u5143\u6570\u636e\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5143\u6570\u636e\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter META_FILE_DIFF=1","title":"15\u3001\u4f7f\u7528--diff"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#16-update","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7update\u53c2\u6570\u4f7f\u7528\u589e\u91cf\u540c\u6b65\u529f\u80fd\uff0c\u8df3\u8fc7\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u548c\u76ee\u5f55\uff0c\u76f4\u63a5\u5c06src\u4e2d\u65b0\u589e\u6216\u53d1\u751f\u6539\u53d8\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u540c\u6b65\u5230dest\u4e0a\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --update --parallelism 20","title":"16\u3001\u4f7f\u7528--update"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#17-preservemeta","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht/tmp --preserveMeta","title":"17\u3001\u4f7f\u7528--preserveMeta"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#18-jobbatch","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--jobBatch\u6765\u6307\u5b9a\u6bcf\u4e2adistcp job\u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10000\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --jobBatch 50000","title":"18\u3001\u4f7f\u7528--jobBatch"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_api/","text":"JindoDistCp API \u4f7f\u7528\u8bf4\u660e \u00a7 \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 \u5b89\u88c5\u4f9d\u8d56\u5305 \u00a7 \u76ee\u524d JindoSDK/JindoDistCP \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u4ee54.6.12\u7248\u672c\u4e3a\u4f8b\uff0c\u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoDistCP \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-distcp --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop with provided scope, your runtime should install hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> </dependencies> </project> \u793a\u4f8b\u4ee3\u7801 \u00a7 \u4ee5\u62f7\u8d1d HDFS \u6587\u4ef6\u5939\u5230\u963f\u91cc\u4e91 OSS \u4e3a\u4f8b\u3002 import com.aliyun.jindodata.distcp.util.DistCpCounter; import com.aliyun.jindodata.distjob.request.DistJobResponse; import org.apache.hadoop.conf.Configuration; import java.io.IOException; public class DistcpExample { static public void main(String[] args) throws IOException { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. conf.set(\"fs.oss.credentials.provider\", \"com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider\"); conf.set(\"fs.oss.accessKeyId\", \"XXX\"); conf.set(\"fs.oss.accessKeySecret\", \"XXX\"); conf.set(\"fs.oss.endpoint\", \"oss-cn-xxx.aliyuncs.com\"); DistCpRequest request = new DistCpRequest(); request.setSrcDir(\"hdfs:///user/root/random-data\"); request.setDestDir(\"oss://oss-bucket/dest\"); request.setWorkTempDir(\"hdfs:///user/root/tmp\"); request.setConf(conf); // Jindo DistCp Service DistCpService service = new DistCpService(); service.buildAllJobs(request); service.runAllJobs(); DistJobResponse response = service.collectAllJobResult(); printResponse(response); } static public void printResponse(DistJobResponse response) { System.out.println(\"ErrorCode: \" + response.getErrorCode()); System.out.println(\"ErrorMsg: \" + response.getErrorMsg()); System.out.println(\"FileCopied: \" + response.getCount(DistCpCounter.FILES_COPIED.getName())); System.out.println(\"BytesCopied: \" + response.getCount(DistCpCounter.BYTES_COPIED.getName())); } } \u8bf7\u6c42\u53c2\u6570 \u00a7 \u76f8\u5173\u65b9\u6cd5\u7684\u5177\u4f53\u63cf\u8ff0\u53ef\u53c2\u8003 \u300a\u4f7f\u7528Jindo DistCp\u300b \u65b9\u6cd5 \u53c2\u6570\u7c7b\u578b \u8bf4\u660e \u9ed8\u8ba4\u503c \u517c\u5bb9 void setSrc(String src) \u5fc5\u9009 \u8bbe\u7f6e\u6e90\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setDest(String dest) \u5fc5\u9009 \u8bbe\u7f6e\u76ee\u6807\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setConf(Configuration conf) \u5fc5\u9009 \u8bbe\u7f6e Configuration - 4.3.0+ void setMode(DistCpMode mode) \u53ef\u9009 \u8bbe\u7f6eDistCp\u6a21\u5f0f\uff0c\u652f\u6301\u7684\u6a21\u5f0f\u6709 COPY\uff08\u62f7\u8d1d\uff09 DIFF\uff08\u5dee\u5f02\u6bd4\u8f83\uff09 UPADTE\uff08\u589e\u91cf\u62f7\u8d1d\uff09 DistCpMode.COPY 4.3.0+ void setNumWorkers(int numWorkers) \u53ef\u9009 \u8bbe\u7f6eDistCp\u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\uff0c\u5bf9\u5e94MR\u4efb\u52a1\u4e2d\u7684 mapreduce.job.maps 10 4.3.0+ void setBandWidthLimit(int bandWidthLimit) \u53ef\u9009 \u8bbe\u7f6e\u5355\u4e2a\u8282\u70b9\u7684\u5e26\u5bbd\u9650\u5236\uff0c\u5355\u4f4dM -1 4.3.0+ void setWorkTempDir(String workTempDir) \u53ef\u9009 \u8bbe\u7f6e\u4e34\u65f6\u76ee\u5f55 /tmp 4.3.0+ void setOutputCodec(String outputCodec) \u53ef\u9009 \u8bbe\u7f6e\u538b\u7f29\u7c7b\u578b\uff0c\u652f\u6301\u7f16\u89e3\u7801\u5668\u6709 gzip\u3001gz\u3001lzo\u3001lzop\u3001lzop\u3001snappy keep\uff08\u4e0d\u66f4\u6539\u538b\u7f29\u7c7b\u578b\uff09 4.3.0+ void setEnableCMS(boolean enableCMS) \u53ef\u9009 \u662f\u5426\u5f00\u542f\u76d1\u63a7\u544a\u8b66 false 4.3.0+ void setDisableChecksum(boolean disableChecksum) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5173\u95edChecksum\u68c0\u67e5 false 4.3.0+ void setDeleteOnSuccess(boolean deleteOnSuccess) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5220\u9664\u6e90\u6587\u4ef6\uff0c\u7528\u4e8e\u79fb\u52a8\u6570\u636e false 4.3.0+ void setIncludeMatcherFile(String includeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setExcludeMatcherFile(String excludeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setIgnoreFailures(boolean ignoreFailures) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5ffd\u7565\u62f7\u8d1d\u4efb\u52a1\u4e2d\u629b\u51fa\u7684\u5f02\u5e38\uff0c\u907f\u514d\u4e2d\u65ad\u4efb\u52a1 False 4.3.0+ void setStorageClass(String storageClass) \u53ef\u9009 \u8bbe\u7f6e\u76ee\u6807\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301Standard\u3001IA\u3001Archive\u3001ColdArchive Standard 4.3.0+ void setEnableTransaction(boolean enableTransaction) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4e8b\u52a1\uff0c\u4ee5\u4fdd\u8bc1Job\u7ea7\u522b\u7684\u539f\u5b50\u6027 false 4.3.0+ void setEnablePreserveMetaStatus(boolean enablePreserveMetaStatus) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f false 4.4.0+ \u54cd\u5e94\u53c2\u6570 \u00a7 \u65b9\u6cd5 \u8bf4\u660e \u9ed8\u8ba4\u503c int getErrorCode() \u83b7\u53d6\u9519\u8bef\u7801 0 String getErrorMsg() \u83b7\u53d6\u9519\u8bef\u539f\u56e0 - long getStartTime() \u83b7\u53d6\u4efb\u52a1\u7684\u5f00\u59cb\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - long getEndTime() \u83b7\u53d6\u4efb\u52a1\u7684\u7ed3\u675f\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - getCount(String key) \u83b7\u53d6\u8ba1\u6570\u5668\u503c\uff0ckey\u4e3a\u8ba1\u6570\u5668\u540d\u79f0\uff0c - \u8ba1\u6570\u5668\u8bf4\u660e \u00a7 \u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf BYTES_EXPECTED \u9884\u671f\u7684copy\u5b57\u8282\u6570 FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 BYTES_COPIED copy\u6210\u529f\u7684\u5b57\u8282\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"JindoDistCp API \u4f7f\u7528\u8bf4\u660e"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_api/#jindodistcp-api","text":"","title":"JindoDistCp API \u4f7f\u7528\u8bf4\u660e"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_api/#jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"\u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_api/#_1","text":"\u76ee\u524d JindoSDK/JindoDistCP \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u4ee54.6.12\u7248\u672c\u4e3a\u4f8b\uff0c\u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoDistCP \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-distcp --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop with provided scope, your runtime should install hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> </dependencies> </project>","title":"\u5b89\u88c5\u4f9d\u8d56\u5305"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_api/#_2","text":"\u4ee5\u62f7\u8d1d HDFS \u6587\u4ef6\u5939\u5230\u963f\u91cc\u4e91 OSS \u4e3a\u4f8b\u3002 import com.aliyun.jindodata.distcp.util.DistCpCounter; import com.aliyun.jindodata.distjob.request.DistJobResponse; import org.apache.hadoop.conf.Configuration; import java.io.IOException; public class DistcpExample { static public void main(String[] args) throws IOException { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. conf.set(\"fs.oss.credentials.provider\", \"com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider\"); conf.set(\"fs.oss.accessKeyId\", \"XXX\"); conf.set(\"fs.oss.accessKeySecret\", \"XXX\"); conf.set(\"fs.oss.endpoint\", \"oss-cn-xxx.aliyuncs.com\"); DistCpRequest request = new DistCpRequest(); request.setSrcDir(\"hdfs:///user/root/random-data\"); request.setDestDir(\"oss://oss-bucket/dest\"); request.setWorkTempDir(\"hdfs:///user/root/tmp\"); request.setConf(conf); // Jindo DistCp Service DistCpService service = new DistCpService(); service.buildAllJobs(request); service.runAllJobs(); DistJobResponse response = service.collectAllJobResult(); printResponse(response); } static public void printResponse(DistJobResponse response) { System.out.println(\"ErrorCode: \" + response.getErrorCode()); System.out.println(\"ErrorMsg: \" + response.getErrorMsg()); System.out.println(\"FileCopied: \" + response.getCount(DistCpCounter.FILES_COPIED.getName())); System.out.println(\"BytesCopied: \" + response.getCount(DistCpCounter.BYTES_COPIED.getName())); } }","title":"\u793a\u4f8b\u4ee3\u7801"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_api/#_3","text":"\u76f8\u5173\u65b9\u6cd5\u7684\u5177\u4f53\u63cf\u8ff0\u53ef\u53c2\u8003 \u300a\u4f7f\u7528Jindo DistCp\u300b \u65b9\u6cd5 \u53c2\u6570\u7c7b\u578b \u8bf4\u660e \u9ed8\u8ba4\u503c \u517c\u5bb9 void setSrc(String src) \u5fc5\u9009 \u8bbe\u7f6e\u6e90\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setDest(String dest) \u5fc5\u9009 \u8bbe\u7f6e\u76ee\u6807\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setConf(Configuration conf) \u5fc5\u9009 \u8bbe\u7f6e Configuration - 4.3.0+ void setMode(DistCpMode mode) \u53ef\u9009 \u8bbe\u7f6eDistCp\u6a21\u5f0f\uff0c\u652f\u6301\u7684\u6a21\u5f0f\u6709 COPY\uff08\u62f7\u8d1d\uff09 DIFF\uff08\u5dee\u5f02\u6bd4\u8f83\uff09 UPADTE\uff08\u589e\u91cf\u62f7\u8d1d\uff09 DistCpMode.COPY 4.3.0+ void setNumWorkers(int numWorkers) \u53ef\u9009 \u8bbe\u7f6eDistCp\u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\uff0c\u5bf9\u5e94MR\u4efb\u52a1\u4e2d\u7684 mapreduce.job.maps 10 4.3.0+ void setBandWidthLimit(int bandWidthLimit) \u53ef\u9009 \u8bbe\u7f6e\u5355\u4e2a\u8282\u70b9\u7684\u5e26\u5bbd\u9650\u5236\uff0c\u5355\u4f4dM -1 4.3.0+ void setWorkTempDir(String workTempDir) \u53ef\u9009 \u8bbe\u7f6e\u4e34\u65f6\u76ee\u5f55 /tmp 4.3.0+ void setOutputCodec(String outputCodec) \u53ef\u9009 \u8bbe\u7f6e\u538b\u7f29\u7c7b\u578b\uff0c\u652f\u6301\u7f16\u89e3\u7801\u5668\u6709 gzip\u3001gz\u3001lzo\u3001lzop\u3001lzop\u3001snappy keep\uff08\u4e0d\u66f4\u6539\u538b\u7f29\u7c7b\u578b\uff09 4.3.0+ void setEnableCMS(boolean enableCMS) \u53ef\u9009 \u662f\u5426\u5f00\u542f\u76d1\u63a7\u544a\u8b66 false 4.3.0+ void setDisableChecksum(boolean disableChecksum) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5173\u95edChecksum\u68c0\u67e5 false 4.3.0+ void setDeleteOnSuccess(boolean deleteOnSuccess) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5220\u9664\u6e90\u6587\u4ef6\uff0c\u7528\u4e8e\u79fb\u52a8\u6570\u636e false 4.3.0+ void setIncludeMatcherFile(String includeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setExcludeMatcherFile(String excludeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setIgnoreFailures(boolean ignoreFailures) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5ffd\u7565\u62f7\u8d1d\u4efb\u52a1\u4e2d\u629b\u51fa\u7684\u5f02\u5e38\uff0c\u907f\u514d\u4e2d\u65ad\u4efb\u52a1 False 4.3.0+ void setStorageClass(String storageClass) \u53ef\u9009 \u8bbe\u7f6e\u76ee\u6807\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301Standard\u3001IA\u3001Archive\u3001ColdArchive Standard 4.3.0+ void setEnableTransaction(boolean enableTransaction) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4e8b\u52a1\uff0c\u4ee5\u4fdd\u8bc1Job\u7ea7\u522b\u7684\u539f\u5b50\u6027 false 4.3.0+ void setEnablePreserveMetaStatus(boolean enablePreserveMetaStatus) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f false 4.4.0+","title":"\u8bf7\u6c42\u53c2\u6570"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_api/#_4","text":"\u65b9\u6cd5 \u8bf4\u660e \u9ed8\u8ba4\u503c int getErrorCode() \u83b7\u53d6\u9519\u8bef\u7801 0 String getErrorMsg() \u83b7\u53d6\u9519\u8bef\u539f\u56e0 - long getStartTime() \u83b7\u53d6\u4efb\u52a1\u7684\u5f00\u59cb\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - long getEndTime() \u83b7\u53d6\u4efb\u52a1\u7684\u7ed3\u675f\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - getCount(String key) \u83b7\u53d6\u8ba1\u6570\u5668\u503c\uff0ckey\u4e3a\u8ba1\u6570\u5668\u540d\u79f0\uff0c -","title":"\u54cd\u5e94\u53c2\u6570"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_api/#_5","text":"\u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf BYTES_EXPECTED \u9884\u671f\u7684copy\u5b57\u8282\u6570 FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 BYTES_COPIED copy\u6210\u529f\u7684\u5b57\u8282\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"\u8ba1\u6570\u5668\u8bf4\u660e"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/","text":"JindoDistCp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66 \u00a7 JindoDistCp\uff08\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u5de5\u5177\uff09\u662f\u7528\u4e8e\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u62f7\u8d1d\u6587\u4ef6\u7684\u5de5\u5177\u3002\u4f7f\u7528MapReduce\u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3amap/reduce\u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002 CMS\uff08\u4e91\u76d1\u63a7\u670d\u52a1\uff09\uff0c\u53ef\u7528\u4e8e\u6536\u96c6\u963f\u91cc\u4e91\u8d44\u6e90\u7684\u76d1\u63a7\u6307\u6807\u6216\u7528\u6237\u81ea\u5b9a\u4e49\u7684\u76d1\u63a7\u6307\u6807\uff0c\u63a2\u6d4b\u670d\u52a1\u53ef\u7528\u6027\uff0c\u4ee5\u53ca\u9488\u5bf9\u6307\u6807\u8bbe\u7f6e\u8b66\u62a5\u3002\u4f7f\u60a8\u5168\u9762\u4e86\u89e3\u963f\u91cc\u4e91\u4e0a\u8d44\u6e90\u7684\u4f7f\u7528\u60c5\u51b5\u548c\u4e1a\u52a1\u8fd0\u884c\u72b6\u51b5\uff0c\u5e76\u53ca\u65f6\u5bf9\u6545\u969c\u8d44\u6e90\u8fdb\u884c\u5904\u7406\uff0c\u4fdd\u8bc1\u4e1a\u52a1\u6b63\u5e38\u8fd0\u884c\u3002 \u60a8\u53ef\u4ee5\u6307\u5b9a\u672c\u6b21JindoDistCp\u4efb\u52a1\u7ed3\u675f\u540e\u662f\u5426\u9700\u8981\u5f00\u542fCMS\uff0c\u4ee5\u4e0a\u62a5\u4efb\u52a1\u5931\u8d25\u4fe1\u606f\uff0c\u5e76\u4f7f\u7528CMS\u63a7\u5236\u53f0\u6765\u914d\u7f6e\u544a\u8b66\u529f\u80fd\u3002\u544a\u8b66\u6761\u4ef6\u53c2\u89c1 \u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668 \u3002 \u6b65\u9aa4 \u00a7 1. \u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u548c\u62a5\u8b66\u8054\u7cfb\u7ec4 \u00a7 \u5177\u4f53\u8bf7\u53c2\u8003\u6587\u6863 \u300a\u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u6216\u62a5\u8b66\u8054\u7cfb\u7ec4\u300b 2. \u83b7\u53d6\u62a5\u8b66Token \u00a7 \u70b9\u51fb\u5de6\u4fa7\u5bfc\u822a\u680f\u201c\u62a5\u8b66\u670d\u52a1\u201d->\"\u62a5\u8b66\u8054\u7cfb\u4eba\"->\u5207\u6362\u4e0a\u6d4b\u9875\u7b7e\u5230\"\u62a5\u8b66\u8054\u7cfb\u7ec4\"->\u9009\u62e9\u5bf9\u5e94\u7684\u62a5\u8b66\u8054\u7cfb\u7ec4->\u70b9\u51fb\u53f3\u4fa7\u63a5\u5165\u5916\u90e8\u62a5\u8b66 3. \u914d\u7f6e\u73af\u5883\u53d8\u91cf \u00a7 \u73af\u5883\u53d8\u91cf \u8bf4\u660e cmsAccessKeyId \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94AccessKey ID cmsAccessSecret \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94Access Key Secret cmsRegion \u8bbe\u7f6eCMS\u6240\u5728Region\uff0c\u9700\u8981\u4e0e\u5176\u4ed6\u76f8\u5173\u7684\u4e91\u4e0a\u8d44\u6e90\u4e00\u81f4\uff0c\u4f8b\u5982ECS\uff0cOSS\uff0cEMR\u6240\u5bf9\u5e94\u7684Region cmsLevel \u8bbe\u7f6e\u6807\u8bc6\u544a\u8b66\u901a\u77e5\u65b9\u5f0f\uff0c\u5982\u679c\u672a\u8bbe\u7f6e\uff0c\u9ed8\u8ba4\u4e3aWARN\u3002INFO\u4e3a \u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cWARN\u4e3a \u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cCRITICAL\u4e3a \u7535\u8bdd+\u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \u793a\u4f8b\u5982\u4e0b export cmsAccessKeyId=<your_key_id> export cmsAccessSecret=<your_key_secret> export cmsRegion=cn-hangzhou export cmsToken=<your_cms_token> export cmsLevel=WARN 4. \u5f00\u542fCMS\u5bf9\u5931\u8d25\u4efb\u52a1\u8fdb\u884c\u544a\u8b66 \u00a7 copy\u64cd\u4f5c \u00a7 hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht/hourly_table \\ --enableCMS diff\u64cd\u4f5c \u00a7 hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht/hourly_table \\ --diff \\ --enableCMS update\u64cd\u4f5c \u00a7 hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht/hourly_table \\ --update \\ --enableCMS \u544a\u8b66\u5185\u5bb9 \u00a7 \u544a\u8b66\u5185\u5bb9\u683c\u5f0f \u00a7 { \"JobTime\": \"2021-03-09 19:45:27~2021-03-09 19:45:28\", \"SrcPath\": \"hdfs://cluster-xxx:9000//data/incoming/hourly_table\", \"DestPath\": \"oss://yang-hhht/hourly_table\", \"Counter\": { \"COUNTER_1\": XXX, \"COUNTER_2\": XXX, ... } } ... \u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668 \u00a7 \u544a\u8b66\u5185\u5bb9\u4e2d\u7684counter\u8ba1\u6570\u5668\u6709\u4ee5\u4e0b\u51e0\u79cd\uff1a copy\u64cd\u4f5c \u00a7 \u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 diff\u64cd\u4f5c \u00a7 \u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"JindoDistCp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#jindodistcpcms","text":"JindoDistCp\uff08\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u5de5\u5177\uff09\u662f\u7528\u4e8e\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u62f7\u8d1d\u6587\u4ef6\u7684\u5de5\u5177\u3002\u4f7f\u7528MapReduce\u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3amap/reduce\u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002 CMS\uff08\u4e91\u76d1\u63a7\u670d\u52a1\uff09\uff0c\u53ef\u7528\u4e8e\u6536\u96c6\u963f\u91cc\u4e91\u8d44\u6e90\u7684\u76d1\u63a7\u6307\u6807\u6216\u7528\u6237\u81ea\u5b9a\u4e49\u7684\u76d1\u63a7\u6307\u6807\uff0c\u63a2\u6d4b\u670d\u52a1\u53ef\u7528\u6027\uff0c\u4ee5\u53ca\u9488\u5bf9\u6307\u6807\u8bbe\u7f6e\u8b66\u62a5\u3002\u4f7f\u60a8\u5168\u9762\u4e86\u89e3\u963f\u91cc\u4e91\u4e0a\u8d44\u6e90\u7684\u4f7f\u7528\u60c5\u51b5\u548c\u4e1a\u52a1\u8fd0\u884c\u72b6\u51b5\uff0c\u5e76\u53ca\u65f6\u5bf9\u6545\u969c\u8d44\u6e90\u8fdb\u884c\u5904\u7406\uff0c\u4fdd\u8bc1\u4e1a\u52a1\u6b63\u5e38\u8fd0\u884c\u3002 \u60a8\u53ef\u4ee5\u6307\u5b9a\u672c\u6b21JindoDistCp\u4efb\u52a1\u7ed3\u675f\u540e\u662f\u5426\u9700\u8981\u5f00\u542fCMS\uff0c\u4ee5\u4e0a\u62a5\u4efb\u52a1\u5931\u8d25\u4fe1\u606f\uff0c\u5e76\u4f7f\u7528CMS\u63a7\u5236\u53f0\u6765\u914d\u7f6e\u544a\u8b66\u529f\u80fd\u3002\u544a\u8b66\u6761\u4ef6\u53c2\u89c1 \u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668 \u3002","title":"JindoDistCp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#1","text":"\u5177\u4f53\u8bf7\u53c2\u8003\u6587\u6863 \u300a\u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u6216\u62a5\u8b66\u8054\u7cfb\u7ec4\u300b","title":"1. \u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u548c\u62a5\u8b66\u8054\u7cfb\u7ec4"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#2-token","text":"\u70b9\u51fb\u5de6\u4fa7\u5bfc\u822a\u680f\u201c\u62a5\u8b66\u670d\u52a1\u201d->\"\u62a5\u8b66\u8054\u7cfb\u4eba\"->\u5207\u6362\u4e0a\u6d4b\u9875\u7b7e\u5230\"\u62a5\u8b66\u8054\u7cfb\u7ec4\"->\u9009\u62e9\u5bf9\u5e94\u7684\u62a5\u8b66\u8054\u7cfb\u7ec4->\u70b9\u51fb\u53f3\u4fa7\u63a5\u5165\u5916\u90e8\u62a5\u8b66","title":"2. \u83b7\u53d6\u62a5\u8b66Token"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#3","text":"\u73af\u5883\u53d8\u91cf \u8bf4\u660e cmsAccessKeyId \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94AccessKey ID cmsAccessSecret \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94Access Key Secret cmsRegion \u8bbe\u7f6eCMS\u6240\u5728Region\uff0c\u9700\u8981\u4e0e\u5176\u4ed6\u76f8\u5173\u7684\u4e91\u4e0a\u8d44\u6e90\u4e00\u81f4\uff0c\u4f8b\u5982ECS\uff0cOSS\uff0cEMR\u6240\u5bf9\u5e94\u7684Region cmsLevel \u8bbe\u7f6e\u6807\u8bc6\u544a\u8b66\u901a\u77e5\u65b9\u5f0f\uff0c\u5982\u679c\u672a\u8bbe\u7f6e\uff0c\u9ed8\u8ba4\u4e3aWARN\u3002INFO\u4e3a \u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cWARN\u4e3a \u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cCRITICAL\u4e3a \u7535\u8bdd+\u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \u793a\u4f8b\u5982\u4e0b export cmsAccessKeyId=<your_key_id> export cmsAccessSecret=<your_key_secret> export cmsRegion=cn-hangzhou export cmsToken=<your_cms_token> export cmsLevel=WARN","title":"3. \u914d\u7f6e\u73af\u5883\u53d8\u91cf"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#4-cms","text":"","title":"4. \u5f00\u542fCMS\u5bf9\u5931\u8d25\u4efb\u52a1\u8fdb\u884c\u544a\u8b66"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#copy","text":"hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht/hourly_table \\ --enableCMS","title":"copy\u64cd\u4f5c"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#diff","text":"hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht/hourly_table \\ --diff \\ --enableCMS","title":"diff\u64cd\u4f5c"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#update","text":"hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht/hourly_table \\ --update \\ --enableCMS","title":"update\u64cd\u4f5c"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#_2","text":"","title":"\u544a\u8b66\u5185\u5bb9"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#_3","text":"{ \"JobTime\": \"2021-03-09 19:45:27~2021-03-09 19:45:28\", \"SrcPath\": \"hdfs://cluster-xxx:9000//data/incoming/hourly_table\", \"DestPath\": \"oss://yang-hhht/hourly_table\", \"Counter\": { \"COUNTER_1\": XXX, \"COUNTER_2\": XXX, ... } } ...","title":"\u544a\u8b66\u5185\u5bb9\u683c\u5f0f"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#_4","text":"\u544a\u8b66\u5185\u5bb9\u4e2d\u7684counter\u8ba1\u6570\u5668\u6709\u4ee5\u4e0b\u51e0\u79cd\uff1a","title":"\u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#copy_1","text":"\u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570","title":"copy\u64cd\u4f5c"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#diff_1","text":"\u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"diff\u64cd\u4f5c"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/","text":"\u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS \u4e0a \u00a7 \u4f7f\u7528\u524d\u987b\u77e5 \u00a7 \u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\uff0c\u4e5f\u53ef \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988 1\u3001\u62f7\u8d1d\u6570\u636e\u5230 OSS \u4e0a \u00a7 \u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06 HDFS \u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS \u4e0a hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --parallelism 10 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src HDFS \u7684\u6e90\u8def\u5f84\u3002 /data --dest OSS \u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket/ --hadoopConf \u6307\u5b9a OSS Access Key ID , Access Key Secret , Endpoint * \u914d\u7f6e OSS \u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS \u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret * \u914d\u7f6e OSS \u7684 Endpoint \u4fe1\u606f: fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10 2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6 \u00a7 \u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53eaCopy\u5269\u4e0b\u672aCopy\u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a \u4f7f\u7528 --update \u547d\u4ee4 \u00a7 \u4f7f\u7528 --update \u547d\u4ee4\u65f6\uff0c\u9ed8\u8ba4\u5f00\u542f checksum \u6bd4\u8f83\uff0c\u4e5f\u53ef\u901a\u8fc7 --disableChecksum \u5173\u95ed\u3002 \u5f00\u542f\u65f6\uff0c\u6bd4\u8f83\u7684\u65b9\u5f0f\u662f\uff0c\u4ece hdfs \u4e2d\u83b7\u53d6\u7684 checksum\uff0c\u5224\u65ad\u4e0e\u4e0a\u6b21\u62f7\u8d1d\u65f6\u8bb0\u5f55\u5728 OSS \u4e2d\u7684 checksum \u662f\u5426\u76f8\u540c\u3002\u56e0\u6b64\u4ec5\u652f\u6301\u6bd4\u8f83\u901a\u8fc7 3.4.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u62f7\u8d1d\u5f97\u5230\u7684\u6587\u4ef6\uff0c\u5982\u5e0c\u671b\u589e\u91cf\u6bd4\u8f83\u8001\u7248\u672c\u62f7\u8d1d\u5f97\u5230\u7684\u6587\u4ef6\uff0c\u63a8\u8350\u5173\u95ed checksum \u6bd4\u8f83\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --update --parallelism 20 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) \u5173\u95ed\u65f6\uff0c\u4ec5\u5bf9\u6587\u4ef6\u540d\u548c\u6587\u4ef6\u5927\u5c0f\u505a\u6bd4\u8f83\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --update --disableChecksum --parallelism 20 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) --disableChecksum\uff1a\u8df3\u8fc7\u4f20\u8f93\u65f6\u5bf9 checksum \u7684\u8ba1\u7b97\u548c\u68c0\u67e5 3\u3001\u6587\u4ef6\u51b7\u5907\u4efd \u00a7 \u5982\u60a8\u60f3\u5bf9\u5199\u5165\u5230 OSS \u4e0a\u7684\u6587\u4ef6\u8fdb\u884c\u51b7\u5907\uff0c\u5982\u8f6c\u5316\u6210\u51b7\u5f52\u6863\uff08coldArchive\uff09\u3001\u5f52\u6863\uff08archive\uff09\u548c\u4f4e\u9891\uff08ia\uff09\u6587\u4ef6\uff0c\u53ef\u5229\u7528 JindoDistCp \u76f4\u63a5\u8fdb\u884c\u8be5\u6d41\u7a0b \u5199\u5165\u51b7\u5f52\u6863\u6587\u4ef6 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy coldArchive --parallelism 10 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) --policy coldArchive\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u51b7\u5f52\u6863\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e\uff0c\u51b7\u5f52\u6863\u76ee\u524d\u53ea\u5728\u90e8\u5206region\u53ef\u7528\uff0c\u5177\u4f53\u53c2\u89c1 OSS\u5b58\u50a8\u7c7b\u578b\u4ecb\u7ecd \u5199\u5165\u5f52\u6863\u6587\u4ef6 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy archive --parallelism 10 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) --policy archive\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u5f52\u6863\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e \u5199\u5165\u4f4e\u9891\u6587\u4ef6 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy ia --parallelism 10 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) --policy ia\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u4f4e\u9891\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e 4\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9 \u00a7 \u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB 5\u3001\u514d\u5bc6\u53ca\u5bc6\u94a5\u56fa\u5b9a\u5b58\u50a8 \u00a7 \u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS AccessKey/AccessSecret/EndPoint \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662fJindoDistCp\u53ef\u4ee5\u5c06\u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> 6\u3001\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f \u00a7 \u4f7f\u7528preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --preserveMeta 7\u3001\u5176\u4ed6\u529f\u80fd \u00a7 \u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 * JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e * JindoDistCp \u4f7f\u7528 CMS \u8fdb\u884c\u544a\u8b66","title":"\u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS \u4e0a"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#hdfs-oss","text":"","title":"\u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS \u4e0a"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#_1","text":"\u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\uff0c\u4e5f\u53ef \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988","title":"\u4f7f\u7528\u524d\u987b\u77e5"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#1-oss","text":"\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06 HDFS \u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS \u4e0a hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --parallelism 10 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src HDFS \u7684\u6e90\u8def\u5f84\u3002 /data --dest OSS \u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket/ --hadoopConf \u6307\u5b9a OSS Access Key ID , Access Key Secret , Endpoint * \u914d\u7f6e OSS \u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS \u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret * \u914d\u7f6e OSS \u7684 Endpoint \u4fe1\u606f: fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10","title":"1\u3001\u62f7\u8d1d\u6570\u636e\u5230 OSS \u4e0a"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#2","text":"\u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53eaCopy\u5269\u4e0b\u672aCopy\u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a","title":"2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#-update","text":"\u4f7f\u7528 --update \u547d\u4ee4\u65f6\uff0c\u9ed8\u8ba4\u5f00\u542f checksum \u6bd4\u8f83\uff0c\u4e5f\u53ef\u901a\u8fc7 --disableChecksum \u5173\u95ed\u3002 \u5f00\u542f\u65f6\uff0c\u6bd4\u8f83\u7684\u65b9\u5f0f\u662f\uff0c\u4ece hdfs \u4e2d\u83b7\u53d6\u7684 checksum\uff0c\u5224\u65ad\u4e0e\u4e0a\u6b21\u62f7\u8d1d\u65f6\u8bb0\u5f55\u5728 OSS \u4e2d\u7684 checksum \u662f\u5426\u76f8\u540c\u3002\u56e0\u6b64\u4ec5\u652f\u6301\u6bd4\u8f83\u901a\u8fc7 3.4.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u62f7\u8d1d\u5f97\u5230\u7684\u6587\u4ef6\uff0c\u5982\u5e0c\u671b\u589e\u91cf\u6bd4\u8f83\u8001\u7248\u672c\u62f7\u8d1d\u5f97\u5230\u7684\u6587\u4ef6\uff0c\u63a8\u8350\u5173\u95ed checksum \u6bd4\u8f83\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --update --parallelism 20 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) \u5173\u95ed\u65f6\uff0c\u4ec5\u5bf9\u6587\u4ef6\u540d\u548c\u6587\u4ef6\u5927\u5c0f\u505a\u6bd4\u8f83\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --update --disableChecksum --parallelism 20 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) --disableChecksum\uff1a\u8df3\u8fc7\u4f20\u8f93\u65f6\u5bf9 checksum \u7684\u8ba1\u7b97\u548c\u68c0\u67e5","title":"\u4f7f\u7528 --update \u547d\u4ee4"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#3","text":"\u5982\u60a8\u60f3\u5bf9\u5199\u5165\u5230 OSS \u4e0a\u7684\u6587\u4ef6\u8fdb\u884c\u51b7\u5907\uff0c\u5982\u8f6c\u5316\u6210\u51b7\u5f52\u6863\uff08coldArchive\uff09\u3001\u5f52\u6863\uff08archive\uff09\u548c\u4f4e\u9891\uff08ia\uff09\u6587\u4ef6\uff0c\u53ef\u5229\u7528 JindoDistCp \u76f4\u63a5\u8fdb\u884c\u8be5\u6d41\u7a0b","title":"3\u3001\u6587\u4ef6\u51b7\u5907\u4efd"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#_2","text":"hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy coldArchive --parallelism 10 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) --policy coldArchive\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u51b7\u5f52\u6863\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e\uff0c\u51b7\u5f52\u6863\u76ee\u524d\u53ea\u5728\u90e8\u5206region\u53ef\u7528\uff0c\u5177\u4f53\u53c2\u89c1 OSS\u5b58\u50a8\u7c7b\u578b\u4ecb\u7ecd","title":"\u5199\u5165\u51b7\u5f52\u6863\u6587\u4ef6"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#_3","text":"hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy archive --parallelism 10 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) --policy archive\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u5f52\u6863\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e","title":"\u5199\u5165\u5f52\u6863\u6587\u4ef6"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#_4","text":"hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy ia --parallelism 10 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) --policy ia\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u4f4e\u9891\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e","title":"\u5199\u5165\u4f4e\u9891\u6587\u4ef6"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#4yarn","text":"\u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB","title":"4\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#5","text":"\u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS AccessKey/AccessSecret/EndPoint \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662fJindoDistCp\u53ef\u4ee5\u5c06\u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration>","title":"5\u3001\u514d\u5bc6\u53ca\u5bc6\u94a5\u56fa\u5b9a\u5b58\u50a8"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#6","text":"\u4f7f\u7528preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --preserveMeta","title":"6\u3001\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#7","text":"\u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 * JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e * JindoDistCp \u4f7f\u7528 CMS \u8fdb\u884c\u544a\u8b66","title":"7\u3001\u5176\u4ed6\u529f\u80fd"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/","text":"\u6570\u636e\u5728 OSS \u4e0d\u540c bucket \u4e4b\u95f4\u8fc1\u79fb \u00a7 \u4f7f\u7528\u524d\u987b\u77e5 \u00a7 \u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\uff0c\u4e5f\u53ef \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988 1\u3001\u62f7\u8d1d\u6570\u636e\u5230 OSS \u4e0a \u00a7 \u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06 OSS \u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS \u4e0a\u3002 \u5982\u679c\u60a8\u7684\u4e24\u4e2a\u8981\u62f7\u8d1d\u7684 bucket \u90fd\u5728\u540c\u4e00\u4e2a region \u4e0b\uff0c\u4e14\u53ef\u7528\u540c\u4e00\u4e2a ossKey \u548c ossSecret \u8fdb\u884c\u8bfb\u5199\uff0c\u60a8\u53ef\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src OSS \u7684\u6e90\u8def\u5f84\u3002 oss://srcBucket/ --dest OSS \u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket/ --hadoopConf \u6307\u5b9a OSS Access Key ID , Access Key Secret , Endpoint * \u914d\u7f6e OSS \u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS \u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret * \u914d\u7f6e OSS \u7684 Endpoint \u4fe1\u606f: fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10 \u5982\u679c\u60a8\u7684 src \u548c dest \u7684 bucket \u4e0d\u5728\u540c\u4e00\u4e2a region \u6216\u8005\u4e0d\u80fd\u4f7f\u7528\u540c\u4e00\u4e2a ossKey \u548c ossSecret \u8fdb\u884c\u8bfb\u5199\uff0c\u90a3\u4e48\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcbucket.oss-cn-xxx.aliyuncs.com/ --dest oss://destBucket.oss-cn-xxx.aliyuncs.com/ --hadoopConf fs.oss.bucket.srcbucket.accessKeyId=yourkey --hadoopConf fs.oss.bucket.srcbucket.accessKeySecret=yoursecret --hadoopConf fs.oss.bucket.srcbucket.endpoint=oss-cn-xxx.aliyuncs.com --hadoopConf fs.oss.bucket.destBucket.accessKeyId=yourkey --hadoopConf fs.oss.bucket.destBucket.accessKeySecret=yoursecret --hadoopConf fs.oss.bucket.destBucket.endpoint=oss-cn-xxx.aliyuncs.com --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --hadoopConf \u7ed9 src \u548c dest \u7684 bucket \u6307\u5b9a\u4e0d\u540c\u7684 Access Key ID , Access Key Secret , Endpoint * \u914d\u7f6e XXX bucket \u7684 AccessKeyId: --hadoopConf fs.oss.bucket.XXX.accessKeyId=yourkey * \u914d\u7f6e XXX bucket \u7684 \u7684 AccessKeySecret: --hadoopConf fs.oss.bucket.XXX.accessKeySecret=yoursecret * \u914d\u7f6e XXX bucket \u7684 Endpoint \u4fe1\u606f: fs.oss.bucket.XXX.endpoint=oss-cn-xxx.aliyuncs.com 2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6 \u00a7 \u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53eaCopy\u5269\u4e0b\u672aCopy\u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a \u4f7f\u7528 --update \u547d\u4ee4\uff0c\u83b7\u5f97\u589e\u91cf\u7684\u6587\u4ef6\u5217\u8868 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --update --parallelism 20 3\u3001\u6587\u4ef6\u51b7\u5907\u4efd \u00a7 \u5982\u60a8\u60f3\u5bf9\u5199\u5165\u5230 OSS \u4e0a\u7684\u6587\u4ef6\u8fdb\u884c\u51b7\u5907\uff0c\u5982\u8f6c\u5316\u6210\u51b7\u5f52\u6863\uff08coldArchive\uff09\u3001\u5f52\u6863\uff08archive\uff09\u548c\u4f4e\u9891\uff08ia\uff09\u6587\u4ef6\uff0c\u53ef\u5229\u7528 JindoDistCp \u76f4\u63a5\u8fdb\u884c\u8be5\u6d41\u7a0b \u5199\u5165\u51b7\u5f52\u6863\u6587\u4ef6 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy coldArchive --parallelism 10 --policy coldArchive\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u51b7\u5f52\u6863\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e\uff0c\u51b7\u5f52\u6863\u76ee\u524d\u53ea\u5728\u90e8\u5206region\u53ef\u7528\uff0c\u5177\u4f53\u53c2\u89c1 OSS\u5b58\u50a8\u7c7b\u578b\u4ecb\u7ecd \u5199\u5165\u5f52\u6863\u6587\u4ef6 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy archive --parallelism 10 --policy archive\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u5f52\u6863\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e \u5199\u5165\u4f4e\u9891\u6587\u4ef6 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy ia --parallelism 10 --policy ia\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u4f4e\u9891\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e 4\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9 \u00a7 \u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB 5\u3001\u514d\u5bc6\u53ca\u5bc6\u94a5\u56fa\u5b9a\u5b58\u50a8 \u00a7 \u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS AccessKey/AccessSecret/EndPoint \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662fJindoDistCp\u53ef\u4ee5\u5c06 \u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> 6\u3001\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f \u00a7 \u4f7f\u7528preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --preserveMeta 7\u3001\u5176\u4ed6\u529f\u80fd \u00a7 \u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 * JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e","title":"\u6570\u636e\u5728 OSS \u4e0d\u540c bucket \u4e4b\u95f4\u8fc1\u79fb"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#oss-bucket","text":"","title":"\u6570\u636e\u5728 OSS \u4e0d\u540c bucket \u4e4b\u95f4\u8fc1\u79fb"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#_1","text":"\u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\uff0c\u4e5f\u53ef \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988","title":"\u4f7f\u7528\u524d\u987b\u77e5"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#1-oss","text":"\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06 OSS \u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS \u4e0a\u3002 \u5982\u679c\u60a8\u7684\u4e24\u4e2a\u8981\u62f7\u8d1d\u7684 bucket \u90fd\u5728\u540c\u4e00\u4e2a region \u4e0b\uff0c\u4e14\u53ef\u7528\u540c\u4e00\u4e2a ossKey \u548c ossSecret \u8fdb\u884c\u8bfb\u5199\uff0c\u60a8\u53ef\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src OSS \u7684\u6e90\u8def\u5f84\u3002 oss://srcBucket/ --dest OSS \u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket/ --hadoopConf \u6307\u5b9a OSS Access Key ID , Access Key Secret , Endpoint * \u914d\u7f6e OSS \u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS \u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret * \u914d\u7f6e OSS \u7684 Endpoint \u4fe1\u606f: fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10 \u5982\u679c\u60a8\u7684 src \u548c dest \u7684 bucket \u4e0d\u5728\u540c\u4e00\u4e2a region \u6216\u8005\u4e0d\u80fd\u4f7f\u7528\u540c\u4e00\u4e2a ossKey \u548c ossSecret \u8fdb\u884c\u8bfb\u5199\uff0c\u90a3\u4e48\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcbucket.oss-cn-xxx.aliyuncs.com/ --dest oss://destBucket.oss-cn-xxx.aliyuncs.com/ --hadoopConf fs.oss.bucket.srcbucket.accessKeyId=yourkey --hadoopConf fs.oss.bucket.srcbucket.accessKeySecret=yoursecret --hadoopConf fs.oss.bucket.srcbucket.endpoint=oss-cn-xxx.aliyuncs.com --hadoopConf fs.oss.bucket.destBucket.accessKeyId=yourkey --hadoopConf fs.oss.bucket.destBucket.accessKeySecret=yoursecret --hadoopConf fs.oss.bucket.destBucket.endpoint=oss-cn-xxx.aliyuncs.com --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --hadoopConf \u7ed9 src \u548c dest \u7684 bucket \u6307\u5b9a\u4e0d\u540c\u7684 Access Key ID , Access Key Secret , Endpoint * \u914d\u7f6e XXX bucket \u7684 AccessKeyId: --hadoopConf fs.oss.bucket.XXX.accessKeyId=yourkey * \u914d\u7f6e XXX bucket \u7684 \u7684 AccessKeySecret: --hadoopConf fs.oss.bucket.XXX.accessKeySecret=yoursecret * \u914d\u7f6e XXX bucket \u7684 Endpoint \u4fe1\u606f: fs.oss.bucket.XXX.endpoint=oss-cn-xxx.aliyuncs.com","title":"1\u3001\u62f7\u8d1d\u6570\u636e\u5230 OSS \u4e0a"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#2","text":"\u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53eaCopy\u5269\u4e0b\u672aCopy\u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a","title":"2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#-update","text":"hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --update --parallelism 20","title":"\u4f7f\u7528 --update \u547d\u4ee4\uff0c\u83b7\u5f97\u589e\u91cf\u7684\u6587\u4ef6\u5217\u8868"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#3","text":"\u5982\u60a8\u60f3\u5bf9\u5199\u5165\u5230 OSS \u4e0a\u7684\u6587\u4ef6\u8fdb\u884c\u51b7\u5907\uff0c\u5982\u8f6c\u5316\u6210\u51b7\u5f52\u6863\uff08coldArchive\uff09\u3001\u5f52\u6863\uff08archive\uff09\u548c\u4f4e\u9891\uff08ia\uff09\u6587\u4ef6\uff0c\u53ef\u5229\u7528 JindoDistCp \u76f4\u63a5\u8fdb\u884c\u8be5\u6d41\u7a0b","title":"3\u3001\u6587\u4ef6\u51b7\u5907\u4efd"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#_2","text":"hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy coldArchive --parallelism 10 --policy coldArchive\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u51b7\u5f52\u6863\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e\uff0c\u51b7\u5f52\u6863\u76ee\u524d\u53ea\u5728\u90e8\u5206region\u53ef\u7528\uff0c\u5177\u4f53\u53c2\u89c1 OSS\u5b58\u50a8\u7c7b\u578b\u4ecb\u7ecd","title":"\u5199\u5165\u51b7\u5f52\u6863\u6587\u4ef6"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#_3","text":"hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy archive --parallelism 10 --policy archive\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u5f52\u6863\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e","title":"\u5199\u5165\u5f52\u6863\u6587\u4ef6"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#_4","text":"hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy ia --parallelism 10 --policy ia\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u4f4e\u9891\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e","title":"\u5199\u5165\u4f4e\u9891\u6587\u4ef6"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#4yarn","text":"\u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB","title":"4\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#5","text":"\u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS AccessKey/AccessSecret/EndPoint \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662fJindoDistCp\u53ef\u4ee5\u5c06 \u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration>","title":"5\u3001\u514d\u5bc6\u53ca\u5bc6\u94a5\u56fa\u5b9a\u5b58\u50a8"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#6","text":"\u4f7f\u7528preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --preserveMeta","title":"6\u3001\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#7","text":"\u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 * JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e","title":"7\u3001\u5176\u4ed6\u529f\u80fd"},{"location":"backup/oss/jindo_distcp/jindo_distcp_overview/","text":"JindoDistCp \u6570\u636e\u8fc1\u79fb \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 JDK 1.8\u53ca\u4ee5\u4e0a Hadoop 2.3+\u7248\u672c\uff0c\u8bf7\u4e0b\u8f7d\u6700\u65b0\u7248\u7684 jindo-distcp-tool-x.x.x.jar \uff0c\u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz \u5185\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 tools/ \u76ee\u5f55\u4e0b\u627e\u5230\uff0c \u4e0b\u8f7d\u9875\u9762 \u3002 \u4ec0\u4e48\u662f JindoDistCp \u00a7 \u968f\u7740\u963f\u91cc\u4e91 JindoSDK \u7684\u5168\u9762\u653e\u5f00\u4f7f\u7528\uff0c\u57fa\u4e8e JindoSDK \u7684\u963f\u91cc\u4e91\u6570\u636e\u8fc1\u79fb\u5229\u5668 JindoDistCp \u73b0\u5728\u4e5f\u5168\u9762\u9762\u5411\u7528\u6237\u5f00\u653e\u4f7f\u7528\u3002JindoDistCp \u662f\u963f\u91cc\u4e91\u6570\u636e\u6e56\u5b58\u50a8\u56e2\u961f\u5f00\u53d1\u7684\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u7684\u5de5\u5177\u3002\u5176\u4f7f\u7528 MapReduce \u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3a map/reduce \u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002\u76ee\u524d\u5168\u91cf\u652f\u6301 HDFS/OSS/S3 \u4e4b\u95f4\u7684\u7684\u6570\u636e\u62f7\u8d1d\u573a\u666f\uff0c\u63d0\u4f9b\u591a\u79cd\u4e2a\u6027\u5316\u62f7\u8d1d\u53c2\u6570\u548c\u591a\u79cd\u62f7\u8d1d\u7b56\u7565\u3002\u91cd\u70b9\u4f18\u5316 HDFS \u5230 OSS \u7684\u6570\u636e\u62f7\u8d1d\uff0c\u901a\u8fc7\u5b9a\u5236\u5316 CopyCommitter\uff0c\u5b9e\u73b0 No-Rename \u62f7\u8d1d\uff0c\u5e76\u4fdd\u8bc1\u6570\u636e\u62f7\u8d1d\u843d\u5730\u7684\u4e00\u81f4\u6027\u3002\u529f\u80fd\u5168\u91cf\u5bf9\u9f50 S3 DistCp \u548c HDFS DistCp\uff0c\u6027\u80fd\u8f83 HDFS DistCp \u6709\u8f83\u5927\u63d0\u5347\uff0c\u76ee\u6807\u63d0\u4f9b\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177\u3002 JindoDistCp \u4f7f\u7528\u573a\u666f \u00a7 \u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS \u4e0a \u6570\u636e\u5728 OSS \u4e0d\u540c bucket \u4e4b\u95f4\u8fc1\u79fb JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u00a7 \u6211\u4eec\u603b\u7ed3\u4e86\u4e00\u4e9b\u5728\u4e0d\u540c\u73af\u5883\u4f7f\u7528 JindoDistCp \u7528\u6237\u53ef\u80fd\u4f1a\u9047\u5230\u7684\u4e00\u4e9b\u95ee\u9898\uff0c\u60a8\u53ef\u4ee5\u53c2\u7167\u4e0b\u6587\u8fdb\u884c\u5e38\u89c1\u95ee\u9898\u6392\u67e5\u548c\u89e3\u51b3 * JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 JindoDistCp \u7cfb\u5217\u6587\u7ae0 \u00a7 \u300a\u518d\u51fa\u738b\u724c\uff1a\u963f\u91cc\u4e91 JindoDistCp \u5168\u9762\u5f00\u653e\u4f7f\u7528\uff0c\u6210\u4e3a\u963f\u91cc\u4e91\u6570\u636e\u8fc1\u79fb\u5229\u5668\u300b JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e (\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177) JindoDistCp API \u4f7f\u7528\u8bf4\u660e \u00a7 * JindoDistCp API \u4f7f\u7528\u8bf4\u660e","title":"JindoDistCp \u6570\u636e\u8fc1\u79fb"},{"location":"backup/oss/jindo_distcp/jindo_distcp_overview/#jindodistcp","text":"","title":"JindoDistCp \u6570\u636e\u8fc1\u79fb"},{"location":"backup/oss/jindo_distcp/jindo_distcp_overview/#_1","text":"JDK 1.8\u53ca\u4ee5\u4e0a Hadoop 2.3+\u7248\u672c\uff0c\u8bf7\u4e0b\u8f7d\u6700\u65b0\u7248\u7684 jindo-distcp-tool-x.x.x.jar \uff0c\u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz \u5185\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 tools/ \u76ee\u5f55\u4e0b\u627e\u5230\uff0c \u4e0b\u8f7d\u9875\u9762 \u3002","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/oss/jindo_distcp/jindo_distcp_overview/#jindodistcp_1","text":"\u968f\u7740\u963f\u91cc\u4e91 JindoSDK \u7684\u5168\u9762\u653e\u5f00\u4f7f\u7528\uff0c\u57fa\u4e8e JindoSDK \u7684\u963f\u91cc\u4e91\u6570\u636e\u8fc1\u79fb\u5229\u5668 JindoDistCp \u73b0\u5728\u4e5f\u5168\u9762\u9762\u5411\u7528\u6237\u5f00\u653e\u4f7f\u7528\u3002JindoDistCp \u662f\u963f\u91cc\u4e91\u6570\u636e\u6e56\u5b58\u50a8\u56e2\u961f\u5f00\u53d1\u7684\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u7684\u5de5\u5177\u3002\u5176\u4f7f\u7528 MapReduce \u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3a map/reduce \u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002\u76ee\u524d\u5168\u91cf\u652f\u6301 HDFS/OSS/S3 \u4e4b\u95f4\u7684\u7684\u6570\u636e\u62f7\u8d1d\u573a\u666f\uff0c\u63d0\u4f9b\u591a\u79cd\u4e2a\u6027\u5316\u62f7\u8d1d\u53c2\u6570\u548c\u591a\u79cd\u62f7\u8d1d\u7b56\u7565\u3002\u91cd\u70b9\u4f18\u5316 HDFS \u5230 OSS \u7684\u6570\u636e\u62f7\u8d1d\uff0c\u901a\u8fc7\u5b9a\u5236\u5316 CopyCommitter\uff0c\u5b9e\u73b0 No-Rename \u62f7\u8d1d\uff0c\u5e76\u4fdd\u8bc1\u6570\u636e\u62f7\u8d1d\u843d\u5730\u7684\u4e00\u81f4\u6027\u3002\u529f\u80fd\u5168\u91cf\u5bf9\u9f50 S3 DistCp \u548c HDFS DistCp\uff0c\u6027\u80fd\u8f83 HDFS DistCp \u6709\u8f83\u5927\u63d0\u5347\uff0c\u76ee\u6807\u63d0\u4f9b\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177\u3002","title":"\u4ec0\u4e48\u662f JindoDistCp"},{"location":"backup/oss/jindo_distcp/jindo_distcp_overview/#jindodistcp_2","text":"\u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS \u4e0a \u6570\u636e\u5728 OSS \u4e0d\u540c bucket \u4e4b\u95f4\u8fc1\u79fb","title":"JindoDistCp \u4f7f\u7528\u573a\u666f"},{"location":"backup/oss/jindo_distcp/jindo_distcp_overview/#jindodistcp_3","text":"\u6211\u4eec\u603b\u7ed3\u4e86\u4e00\u4e9b\u5728\u4e0d\u540c\u73af\u5883\u4f7f\u7528 JindoDistCp \u7528\u6237\u53ef\u80fd\u4f1a\u9047\u5230\u7684\u4e00\u4e9b\u95ee\u9898\uff0c\u60a8\u53ef\u4ee5\u53c2\u7167\u4e0b\u6587\u8fdb\u884c\u5e38\u89c1\u95ee\u9898\u6392\u67e5\u548c\u89e3\u51b3 * JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357","title":"JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357"},{"location":"backup/oss/jindo_distcp/jindo_distcp_overview/#jindodistcp_4","text":"\u300a\u518d\u51fa\u738b\u724c\uff1a\u963f\u91cc\u4e91 JindoDistCp \u5168\u9762\u5f00\u653e\u4f7f\u7528\uff0c\u6210\u4e3a\u963f\u91cc\u4e91\u6570\u636e\u8fc1\u79fb\u5229\u5668\u300b JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e (\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177)","title":"JindoDistCp \u7cfb\u5217\u6587\u7ae0"},{"location":"backup/oss/jindo_distcp/jindo_distcp_overview/#jindodistcp-api","text":"* JindoDistCp API \u4f7f\u7528\u8bf4\u660e","title":"JindoDistCp API \u4f7f\u7528\u8bf4\u660e"},{"location":"backup/oss/jindo_fuse/jindo_fuse_benchmark/","text":"Jindo Fuse \u6027\u80fd\u6d4b\u8bd5 \u00a7 \u6d4b\u8bd5\u73af\u5883 \u00a7 CPU: 80\u6838 | \u5185\u5b58\uff1a 128G \u7f51\u7edc\u89c4\u683c\uff1a25Gbps ECS\u89c4\u683c\uff1a ecs.hfc6.20xlarge \u6d4b\u8bd5\u6570\u636e\u96c6 \u00a7 \u6570\u636e\u96c6A: \u5927\u6587\u4ef6\u96c6\u5408\uff0c\u6bcf\u4e2a\u6587\u4ef6140MB\uff0c\u603b\u5171140GB \u6570\u636e\u96c6B: \u5c0f\u6587\u4ef6\u96c6\u5408\uff0c\u6bcf\u4e2a\u6587\u4ef623KB\uff0c\u603b\u51711w\u4e2a \u6d4b\u8bd5\u7ed3\u679c \u00a7 \u5355\u7ebf\u7a0b\u6d4b\u8bd5 \u6570\u636e\u96c6 jindo-fuse ossfs OSS2/Python \u6570\u636e\u96c6A 191 MB/s 72 MB/s 17 MB/s \u6570\u636e\u96c6B 1.16 MB/s 1.25 MB/s 2.1 MB/s \u591a\u7ebf\u7a0b\u6d4b\u8bd5\uff0810\u4e2a\u7ebf\u7a0b\uff09 \u6570\u636e\u96c6 jindo-fuse ossfs OSS2/Python \u6570\u636e\u96c6A 1693MB/s 686 MB/s 162 MB/s \u6570\u636e\u96c6B 18 MB/s 10 MB/s 9 MB/s \u7ed3\u679c\u5206\u6790 \u00a7 jindo-fuse \u5bf9\u4e8e\u5927\u6587\u4ef6 \u6709\u9884\u8bfb\uff0c\u56e0\u6b64\u6bd4\u8f83\u5feb\u3002 OSS2 \u5bf9\u5c0f\u6587\u4ef6\u6bd4\u8f83\u5feb\uff0c\u5c0f\u6587\u4ef6 fuse \u65e0\u6cd5\u505a\u9884\u8bfb\uff0cOSS2 \u76f4\u63a5 http \u8bf7\u6c42\u6ca1\u6709\u989d\u5916\u5f00\u9500\uff0c\u6240\u4ee5\u6bd4\u8f83\u5feb\u3002 \u5c0f\u6587\u4ef6\u7ecf\u8fc7 fuse \u6709\u989d\u5916\u5f00\u9500\uff0c \u56e0\u6b64 jindo-fuse\u3001ossfs \u901f\u5ea6\u6bd4 OSS2 \u5dee\u3002 OSS2 \u5bf9\u5355\u8fdb\u7a0b\u591a\u7ebf\u7a0b\u6548\u679c\u4e0d\u597d\u3002 \u6211\u4eec\u53ef\u4ee5\u63a8\u6d4b OSS2 \u591a\u8fdb\u7a0b\u6210\u7ee9 2.1MB/s *10 = 21MB/s\u3002 ossfs \u5bf9\u5c0f\u6587\u4ef6\u56e0\u4e3a\u6709\u989d\u5916\u4e00\u6b21get_attr\u64cd\u4f5c\uff0c\u5373\u4f7f\u8c03\u5927 max_stat_cache_size=500000 \u6548\u679c\u4e5f\u4e0d\u7406\u60f3\uff0c\u76f8\u5f53\u4e8e get_attr + get_object \u4e24\u6b21 rpc\uff0c\u56e0\u6b64\u901f\u5ea6\u662f\u53e6\u5916\u4e8c\u8005\u7684\u4e00\u534a\uff0c20MB/s /2 = 10MB/s \u7b26\u5408\u9884\u671f\u3002","title":"Jindo Fuse \u6027\u80fd\u6d4b\u8bd5"},{"location":"backup/oss/jindo_fuse/jindo_fuse_benchmark/#jindo-fuse","text":"","title":"Jindo Fuse \u6027\u80fd\u6d4b\u8bd5"},{"location":"backup/oss/jindo_fuse/jindo_fuse_benchmark/#_1","text":"CPU: 80\u6838 | \u5185\u5b58\uff1a 128G \u7f51\u7edc\u89c4\u683c\uff1a25Gbps ECS\u89c4\u683c\uff1a ecs.hfc6.20xlarge","title":"\u6d4b\u8bd5\u73af\u5883"},{"location":"backup/oss/jindo_fuse/jindo_fuse_benchmark/#_2","text":"\u6570\u636e\u96c6A: \u5927\u6587\u4ef6\u96c6\u5408\uff0c\u6bcf\u4e2a\u6587\u4ef6140MB\uff0c\u603b\u5171140GB \u6570\u636e\u96c6B: \u5c0f\u6587\u4ef6\u96c6\u5408\uff0c\u6bcf\u4e2a\u6587\u4ef623KB\uff0c\u603b\u51711w\u4e2a","title":"\u6d4b\u8bd5\u6570\u636e\u96c6"},{"location":"backup/oss/jindo_fuse/jindo_fuse_benchmark/#_3","text":"\u5355\u7ebf\u7a0b\u6d4b\u8bd5 \u6570\u636e\u96c6 jindo-fuse ossfs OSS2/Python \u6570\u636e\u96c6A 191 MB/s 72 MB/s 17 MB/s \u6570\u636e\u96c6B 1.16 MB/s 1.25 MB/s 2.1 MB/s \u591a\u7ebf\u7a0b\u6d4b\u8bd5\uff0810\u4e2a\u7ebf\u7a0b\uff09 \u6570\u636e\u96c6 jindo-fuse ossfs OSS2/Python \u6570\u636e\u96c6A 1693MB/s 686 MB/s 162 MB/s \u6570\u636e\u96c6B 18 MB/s 10 MB/s 9 MB/s","title":"\u6d4b\u8bd5\u7ed3\u679c"},{"location":"backup/oss/jindo_fuse/jindo_fuse_benchmark/#_4","text":"jindo-fuse \u5bf9\u4e8e\u5927\u6587\u4ef6 \u6709\u9884\u8bfb\uff0c\u56e0\u6b64\u6bd4\u8f83\u5feb\u3002 OSS2 \u5bf9\u5c0f\u6587\u4ef6\u6bd4\u8f83\u5feb\uff0c\u5c0f\u6587\u4ef6 fuse \u65e0\u6cd5\u505a\u9884\u8bfb\uff0cOSS2 \u76f4\u63a5 http \u8bf7\u6c42\u6ca1\u6709\u989d\u5916\u5f00\u9500\uff0c\u6240\u4ee5\u6bd4\u8f83\u5feb\u3002 \u5c0f\u6587\u4ef6\u7ecf\u8fc7 fuse \u6709\u989d\u5916\u5f00\u9500\uff0c \u56e0\u6b64 jindo-fuse\u3001ossfs \u901f\u5ea6\u6bd4 OSS2 \u5dee\u3002 OSS2 \u5bf9\u5355\u8fdb\u7a0b\u591a\u7ebf\u7a0b\u6548\u679c\u4e0d\u597d\u3002 \u6211\u4eec\u53ef\u4ee5\u63a8\u6d4b OSS2 \u591a\u8fdb\u7a0b\u6210\u7ee9 2.1MB/s *10 = 21MB/s\u3002 ossfs \u5bf9\u5c0f\u6587\u4ef6\u56e0\u4e3a\u6709\u989d\u5916\u4e00\u6b21get_attr\u64cd\u4f5c\uff0c\u5373\u4f7f\u8c03\u5927 max_stat_cache_size=500000 \u6548\u679c\u4e5f\u4e0d\u7406\u60f3\uff0c\u76f8\u5f53\u4e8e get_attr + get_object \u4e24\u6b21 rpc\uff0c\u56e0\u6b64\u901f\u5ea6\u662f\u53e6\u5916\u4e8c\u8005\u7684\u4e00\u534a\uff0c20MB/s /2 = 10MB/s \u7b26\u5408\u9884\u671f\u3002","title":"\u7ed3\u679c\u5206\u6790"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/","text":"\u4f7f\u7528 JindoFuse \u8bbf\u95ee OSS \u00a7 OSS \u901a\u8fc7 JindoFuse \u63d0\u4f9b POSIX \u652f\u6301\u3002JindoFuse \u53ef\u4ee5\u628a OSS \u4e0a\u7684\u6587\u4ef6\u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c OSS \u4e0a\u7684\u6587\u4ef6\u3002 \u57fa\u672c\u4f7f\u7528 \u00a7 \u73af\u5883\u51c6\u5907 \u00a7 \u652f\u6301 libfuse 3.7 \u4ee5\u4e0a\u7248\u672c\uff0c\u63a8\u8350\u4f7f\u7528\u6700\u65b0\u7684 libfuse \u7248\u672c\u4ee5\u83b7\u5f97\u6700\u4f73\u6027\u80fd\u3002 # build fuse required meson & ninja, for debian: apt install -y pkg-config meson ninja-build sudo yum install -y meson ninja-build # compile fuse required newer g++ (only CentOS) g++ -v # compile & install libfuse wget https://github.com/libfuse/libfuse/releases/download/fuse-3.11.0/fuse-3.11.0.tar.xz xz -d fuse-3.11.0.tar.xz tar xf fuse-3.11.0.tar cd fuse-3.11.0/ mkdir build; cd build meson .. sudo ninja install \u914d\u7f6e\u5ba2\u6237\u7aef \u00a7 \u914d\u7f6e\u76ee\u5f55 \u00a7 \u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf \u914d\u7f6e\u6587\u4ef6 \u00a7 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95eeOSS\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> \u514d\u5bc6\u8bbf\u95ee \u00a7 \u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7\u89d2\u8272\u6388\u6743\u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE \u6302\u8f7d JindoFuse \u00a7 \u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002 \u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 mkdir -p <mount_point> \u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 jindo-fuse <mount_point> -ouri=[<oss_path>] -ouri\u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684oss\u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3aBucket\u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002 \u8bbf\u95ee JindoFuse \u00a7 \u5982\u679c\u5c06 OSS \u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1 ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/ \u5378\u8f7d JindoFuse \u00a7 \u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point> \u81ea\u52a8\u5378\u8f7d JindoFuse \u00a7 \u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u7279\u6027\u652f\u6301 \u00a7 \u76ee\u524d JindoFuse \u5df2\u7ecf\u652f\u6301\u4ee5\u4e0b POSIX API\uff1a \u7279\u6027 \u8bf4\u660e getattr() \u67e5\u8be2\u6587\u4ef6\u5c5e\u6027\uff0c\u7c7b\u4f3c ls mkdir() \u521b\u5efa\u76ee\u5f55\uff0c\u7c7b\u4f3c mkdir rmdir() \u5220\u9664\u76ee\u5f55\uff0c\u7c7b\u4f3c rm -rf unlink() \u5220\u9664\u6587\u4ef6\uff0c\u7c7b\u4f3c unlink rename() \u91cd\u547d\u540d\uff0c\u7c7b\u4f3c mv read() \u987a\u5e8f\u8bfb\u53d6 pread() \u968f\u673a\u8bfb write() \u987a\u5e8f\u5199 flush() \u5237\u65b0\u5185\u5b58\u5230\u5185\u6838\u7f13\u51b2\u533a\uff08\u4ec5\u652f\u6301\u4ee5\u6700\u8ffd\u52a0\u5199\u65b9\u5f0f\u6253\u5f00\u7684\u6587\u4ef6\uff09 fsync() \u5237\u65b0\u5185\u5b58\u5230\u78c1\u76d8\uff08\u4ec5\u652f\u6301\u4ee5\u8ffd\u52a0\u5199\u65b9\u5f0f\u6253\u5f00\u7684\u6587\u4ef6\uff09 release() \u5173\u95ed\u6587\u4ef6 readdir() \u8bfb\u53d6\u76ee\u5f55 create() \u521b\u5efa\u6587\u4ef6 open() O_APPEND \u901a\u8fc7\u8ffd\u52a0\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 open() O_TRUNC \u901a\u8fc7\u8986\u76d6\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 access() \u67e5\u8be2\u6587\u4ef6\u6743\u9650 \u9ad8\u9636\u4f7f\u7528 \u00a7 \u6302\u8f7d\u9009\u9879 \u00a7 \u53c2\u6570\u540d\u79f0 \u5fc5\u9009 \u53c2\u6570\u8bf4\u660e \u4f7f\u7528\u8303\u4f8b uri \u2713 \u914d\u7f6e\u9700\u8981\u6620\u5c04\u7684 oss \u8def\u5f84\u3002\u8def\u5f84\u53ef\u4ee5\u662f\u6839\u76ee\u5f55\uff0c\u4e5f\u53ef\u4ee5\u662f\u5b50\u76ee\u5f55\u3002\u4f8b\u5982\uff1aoss://bucket/ \u6216 oss://bucket/subdir\u3002 -ouri=oss://bucket/ f \u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u9ed8\u8ba4\u4f7f\u7528\u5b88\u62a4\u8fdb\u7a0b\u65b9\u5f0f\u540e\u53f0\u542f\u52a8\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -f d \u4f7f\u7528 Debug \u6a21\u5f0f\uff0c\u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -d auto_unmount fuse\u8fdb\u7a0b\u9000\u51fa\u540e\u81ea\u52a8umount\u6302\u8f7d\u8282\u70b9\u3002 -oauto_unmount ro \u53ea\u8bfb\u6302\u8f7d\uff0c\u542f\u7528\u53c2\u6570\u540e\u4e0d\u5141\u8bb8\u5199\u64cd\u4f5c\u3002 -oro direct_io \u5f00\u542f\u540e\uff0c\u8bfb\u5199\u6587\u4ef6\u53ef\u4ee5\u7ed5\u8fc7page cache\u3002 -odirect_io kernel_cache \u5f00\u542f\u540e\uff0c\u5229\u7528\u5185\u6838\u7f13\u5b58\u4f18\u5316\u8bfb\u6027\u80fd\u3002 -okernel_cache auto_cache \u9ed8\u8ba4\u5f00\u542f\uff0c\u4e0ekernel_cache \u4e8c\u9009\u4e00\uff0c\u4e0ekernel_cache\u4e0d\u540c\u7684\u662f\uff0c\u5982\u679c\u6587\u4ef6\u5927\u5c0f\u6216\u4fee\u6539\u65f6\u95f4\u53d1\u751f\u53d8\u5316\uff0c\u7f13\u5b58\u5c31\u4f1a\u5931\u6548\u3002 entry_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u76ee\u5f55\u6761\u76ee\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oentry_timeout=60 attr_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oattr_timeout=60 negative_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u5931\u8d25\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -onegative_timeout=0 jindo_entry_size \u9ed8\u8ba4\u503c\uff0c5000\u3002\u76ee\u5f55\u6761\u76ee\u7f13\u5b58\u6570\u91cf\uff0c\u7528\u4e8e\u4f18\u5316readdir\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -ojindo_entry_size=5000 jindo_attr_size \u9ed8\u8ba4\u503c\uff0c50000\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u6570\u91cf\uff0c\u7528\u4e8e\u4f18\u5316getattr\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -ojindo_attr_sizet=50000 max_idle_threads \u9ed8\u8ba4\u503c\uff0c10\u3002\u6700\u5927\u7a7a\u95f2\u7ebf\u7a0b\u6570\u3002 -omax_idle_threads=10 metrics_port \u9ed8\u8ba4\u503c\uff0c9090\u3002\u5f00\u542fhttp\u7aef\u53e3\uff0c\u8f93\u51fametrics\uff0c\u5982http://localhost:9090/brpc_metrics -ometrics_port=9090 enable_pread \u5f00\u542f\u540e\uff0c\u4f7f\u7528pread\u63a5\u53e3\u8bfb\u53d6\u6587\u4ef6\u3002 -oenable_pread \u914d\u7f6e\u9009\u9879 \u00a7 \u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u914d\u7f6e\u8282 \u8bf4\u660e logger.dir /tmp/fuse-log common \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false common \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false common \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\u3002 \u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002 logger.verbose 0 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false common \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint jindosdk \u8bbf\u95ee OSS \u7684\u5730\u5740\uff0c\u5982oss-cn-xxx.aliyuncs.com fs.oss.accessKeyId jindosdk \u8bbf\u95ee OSS \u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret jindosdk \u8bbf\u95ee OSS \u9700\u8981\u7684 accessKeySecret \u66f4\u591a jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<oss_path>] -ofs.oss.endpoint=[<your_endpoint>] -ofs.oss.accessKeyId=[<your_key_id>] -ofs.oss.accessKeySecret=[<your_key_secret>] \u5e38\u89c1\u95ee\u9898 \u00a7 Input/Output error \u00a7 \u4e0d\u50cf\u4f7f\u7528 JindoSDK \u8c03\u7528 API \u53ef\u4ee5\u83b7\u53d6\u66f4\u4e3a\u5177\u4f53\u7684 ErrorMsg\uff0cJindoFuse \u53ea\u80fd\u663e\u793a\u64cd\u4f5c\u7cfb\u7edf\u9884\u8bbe\u7684\u9519\u8bef\u4fe1\u606f\uff0c\u6bd4\u5982\u4ee5\u4e0b\u9519\u8bef\u5c31\u975e\u5e38\u5e38\u89c1\uff1a $ ls /mnt/oss/ ls: /mnt/oss/: Input/output error \u5982\u679c\u9700\u8981\u5b9a\u4f4d\u5177\u4f53\u7684\u9519\u8bef\u539f\u56e0\uff0c\u53ef\u4ee5\u6839\u636e JindoSDK \u914d\u7f6e\u4e2d\u7684 logger.dir\uff0c\u5728\u6307\u5b9a\u8def\u5f84\u4e0b\u7684 jindosdk.log \u6587\u4ef6\u4e2d\uff0c\u5bfb\u627e\u5177\u4f53\u7684\u9519\u8bef\u3002 \u4e0b\u9762\u5c55\u793a\u7684\u8fd9\u4e2a\u9519\u8bef\u5c31\u6765\u81ea jindosdk.log \uff0c\u6839\u636e\u62a5\u9519\u4fe1\u606f\u53ef\u89c1\uff0c\u8fd9\u4e2a\u9519\u8bef\u662f\u5e38\u89c1\u7684\u9274\u6743\u95ee\u9898\u3002 EMMDD HH:mm:ss jindofs_connectivity.cpp:13] Please check your Endpoint/Bucket/RoleArn. Failed test connectivity, operation: mkdir, errMsg: [RequestId]: 618B8183343EA53531C62B74 [HostId]: oss-cn-shanghai-internal.aliyuncs.com [ErrorMessage]: [E1010]HTTP/1.1 403 Forbidden ...","title":"\u4f7f\u7528 JindoFuse \u8bbf\u95ee OSS"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#jindofuse-oss","text":"OSS \u901a\u8fc7 JindoFuse \u63d0\u4f9b POSIX \u652f\u6301\u3002JindoFuse \u53ef\u4ee5\u628a OSS \u4e0a\u7684\u6587\u4ef6\u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c OSS \u4e0a\u7684\u6587\u4ef6\u3002","title":"\u4f7f\u7528 JindoFuse \u8bbf\u95ee OSS"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_1","text":"","title":"\u57fa\u672c\u4f7f\u7528"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_2","text":"\u652f\u6301 libfuse 3.7 \u4ee5\u4e0a\u7248\u672c\uff0c\u63a8\u8350\u4f7f\u7528\u6700\u65b0\u7684 libfuse \u7248\u672c\u4ee5\u83b7\u5f97\u6700\u4f73\u6027\u80fd\u3002 # build fuse required meson & ninja, for debian: apt install -y pkg-config meson ninja-build sudo yum install -y meson ninja-build # compile fuse required newer g++ (only CentOS) g++ -v # compile & install libfuse wget https://github.com/libfuse/libfuse/releases/download/fuse-3.11.0/fuse-3.11.0.tar.xz xz -d fuse-3.11.0.tar.xz tar xf fuse-3.11.0.tar cd fuse-3.11.0/ mkdir build; cd build meson .. sudo ninja install","title":"\u73af\u5883\u51c6\u5907"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_3","text":"","title":"\u914d\u7f6e\u5ba2\u6237\u7aef"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_4","text":"\u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf","title":"\u914d\u7f6e\u76ee\u5f55"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_5","text":"\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95eeOSS\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret>","title":"\u914d\u7f6e\u6587\u4ef6"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_6","text":"\u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7\u89d2\u8272\u6388\u6743\u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE","title":"\u514d\u5bc6\u8bbf\u95ee"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#jindofuse","text":"\u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002","title":"\u6302\u8f7d JindoFuse"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_7","text":"mkdir -p <mount_point>","title":"\u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#fuse","text":"jindo-fuse <mount_point> -ouri=[<oss_path>] -ouri\u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684oss\u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3aBucket\u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002","title":"\u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#jindofuse_1","text":"\u5982\u679c\u5c06 OSS \u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1 ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/","title":"\u8bbf\u95ee JindoFuse"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#jindofuse_2","text":"\u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point>","title":"\u5378\u8f7d JindoFuse"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#jindofuse_3","text":"\u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002","title":"\u81ea\u52a8\u5378\u8f7d JindoFuse"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_8","text":"\u76ee\u524d JindoFuse \u5df2\u7ecf\u652f\u6301\u4ee5\u4e0b POSIX API\uff1a \u7279\u6027 \u8bf4\u660e getattr() \u67e5\u8be2\u6587\u4ef6\u5c5e\u6027\uff0c\u7c7b\u4f3c ls mkdir() \u521b\u5efa\u76ee\u5f55\uff0c\u7c7b\u4f3c mkdir rmdir() \u5220\u9664\u76ee\u5f55\uff0c\u7c7b\u4f3c rm -rf unlink() \u5220\u9664\u6587\u4ef6\uff0c\u7c7b\u4f3c unlink rename() \u91cd\u547d\u540d\uff0c\u7c7b\u4f3c mv read() \u987a\u5e8f\u8bfb\u53d6 pread() \u968f\u673a\u8bfb write() \u987a\u5e8f\u5199 flush() \u5237\u65b0\u5185\u5b58\u5230\u5185\u6838\u7f13\u51b2\u533a\uff08\u4ec5\u652f\u6301\u4ee5\u6700\u8ffd\u52a0\u5199\u65b9\u5f0f\u6253\u5f00\u7684\u6587\u4ef6\uff09 fsync() \u5237\u65b0\u5185\u5b58\u5230\u78c1\u76d8\uff08\u4ec5\u652f\u6301\u4ee5\u8ffd\u52a0\u5199\u65b9\u5f0f\u6253\u5f00\u7684\u6587\u4ef6\uff09 release() \u5173\u95ed\u6587\u4ef6 readdir() \u8bfb\u53d6\u76ee\u5f55 create() \u521b\u5efa\u6587\u4ef6 open() O_APPEND \u901a\u8fc7\u8ffd\u52a0\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 open() O_TRUNC \u901a\u8fc7\u8986\u76d6\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 access() \u67e5\u8be2\u6587\u4ef6\u6743\u9650","title":"\u7279\u6027\u652f\u6301"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_9","text":"","title":"\u9ad8\u9636\u4f7f\u7528"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_10","text":"\u53c2\u6570\u540d\u79f0 \u5fc5\u9009 \u53c2\u6570\u8bf4\u660e \u4f7f\u7528\u8303\u4f8b uri \u2713 \u914d\u7f6e\u9700\u8981\u6620\u5c04\u7684 oss \u8def\u5f84\u3002\u8def\u5f84\u53ef\u4ee5\u662f\u6839\u76ee\u5f55\uff0c\u4e5f\u53ef\u4ee5\u662f\u5b50\u76ee\u5f55\u3002\u4f8b\u5982\uff1aoss://bucket/ \u6216 oss://bucket/subdir\u3002 -ouri=oss://bucket/ f \u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u9ed8\u8ba4\u4f7f\u7528\u5b88\u62a4\u8fdb\u7a0b\u65b9\u5f0f\u540e\u53f0\u542f\u52a8\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -f d \u4f7f\u7528 Debug \u6a21\u5f0f\uff0c\u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -d auto_unmount fuse\u8fdb\u7a0b\u9000\u51fa\u540e\u81ea\u52a8umount\u6302\u8f7d\u8282\u70b9\u3002 -oauto_unmount ro \u53ea\u8bfb\u6302\u8f7d\uff0c\u542f\u7528\u53c2\u6570\u540e\u4e0d\u5141\u8bb8\u5199\u64cd\u4f5c\u3002 -oro direct_io \u5f00\u542f\u540e\uff0c\u8bfb\u5199\u6587\u4ef6\u53ef\u4ee5\u7ed5\u8fc7page cache\u3002 -odirect_io kernel_cache \u5f00\u542f\u540e\uff0c\u5229\u7528\u5185\u6838\u7f13\u5b58\u4f18\u5316\u8bfb\u6027\u80fd\u3002 -okernel_cache auto_cache \u9ed8\u8ba4\u5f00\u542f\uff0c\u4e0ekernel_cache \u4e8c\u9009\u4e00\uff0c\u4e0ekernel_cache\u4e0d\u540c\u7684\u662f\uff0c\u5982\u679c\u6587\u4ef6\u5927\u5c0f\u6216\u4fee\u6539\u65f6\u95f4\u53d1\u751f\u53d8\u5316\uff0c\u7f13\u5b58\u5c31\u4f1a\u5931\u6548\u3002 entry_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u76ee\u5f55\u6761\u76ee\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oentry_timeout=60 attr_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oattr_timeout=60 negative_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u5931\u8d25\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -onegative_timeout=0 jindo_entry_size \u9ed8\u8ba4\u503c\uff0c5000\u3002\u76ee\u5f55\u6761\u76ee\u7f13\u5b58\u6570\u91cf\uff0c\u7528\u4e8e\u4f18\u5316readdir\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -ojindo_entry_size=5000 jindo_attr_size \u9ed8\u8ba4\u503c\uff0c50000\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u6570\u91cf\uff0c\u7528\u4e8e\u4f18\u5316getattr\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -ojindo_attr_sizet=50000 max_idle_threads \u9ed8\u8ba4\u503c\uff0c10\u3002\u6700\u5927\u7a7a\u95f2\u7ebf\u7a0b\u6570\u3002 -omax_idle_threads=10 metrics_port \u9ed8\u8ba4\u503c\uff0c9090\u3002\u5f00\u542fhttp\u7aef\u53e3\uff0c\u8f93\u51fametrics\uff0c\u5982http://localhost:9090/brpc_metrics -ometrics_port=9090 enable_pread \u5f00\u542f\u540e\uff0c\u4f7f\u7528pread\u63a5\u53e3\u8bfb\u53d6\u6587\u4ef6\u3002 -oenable_pread","title":"\u6302\u8f7d\u9009\u9879"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_11","text":"\u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u914d\u7f6e\u8282 \u8bf4\u660e logger.dir /tmp/fuse-log common \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false common \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false common \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\u3002 \u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002 logger.verbose 0 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false common \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint jindosdk \u8bbf\u95ee OSS \u7684\u5730\u5740\uff0c\u5982oss-cn-xxx.aliyuncs.com fs.oss.accessKeyId jindosdk \u8bbf\u95ee OSS \u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret jindosdk \u8bbf\u95ee OSS \u9700\u8981\u7684 accessKeySecret \u66f4\u591a jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<oss_path>] -ofs.oss.endpoint=[<your_endpoint>] -ofs.oss.accessKeyId=[<your_key_id>] -ofs.oss.accessKeySecret=[<your_key_secret>]","title":"\u914d\u7f6e\u9009\u9879"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_12","text":"","title":"\u5e38\u89c1\u95ee\u9898"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#inputoutput-error","text":"\u4e0d\u50cf\u4f7f\u7528 JindoSDK \u8c03\u7528 API \u53ef\u4ee5\u83b7\u53d6\u66f4\u4e3a\u5177\u4f53\u7684 ErrorMsg\uff0cJindoFuse \u53ea\u80fd\u663e\u793a\u64cd\u4f5c\u7cfb\u7edf\u9884\u8bbe\u7684\u9519\u8bef\u4fe1\u606f\uff0c\u6bd4\u5982\u4ee5\u4e0b\u9519\u8bef\u5c31\u975e\u5e38\u5e38\u89c1\uff1a $ ls /mnt/oss/ ls: /mnt/oss/: Input/output error \u5982\u679c\u9700\u8981\u5b9a\u4f4d\u5177\u4f53\u7684\u9519\u8bef\u539f\u56e0\uff0c\u53ef\u4ee5\u6839\u636e JindoSDK \u914d\u7f6e\u4e2d\u7684 logger.dir\uff0c\u5728\u6307\u5b9a\u8def\u5f84\u4e0b\u7684 jindosdk.log \u6587\u4ef6\u4e2d\uff0c\u5bfb\u627e\u5177\u4f53\u7684\u9519\u8bef\u3002 \u4e0b\u9762\u5c55\u793a\u7684\u8fd9\u4e2a\u9519\u8bef\u5c31\u6765\u81ea jindosdk.log \uff0c\u6839\u636e\u62a5\u9519\u4fe1\u606f\u53ef\u89c1\uff0c\u8fd9\u4e2a\u9519\u8bef\u662f\u5e38\u89c1\u7684\u9274\u6743\u95ee\u9898\u3002 EMMDD HH:mm:ss jindofs_connectivity.cpp:13] Please check your Endpoint/Bucket/RoleArn. Failed test connectivity, operation: mkdir, errMsg: [RequestId]: 618B8183343EA53531C62B74 [HostId]: oss-cn-shanghai-internal.aliyuncs.com [ErrorMessage]: [E1010]HTTP/1.1 403 Forbidden ...","title":"Input/Output error"},{"location":"backup/oss/jindo_fuse/jindo_fuse_overview/","text":"JindoFUSE \u4f7f\u7528 \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 \u76ee\u524d JindoFUSE \u53ea\u652f\u6301 Linux \u64cd\u4f5c\u7cfb\u7edf\u3002\u5927\u591a\u6570 Linux \u53d1\u884c\u7248\u5df2\u7ecf\u5185\u7f6e\u4e86 FUSE \u6a21\u5757\uff0c\u6ca1\u6709\u7684\u8bdd\u60a8\u9700\u8981\u5b89\u88c5\u6216\u8005\u7f16\u8bd1 FUSE \u6a21\u5757\u3002 \u4ec0\u4e48\u662f JindoFUSE \u00a7 FUSE \u662f Linux \u7cfb\u7edf\u5185\u6838\u63d0\u4f9b\u7684\u4e00\u79cd\u6302\u8f7d\u6587\u4ef6\u7cfb\u7edf\u7684\u65b9\u5f0f\u3002 \u901a\u8fc7 JindoFUSE \u5ba2\u6237\u7aef\uff0c\u5c06\u5bf9\u8c61\u5b58\u50a8 OSS \u670d\u52a1\u6216 DLS \u96c6\u7fa4\u4e0a\u7684\u6587\u4ef6\u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u4e00\u6837\u64cd\u4f5c OSS\u3001DLS \u96c6\u7fa4\u4e0a\u7684\u6587\u4ef6\u3002 \u4e3a\u4ec0\u4e48\u4f7f\u7528 JindoFUSE \u00a7 JindoFUSE \u57fa\u4e8e JindoSDK\uff0c\u5728\u8bbf\u95ee OSS \u7684\u80fd\u529b\u4e0a\uff0c\u76f8\u5bf9\u4e8e\u5f00\u6e90\u7684 ossfs\u5ba2\u6237\u7aef \u505a\u4e86\u5f88\u591a\u7684\u6027\u80fd\u4f18\u5316\u3002 JindoFUSE \u57fa\u4e8e C++ \u5b9e\u73b0\uff0c\u4e0d\u4f9d\u8d56 JVM\uff0c\u76f8\u5bf9\u4e8e HDFS \u6216 Alluxio \u7684 FUSE \u5b9e\u73b0\u8282\u7701\u4e86 JNI \u8c03\u7528\u7684\u5f00\u9500\u3002 \u5173\u4e8e JindoFUSE \u5728\u6027\u80fd\u4e0a\u7684\u4f18\u52bf\uff0c\u6211\u4eec\u505a\u4e86\u5bf9\u6bd4\u6d4b\u8bd5\uff0c\u53ef\u4ee5\u53c2\u8003 JindoFUSE \u6027\u80fd\u6d4b\u8bd5 \u3002 JindoFUSE \u4f7f\u7528 \u00a7 \u4f7f\u7528 JindoFUSE \u8bbf\u95ee OSS","title":"JindoFUSE \u4f7f\u7528"},{"location":"backup/oss/jindo_fuse/jindo_fuse_overview/#jindofuse","text":"","title":"JindoFUSE \u4f7f\u7528"},{"location":"backup/oss/jindo_fuse/jindo_fuse_overview/#_1","text":"\u76ee\u524d JindoFUSE \u53ea\u652f\u6301 Linux \u64cd\u4f5c\u7cfb\u7edf\u3002\u5927\u591a\u6570 Linux \u53d1\u884c\u7248\u5df2\u7ecf\u5185\u7f6e\u4e86 FUSE \u6a21\u5757\uff0c\u6ca1\u6709\u7684\u8bdd\u60a8\u9700\u8981\u5b89\u88c5\u6216\u8005\u7f16\u8bd1 FUSE \u6a21\u5757\u3002","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/oss/jindo_fuse/jindo_fuse_overview/#jindofuse_1","text":"FUSE \u662f Linux \u7cfb\u7edf\u5185\u6838\u63d0\u4f9b\u7684\u4e00\u79cd\u6302\u8f7d\u6587\u4ef6\u7cfb\u7edf\u7684\u65b9\u5f0f\u3002 \u901a\u8fc7 JindoFUSE \u5ba2\u6237\u7aef\uff0c\u5c06\u5bf9\u8c61\u5b58\u50a8 OSS \u670d\u52a1\u6216 DLS \u96c6\u7fa4\u4e0a\u7684\u6587\u4ef6\u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u4e00\u6837\u64cd\u4f5c OSS\u3001DLS \u96c6\u7fa4\u4e0a\u7684\u6587\u4ef6\u3002","title":"\u4ec0\u4e48\u662f JindoFUSE"},{"location":"backup/oss/jindo_fuse/jindo_fuse_overview/#jindofuse_2","text":"JindoFUSE \u57fa\u4e8e JindoSDK\uff0c\u5728\u8bbf\u95ee OSS \u7684\u80fd\u529b\u4e0a\uff0c\u76f8\u5bf9\u4e8e\u5f00\u6e90\u7684 ossfs\u5ba2\u6237\u7aef \u505a\u4e86\u5f88\u591a\u7684\u6027\u80fd\u4f18\u5316\u3002 JindoFUSE \u57fa\u4e8e C++ \u5b9e\u73b0\uff0c\u4e0d\u4f9d\u8d56 JVM\uff0c\u76f8\u5bf9\u4e8e HDFS \u6216 Alluxio \u7684 FUSE \u5b9e\u73b0\u8282\u7701\u4e86 JNI \u8c03\u7528\u7684\u5f00\u9500\u3002 \u5173\u4e8e JindoFUSE \u5728\u6027\u80fd\u4e0a\u7684\u4f18\u52bf\uff0c\u6211\u4eec\u505a\u4e86\u5bf9\u6bd4\u6d4b\u8bd5\uff0c\u53ef\u4ee5\u53c2\u8003 JindoFUSE \u6027\u80fd\u6d4b\u8bd5 \u3002","title":"\u4e3a\u4ec0\u4e48\u4f7f\u7528 JindoFUSE"},{"location":"backup/oss/jindo_fuse/jindo_fuse_overview/#jindofuse_3","text":"\u4f7f\u7528 JindoFUSE \u8bbf\u95ee OSS","title":"JindoFUSE \u4f7f\u7528"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/","text":"\u4f7f\u7528 Jindo Tensorflow Collector \u8bbf\u95ee OSS \u00a7 Jindo Tensorflow Collector \u652f\u6301\u4f7f\u7528 Tensoflow \u539f\u751f API \u652f\u6301\u8bbf\u95ee OSS\u3002 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u4fee\u6539\u914d\u7f6e \u00a7 2.1 \u5b89\u88c5 tensorflow \u00a7 \u76ee\u524d\u4ec5\u652f\u6301 tensorflow 1.15 \u6216 tensorflow 2.8 \u7248\u672c\uff0c\u4ee5\u4e0b\u4ee5 tensorflow 2.8 \u4e3a\u4f8b\u3002 pip3 install tensorflow==2.8 2.2 \u6dfb\u52a0\u73af\u5883\u53d8\u91cf \u00a7 2.2.1 JindoSDK \u73af\u5883\u53d8\u91cf \u00a7 export LD_LIBRARY_PATH=\"/path/to/jindosdk-x.x.x/lib/native\":$LD_LIBRARY_PATH export JINDOSDK_CONF_DIR=/path/to/jindosdk-x.x.x/conf 2.2.2 \u4f9d\u8d56\u7684 Tensorflow \u73af\u5883\u53d8\u91cf \u00a7 \u540c\u65f6\uff0cLD_LIBRARY_PATH \u9700\u8981\u6dfb\u52a0 tensorflow \u6240\u5728\u7684\u4f9d\u8d56\u8def\u5f84\uff0c\u4e0d\u540c\u7684 python \u73af\u5883\u5b89\u88c5\u8def\u5f84\u53ef\u80fd\u4e0d\u540c\uff0c\u5982\uff1a # tensorflow1.15 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow/\":$LD_LIBRARY_PATH # tensorflow2.8 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow/\":$LD_LIBRARY_PATH 2.3 \u914d\u7f6e\u6587\u4ef6 \u00a7 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \u3002\u53ef\u4ee5\u9009\u62e9\u56fa\u5b9a\u6216\u8005\u514d\u5bc6\u4e24\u79cd\u914d\u7f6e\u65b9\u5f0f\u3002 2.3.1 \u56fa\u5b9a AK \u8bbf\u95ee \u00a7 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/tfio-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95eeOSS\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> \u914d\u7f6e\u9009\u9879 \u00a7 \u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u914d\u7f6e\u8282 \u8bf4\u660e logger.dir /tmp/tfio-log common \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false common \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false common \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\u3002 \u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002 logger.verbose 0 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false common \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u5730\u5740\uff0c\u5982cn-xxx.oss-dls.aliyuncs.com fs.oss.accessKeyId jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeySecret \u66f4\u591a jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 2.3.2 \u514d\u5bc6\u8bbf\u95ee \u00a7 \u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7\u89d2\u8272\u6388\u6743\u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE 3. \u4f7f\u7528 Jindo Tensorflow \u00a7 \u4ee5\u52a0\u8f7d jindo-tensorflow2.8 \u4e3a\u4f8b 3.1 \u901a\u8fc7 load_library \u6dfb\u52a0 collector \u00a7 import tensorflow as tf tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\") 3.2 \u901a\u8fc7 gfile \u8bbf\u95ee oss-hdfs \u00a7 \u4e3a\u907f\u514d\u4e0e tfio \u4e2d\u7684 oss \u524d\u7f00\u51b2\u7a81\uff0cjindo-tensorflow \u4f7f\u7528 joss \u524d\u7f00\u3002 # \u76ee\u5f55\u64cd\u4f5c tf.io.gfile.mkdir(\"joss://<bucket>/test_mkdir\") content = tf.io.gfile.listdir(\"joss://<bucket>/test_mkdir\") # \u8bfb gmnist = tf.io.gfile.GFile(\"joss://<bucket>/mnist.npz\", \"rb\") gmnist_file = gmnist.read() # \u5199 fh = gfile.Open(f, mode=\"w\") content = \"file content\" fh.write(content) fh.close() 3.3 \u8bfb\u53d6 TFRecordDataset \u00a7 tfRecordOssPath = \"joss://<bucket>/test.tfrecord\" filenames = [tfRecordOssPath] raw_dataset = tf.data.TFRecordDataset(filenames) 3.4 \u5b8c\u6574\u793a\u4f8b \u00a7 import tensorflow as tf import numpy as np tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\") gmnist = tf.io.gfile.GFile(\"joss://<bucket>/mnist.npz\", 'rb') print(gmnist.size()) with np.load(gmnist) as f: x_train, y_train = f['x_train'], f['y_train'] x_test, y_test = f['x_test'], f['y_test'] model = tf.keras.models.Sequential([ tf.keras.layers.Flatten(input_shape=(28, 28)), tf.keras.layers.Dense(128, activation='relu'), tf.keras.layers.Dropout(0.2), tf.keras.layers.Dense(10, activation='softmax') ]) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) model.fit(x_train, y_train, epochs=5)","title":"\u4f7f\u7528 Jindo Tensorflow Collector \u8bbf\u95ee OSS"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#jindo-tensorflow-collector-oss","text":"Jindo Tensorflow Collector \u652f\u6301\u4f7f\u7528 Tensoflow \u539f\u751f API \u652f\u6301\u8bbf\u95ee OSS\u3002","title":"\u4f7f\u7528 Jindo Tensorflow Collector \u8bbf\u95ee OSS"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#2","text":"","title":"2. \u4fee\u6539\u914d\u7f6e"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#21-tensorflow","text":"\u76ee\u524d\u4ec5\u652f\u6301 tensorflow 1.15 \u6216 tensorflow 2.8 \u7248\u672c\uff0c\u4ee5\u4e0b\u4ee5 tensorflow 2.8 \u4e3a\u4f8b\u3002 pip3 install tensorflow==2.8","title":"2.1 \u5b89\u88c5 tensorflow"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#22","text":"","title":"2.2 \u6dfb\u52a0\u73af\u5883\u53d8\u91cf"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#221-jindosdk","text":"export LD_LIBRARY_PATH=\"/path/to/jindosdk-x.x.x/lib/native\":$LD_LIBRARY_PATH export JINDOSDK_CONF_DIR=/path/to/jindosdk-x.x.x/conf","title":"2.2.1 JindoSDK \u73af\u5883\u53d8\u91cf"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#222-tensorflow","text":"\u540c\u65f6\uff0cLD_LIBRARY_PATH \u9700\u8981\u6dfb\u52a0 tensorflow \u6240\u5728\u7684\u4f9d\u8d56\u8def\u5f84\uff0c\u4e0d\u540c\u7684 python \u73af\u5883\u5b89\u88c5\u8def\u5f84\u53ef\u80fd\u4e0d\u540c\uff0c\u5982\uff1a # tensorflow1.15 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow/\":$LD_LIBRARY_PATH # tensorflow2.8 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow/\":$LD_LIBRARY_PATH","title":"2.2.2 \u4f9d\u8d56\u7684 Tensorflow \u73af\u5883\u53d8\u91cf"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#23","text":"\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \u3002\u53ef\u4ee5\u9009\u62e9\u56fa\u5b9a\u6216\u8005\u514d\u5bc6\u4e24\u79cd\u914d\u7f6e\u65b9\u5f0f\u3002","title":"2.3 \u914d\u7f6e\u6587\u4ef6"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#231-ak","text":"\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/tfio-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95eeOSS\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret>","title":"2.3.1 \u56fa\u5b9a AK \u8bbf\u95ee"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#_1","text":"\u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u914d\u7f6e\u8282 \u8bf4\u660e logger.dir /tmp/tfio-log common \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false common \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false common \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\u3002 \u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002 logger.verbose 0 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false common \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u5730\u5740\uff0c\u5982cn-xxx.oss-dls.aliyuncs.com fs.oss.accessKeyId jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeySecret \u66f4\u591a jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002","title":"\u914d\u7f6e\u9009\u9879"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#232","text":"\u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7\u89d2\u8272\u6388\u6743\u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE","title":"2.3.2 \u514d\u5bc6\u8bbf\u95ee"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#3-jindo-tensorflow","text":"\u4ee5\u52a0\u8f7d jindo-tensorflow2.8 \u4e3a\u4f8b","title":"3. \u4f7f\u7528 Jindo Tensorflow"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#31-load_library-collector","text":"import tensorflow as tf tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\")","title":"3.1 \u901a\u8fc7 load_library \u6dfb\u52a0 collector"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#32-gfile-oss-hdfs","text":"\u4e3a\u907f\u514d\u4e0e tfio \u4e2d\u7684 oss \u524d\u7f00\u51b2\u7a81\uff0cjindo-tensorflow \u4f7f\u7528 joss \u524d\u7f00\u3002 # \u76ee\u5f55\u64cd\u4f5c tf.io.gfile.mkdir(\"joss://<bucket>/test_mkdir\") content = tf.io.gfile.listdir(\"joss://<bucket>/test_mkdir\") # \u8bfb gmnist = tf.io.gfile.GFile(\"joss://<bucket>/mnist.npz\", \"rb\") gmnist_file = gmnist.read() # \u5199 fh = gfile.Open(f, mode=\"w\") content = \"file content\" fh.write(content) fh.close()","title":"3.2 \u901a\u8fc7 gfile \u8bbf\u95ee oss-hdfs"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#33-tfrecorddataset","text":"tfRecordOssPath = \"joss://<bucket>/test.tfrecord\" filenames = [tfRecordOssPath] raw_dataset = tf.data.TFRecordDataset(filenames)","title":"3.3 \u8bfb\u53d6 TFRecordDataset"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#34","text":"import tensorflow as tf import numpy as np tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\") gmnist = tf.io.gfile.GFile(\"joss://<bucket>/mnist.npz\", 'rb') print(gmnist.size()) with np.load(gmnist) as f: x_train, y_train = f['x_train'], f['y_train'] x_test, y_test = f['x_test'], f['y_test'] model = tf.keras.models.Sequential([ tf.keras.layers.Flatten(input_shape=(28, 28)), tf.keras.layers.Dense(128, activation='relu'), tf.keras.layers.Dropout(0.2), tf.keras.layers.Dense(10, activation='softmax') ]) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) model.fit(x_train, y_train, epochs=5)","title":"3.4 \u5b8c\u6574\u793a\u4f8b"},{"location":"backup/oss/jindotable/jindotable_archive/","text":"JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a \u00a7 JindoTable \u63d0\u4f9b archiveTable \u548c unarchiveTable \u547d\u4ee4\uff0c\u53ef\u4ee5\u8fdb\u884c\u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a\u7684\u64cd\u4f5c\u3002\u672c\u6587\u4e3a\u60a8\u4ecb\u7ecd archiveTable \u548c unarchiveTable \u547d\u4ee4\u7684\u4f7f\u7528\u65b9\u6cd5\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5f85\u5f52\u6863\u7684\u6570\u636e\u5fc5\u987b\u662f\u8868\u6570\u636e\uff08\u53ef\u4ee5\u662f\u5206\u533a\u8868\u6216\u975e\u5206\u533a\u8868\uff09\uff0c\u4e14\u5df2\u7ecf\u4f4d\u4e8e\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8 OSS\u3002 \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 JindoSDK + \u963f\u91cc\u4e91 OSS \u5feb\u901f\u5165\u95e8 \u3002 archiveTable \u547d\u4ee4 \u00a7 archiveTable \u547d\u4ee4\u53ef\u4ee5\u5bf9 OSS \u4e0a\u7684\u8868\u6216\u5206\u533a\u8fdb\u884c\u5f52\u6863\u3002 \u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f \u00a7 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help archiveTable \u53c2\u6570\u8bf4\u660e \u00a7 jindotable -archiveTable -t <dbName.tableName> -i/-a/-ca [-c \"<condition>\" | -fullTable] [-b/-before <before days>] [-p/-parallel <parallelism>] [-mr/-mapReduce] [-e/-explain] [-w/-workingDir <working directory>][-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u5f85\u5f52\u6863\u7684\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a\u6570\u636e\u5e93\u540d.\u8868\u540d\u3002\u6570\u636e\u5e93\u548c\u8868\u540d\u4e4b\u95f4\u4ee5\u534a\u89d2\u53e5\u53f7\uff08.\uff09\u5206\u9694\u3002\u8868\u53ef\u4ee5\u662f\u5206\u533a\u8868\u6216\u975e\u5206\u533a\u8868\u3002 \u662f -i/-a/-ca \u76ee\u6807\u5b58\u50a8\u65b9\u5f0f\u3002\u652f\u6301\u5982\u4e0b\u65b9\u5f0f\uff1a-a\uff1a\u5f52\u6863\uff08Archive\uff09\u5b58\u50a8\u3002-i\uff1a\u4f4e\u9891 \uff08Infrequent Access\uff0cIA\uff09\u5b58\u50a8\u3002-ca\uff1a\u51b7\u5f52\u6863\uff08Code Archive\uff09\u5b58\u50a8\u3002\u5982\u679c\u4f7f\u7528-i\u5373\u8868\u793a\u4f4e\u9891\u5b58\u50a8\uff0c\u4f1a\u8df3\u8fc7\u5df2\u7ecf\u5904\u4e8e\u5f52\u6863\u5b58\u50a8\u7684\u6587\u4ef6\u3002 \u662f -c \"<condition>\" / -fullTable -fullTable \u548c -c \"<condition>\" \u53ea\u9700\u63d0\u4f9b\u4e00\u4e2a\uff0c\u5373\u8981\u4e48\u6307\u5b9a -c \"<condition>\" \uff0c\u8981\u4e48\u6307\u5b9a -fullTable \u3002\u6307\u5b9a -fullTable \u65f6\uff0c\u5219\u4e3a\u79fb\u52a8\u6574\u8868\uff0c\u65e2\u53ef\u4ee5\u662f\u975e\u5206\u533a\u8868\u4e5f\u53ef\u4ee5\u662f\u5206\u533a\u8868\u3002\u6307\u5b9a -c \"<condition>\" \u65f6\uff0c\u5219\u63d0\u4f9b\u4e86\u4e00\u4e2a\u8fc7\u6ee4\u6761\u4ef6\uff0c\u7528\u6765\u9009\u62e9\u5e0c\u671b\u79fb\u52a8\u7684\u5206\u533a\uff0c\u652f\u6301\u5e38\u89c1\u8fd0\u7b97\u7b26\uff0c\u4f8b\u5982\u5927\u4e8e\u53f7\uff08>\uff09\u3002\u4f8b\u5982\uff0c\u6570\u636e\u7c7b\u578b\u4e3aString\u7684\u5206\u533ads\uff0c\u5e0c\u671b\u5206\u533a\u540d\u5927\u4e8e 'd'\uff0c\u5219\u4ee3\u7801\u4e3a-c \" ds > 'd' \"\u3002 \u5426 -b/before <before days> \u53ea\u6709\u521b\u5efa\u65f6\u95f4\u8ddd\u79bb\u73b0\u5728\u8d85\u8fc7\u4e00\u5b9a\u5929\u6570\u7684\u8868\u6216\u5206\u533a\u624d\u4f1a\u88ab\u5f52\u6863\u3002 \u5426 -p/-parallel <parallelism> \u5f52\u6863\u64cd\u4f5c\u7684\u5e76\u884c\u5ea6\u3002 \u5426 -mr/-mapReduce \u4f7f\u7528Hadoop MapReduce\u800c\u975e\u672c\u5730\u591a\u7ebf\u7a0b\u6765\u5f52\u6863\u6570\u636e\u3002 \u5426 -e/-explain \u5982\u679c\u51fa\u73b0\u8be5\u9009\u9879\uff0c\u5219\u4e3a\u89e3\u91ca\uff08explain\uff09\u6a21\u5f0f\uff0c\u53ea\u4f1a\u663e\u793a\u5f85\u79fb\u52a8\u7684\u5206\u533a\u5217\u8868\uff0c\u800c\u4e0d\u4f1a\u771f\u6b63\u79fb\u52a8\u6570\u636e\u3002 \u5426 -w/-workingDir \u53ea\u5728MapReduce\u4f5c\u4e1a\u65f6\u4f7f\u7528\uff0c\u4e3aMapReduce\u4f5c\u4e1a\u7684\u5de5\u4f5c\u76ee\u5f55\u3002\u5fc5\u987b\u6709\u8bfb\u5199\u6743\u9650\uff0c\u5de5\u4f5c\u76ee\u5f55\u53ef\u4ee5\u975e\u7a7a\uff08\u4f5c\u4e1a\u6267\u884c\u8fc7\u7a0b\u4e2d\u4f1a\u521b\u5efa\u4e34\u65f6\u6587\u4ef6\uff0c\u6267\u884c\u5b8c\u6bd5\u4f1a\u6e05\u7406\u4e34\u65f6\u6587\u4ef6\uff09\u3002 \u5426 -l/-logDir <log directory> \u6307\u5b9aLog\u6587\u4ef6\u7684\u76ee\u5f55\u3002 \u5426 unarchiveTable \u547d\u4ee4 \u00a7 unarchiveTable \u547d\u4ee4\u4e0e archiveTable \u547d\u4ee4\u683c\u5f0f\u57fa\u672c\u4e00\u81f4\uff0c\u4f46\u6548\u679c\u76f8\u53cd\u3002unarchiveTable \u547d\u4ee4\u53ef\u4ee5\u5bf9 OSS \u4e0a\u7684\u8868\u6216\u5206\u533a\u8fdb\u884c\u89e3\u51bb\u3002 \u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f \u00a7 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help unarchiveTable \u53c2\u6570\u8bf4\u660e \u00a7 jindotable -unarchiveTable -t <dbName.tableName> [-i/-a/-o/-cr] [-notWait] [-c \"<condition>\" | -fullTable] [-d/-restoreDays <restore days>] [-b/-before <before days>] [-p/-parallel <parallelism>] [-mr/-mapReduce] [-e/-explain] [-w/-workingDir <working directory>][-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u5f85\u89e3\u51bb\u7684\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a\u6570\u636e\u5e93\u540d.\u8868\u540d\u3002\u6570\u636e\u5e93\u548c\u8868\u540d\u4e4b\u95f4\u4ee5\u534a\u89d2\u53e5\u53f7\uff08.\uff09\u5206\u9694\u3002\u8868\u53ef\u4ee5\u662f\u5206\u533a\u8868\u6216\u975e\u5206\u533a\u8868\u3002 \u662f -i/-a/-o/-cr \u5982\u679c\u4e0d\u6307\u5b9a\u53c2\u6570\uff0c\u5219\u8f6c\u6362\u5b58\u50a8\u683c\u5f0f\u4e3a\u6807\u51c6\uff08Standard\uff09\u5b58\u50a8\u3002\u652f\u6301\u5982\u4e0b\u65b9\u5f0f\uff1a-i\uff1a\u5219\u8f6c\u6362\u5b58\u50a8\u683c\u5f0f\u4e3a\u4f4e\u9891\uff08Infrequent Access\uff0cIA\uff09\u5b58\u50a8\uff0c\u539f\u672c\u4e3a\u6807\u51c6\u5b58\u50a8\u7684\u6587\u4ef6\u88ab\u8df3\u8fc7\u3002-a\uff1a\u5219\u8f6c\u6362\u5b58\u50a8\u683c\u5f0f\u5f52\u6863\uff08Archive\uff09\u5b58\u50a8\u3002-o: \u4ec5\u505a\u89e3\u51bb\uff08Restore\uff09\u64cd\u4f5c\u3002\u539f\u672c\u4e3a\u6807\u51c6\u5b58\u50a8\u6216\u4f4e\u9891\u5b58\u50a8\u7684\u6587\u4ef6\u5747\u88ab\u8df3\u8fc7\u3002\u5df2\u7ecf\u5904\u4e8e\u89e3\u51bb\u72b6\u6001\u7684\u6587\u4ef6\u4e5f\u4f1a\u88ab\u8df3\u8fc7\uff0c\u5373\u4e0d\u4f1a\u91cd\u590d\u89e3\u51bb\u3002-cr\uff1a\u68c0\u67e5\u662f\u5426\u6240\u6709 restore \u4efb\u52a1\u5df2\u5b8c\u6210\u3002 \u5426 -notWait \u53ea\u5bf9 Restore \u64cd\u4f5c\u751f\u6548\u3002\u5982\u679c\u51fa\u73b0\u8be5\u9009\u9879\uff0c\u5219\u4e0d\u4f1a\u7b49 OSS \u670d\u52a1\u7aef\u5b8c\u6210 Restore \u64cd\u4f5c\u3002\u5426\u5219\uff0c\u4f1a\u7b49 Restore \u64cd\u4f5c\u5b8c\u6210\u6216\u8005\u8d85\u65f6\uff0810\u5206\u949f\uff09\u624d\u9000\u51fa \u5426 -c \"<condition>\" / -fullTable -fullTable \u548c -c \"<condition>\" \u53ea\u9700\u63d0\u4f9b\u4e00\u4e2a\uff0c\u5373\u8981\u4e48\u6307\u5b9a -c \"<condition>\" \uff0c\u8981\u4e48\u6307\u5b9a -fullTable \u3002\u6307\u5b9a -fullTable \u65f6\uff0c\u5219\u4e3a\u79fb\u52a8\u6574\u8868\uff0c\u65e2\u53ef\u4ee5\u662f\u975e\u5206\u533a\u8868\u4e5f\u53ef\u4ee5\u662f\u5206\u533a\u8868\u3002\u6307\u5b9a -c \"<condition>\" \u65f6\uff0c\u5219\u63d0\u4f9b\u4e86\u4e00\u4e2a\u8fc7\u6ee4\u6761\u4ef6\uff0c\u7528\u6765\u9009\u62e9\u5e0c\u671b\u79fb\u52a8\u7684\u5206\u533a\uff0c\u652f\u6301\u5e38\u89c1\u8fd0\u7b97\u7b26\uff0c\u4f8b\u5982\u5927\u4e8e\u53f7\uff08>\uff09\u3002\u4f8b\u5982\uff0c\u6570\u636e\u7c7b\u578b\u4e3aString\u7684\u5206\u533ads\uff0c\u5e0c\u671b\u5206\u533a\u540d\u5927\u4e8e 'd'\uff0c\u5219\u4ee3\u7801\u4e3a-c \" ds > 'd' \"\u3002 \u5426 -d/-restoreDays <restore days> \u5f53\u6d89\u53ca\u4ec5\u89e3\u51bb\uff08Restore\uff09\u64cd\u4f5c\u65f6\uff0c\u7ef4\u6301\u89e3\u51bb\u72b6\u6001\u7684\u5929\u6570\uff0c\u9ed8\u8ba4\u4e3a\u4e00\u5929\u3002\u4f8b\u5982\uff0c\u5bf9\u4e00\u4e2a\u51b7\u5f52\u6863\u7684\u8868\u505a\u4ec5\u89e3\u51bb\u64cd\u4f5c\u65f6\uff08-o \u53c2\u6570\uff09\uff0c\u8be5\u53c2\u6570\u5c06\u4ea7\u751f\u4f5c\u7528\uff1b\u5bf9\u4e00\u4e2a\u4ece\u672a\u89e3\u51bb\u8fc7\u7684\u5f52\u6863\u72b6\u6001\u7684\u8868\u8f6c\u4e3a\u6807\u51c6\u5b58\u50a8\u65f6\uff0c\u8be5\u53c2\u6570\u5c06\u5728\u4e2d\u95f4\u73af\u8282\u53d1\u6325\u4f5c\u7528\uff1b\u800c\u5bf9\u4e00\u4e2a\u4f4e\u9891\u72b6\u6001\u7684\u8868\u8f6c\u4e3a\u6807\u51c6\u5b58\u50a8\u65f6\uff0c\u8be5\u53c2\u6570\u4e0d\u4f1a\u4ea7\u751f\u4f5c\u7528\u3002 \u5426 -b/-before <before days> \u53ea\u6709\u521b\u5efa\u65f6\u95f4\u8ddd\u79bb\u73b0\u5728\u8d85\u8fc7\u4e00\u5b9a\u5929\u6570\u7684\u8868\u6216\u5206\u533a\u624d\u4f1a\u88ab\u89e3\u51bb\u3002 \u5426 -p/-parallel <parallelism> \u89e3\u51bb\u64cd\u4f5c\u7684\u5e76\u884c\u5ea6\u3002 \u5426 -mr/-mapReduce \u4f7f\u7528Hadoop MapReduce\u800c\u975e\u672c\u5730\u591a\u7ebf\u7a0b\u6765\u89e3\u51bb\u6570\u636e\u3002 \u5426 -e/-explain \u5982\u679c\u51fa\u73b0\u8be5\u9009\u9879\uff0c\u5219\u4e3a\u89e3\u91ca\uff08explain\uff09\u6a21\u5f0f\uff0c\u53ea\u4f1a\u663e\u793a\u5f85\u79fb\u52a8\u7684\u5206\u533a\u5217\u8868\uff0c\u800c\u4e0d\u4f1a\u771f\u6b63\u79fb\u52a8\u6570\u636e\u3002 \u5426 -w/-workingDir \u53ea\u5728MapReduce\u4f5c\u4e1a\u65f6\u4f7f\u7528\uff0c\u4e3aMapReduce\u4f5c\u4e1a\u7684\u5de5\u4f5c\u76ee\u5f55\u3002\u5fc5\u987b\u6709\u8bfb\u5199\u6743\u9650\uff0c\u5de5\u4f5c\u76ee\u5f55\u53ef\u4ee5\u975e\u7a7a\uff08\u4f5c\u4e1a\u6267\u884c\u8fc7\u7a0b\u4e2d\u4f1a\u521b\u5efa\u4e34\u65f6\u6587\u4ef6\uff0c\u6267\u884c\u5b8c\u6bd5\u4f1a\u6e05\u7406\u4e34\u65f6\u6587\u4ef6\uff09\u3002 \u5426 -l/-logDir <log directory> \u6307\u5b9aLog\u6587\u4ef6\u7684\u76ee\u5f55\u3002 \u5426","title":"JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a"},{"location":"backup/oss/jindotable/jindotable_archive/#jindotable-oss","text":"JindoTable \u63d0\u4f9b archiveTable \u548c unarchiveTable \u547d\u4ee4\uff0c\u53ef\u4ee5\u8fdb\u884c\u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a\u7684\u64cd\u4f5c\u3002\u672c\u6587\u4e3a\u60a8\u4ecb\u7ecd archiveTable \u548c unarchiveTable \u547d\u4ee4\u7684\u4f7f\u7528\u65b9\u6cd5\u3002","title":"JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a"},{"location":"backup/oss/jindotable/jindotable_archive/#_1","text":"\u5f85\u5f52\u6863\u7684\u6570\u636e\u5fc5\u987b\u662f\u8868\u6570\u636e\uff08\u53ef\u4ee5\u662f\u5206\u533a\u8868\u6216\u975e\u5206\u533a\u8868\uff09\uff0c\u4e14\u5df2\u7ecf\u4f4d\u4e8e\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8 OSS\u3002 \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 JindoSDK + \u963f\u91cc\u4e91 OSS \u5feb\u901f\u5165\u95e8 \u3002","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/oss/jindotable/jindotable_archive/#archivetable","text":"archiveTable \u547d\u4ee4\u53ef\u4ee5\u5bf9 OSS \u4e0a\u7684\u8868\u6216\u5206\u533a\u8fdb\u884c\u5f52\u6863\u3002","title":"archiveTable \u547d\u4ee4"},{"location":"backup/oss/jindotable/jindotable_archive/#_2","text":"\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help archiveTable","title":"\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f"},{"location":"backup/oss/jindotable/jindotable_archive/#_3","text":"jindotable -archiveTable -t <dbName.tableName> -i/-a/-ca [-c \"<condition>\" | -fullTable] [-b/-before <before days>] [-p/-parallel <parallelism>] [-mr/-mapReduce] [-e/-explain] [-w/-workingDir <working directory>][-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u5f85\u5f52\u6863\u7684\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a\u6570\u636e\u5e93\u540d.\u8868\u540d\u3002\u6570\u636e\u5e93\u548c\u8868\u540d\u4e4b\u95f4\u4ee5\u534a\u89d2\u53e5\u53f7\uff08.\uff09\u5206\u9694\u3002\u8868\u53ef\u4ee5\u662f\u5206\u533a\u8868\u6216\u975e\u5206\u533a\u8868\u3002 \u662f -i/-a/-ca \u76ee\u6807\u5b58\u50a8\u65b9\u5f0f\u3002\u652f\u6301\u5982\u4e0b\u65b9\u5f0f\uff1a-a\uff1a\u5f52\u6863\uff08Archive\uff09\u5b58\u50a8\u3002-i\uff1a\u4f4e\u9891 \uff08Infrequent Access\uff0cIA\uff09\u5b58\u50a8\u3002-ca\uff1a\u51b7\u5f52\u6863\uff08Code Archive\uff09\u5b58\u50a8\u3002\u5982\u679c\u4f7f\u7528-i\u5373\u8868\u793a\u4f4e\u9891\u5b58\u50a8\uff0c\u4f1a\u8df3\u8fc7\u5df2\u7ecf\u5904\u4e8e\u5f52\u6863\u5b58\u50a8\u7684\u6587\u4ef6\u3002 \u662f -c \"<condition>\" / -fullTable -fullTable \u548c -c \"<condition>\" \u53ea\u9700\u63d0\u4f9b\u4e00\u4e2a\uff0c\u5373\u8981\u4e48\u6307\u5b9a -c \"<condition>\" \uff0c\u8981\u4e48\u6307\u5b9a -fullTable \u3002\u6307\u5b9a -fullTable \u65f6\uff0c\u5219\u4e3a\u79fb\u52a8\u6574\u8868\uff0c\u65e2\u53ef\u4ee5\u662f\u975e\u5206\u533a\u8868\u4e5f\u53ef\u4ee5\u662f\u5206\u533a\u8868\u3002\u6307\u5b9a -c \"<condition>\" \u65f6\uff0c\u5219\u63d0\u4f9b\u4e86\u4e00\u4e2a\u8fc7\u6ee4\u6761\u4ef6\uff0c\u7528\u6765\u9009\u62e9\u5e0c\u671b\u79fb\u52a8\u7684\u5206\u533a\uff0c\u652f\u6301\u5e38\u89c1\u8fd0\u7b97\u7b26\uff0c\u4f8b\u5982\u5927\u4e8e\u53f7\uff08>\uff09\u3002\u4f8b\u5982\uff0c\u6570\u636e\u7c7b\u578b\u4e3aString\u7684\u5206\u533ads\uff0c\u5e0c\u671b\u5206\u533a\u540d\u5927\u4e8e 'd'\uff0c\u5219\u4ee3\u7801\u4e3a-c \" ds > 'd' \"\u3002 \u5426 -b/before <before days> \u53ea\u6709\u521b\u5efa\u65f6\u95f4\u8ddd\u79bb\u73b0\u5728\u8d85\u8fc7\u4e00\u5b9a\u5929\u6570\u7684\u8868\u6216\u5206\u533a\u624d\u4f1a\u88ab\u5f52\u6863\u3002 \u5426 -p/-parallel <parallelism> \u5f52\u6863\u64cd\u4f5c\u7684\u5e76\u884c\u5ea6\u3002 \u5426 -mr/-mapReduce \u4f7f\u7528Hadoop MapReduce\u800c\u975e\u672c\u5730\u591a\u7ebf\u7a0b\u6765\u5f52\u6863\u6570\u636e\u3002 \u5426 -e/-explain \u5982\u679c\u51fa\u73b0\u8be5\u9009\u9879\uff0c\u5219\u4e3a\u89e3\u91ca\uff08explain\uff09\u6a21\u5f0f\uff0c\u53ea\u4f1a\u663e\u793a\u5f85\u79fb\u52a8\u7684\u5206\u533a\u5217\u8868\uff0c\u800c\u4e0d\u4f1a\u771f\u6b63\u79fb\u52a8\u6570\u636e\u3002 \u5426 -w/-workingDir \u53ea\u5728MapReduce\u4f5c\u4e1a\u65f6\u4f7f\u7528\uff0c\u4e3aMapReduce\u4f5c\u4e1a\u7684\u5de5\u4f5c\u76ee\u5f55\u3002\u5fc5\u987b\u6709\u8bfb\u5199\u6743\u9650\uff0c\u5de5\u4f5c\u76ee\u5f55\u53ef\u4ee5\u975e\u7a7a\uff08\u4f5c\u4e1a\u6267\u884c\u8fc7\u7a0b\u4e2d\u4f1a\u521b\u5efa\u4e34\u65f6\u6587\u4ef6\uff0c\u6267\u884c\u5b8c\u6bd5\u4f1a\u6e05\u7406\u4e34\u65f6\u6587\u4ef6\uff09\u3002 \u5426 -l/-logDir <log directory> \u6307\u5b9aLog\u6587\u4ef6\u7684\u76ee\u5f55\u3002 \u5426","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"backup/oss/jindotable/jindotable_archive/#unarchivetable","text":"unarchiveTable \u547d\u4ee4\u4e0e archiveTable \u547d\u4ee4\u683c\u5f0f\u57fa\u672c\u4e00\u81f4\uff0c\u4f46\u6548\u679c\u76f8\u53cd\u3002unarchiveTable \u547d\u4ee4\u53ef\u4ee5\u5bf9 OSS \u4e0a\u7684\u8868\u6216\u5206\u533a\u8fdb\u884c\u89e3\u51bb\u3002","title":"unarchiveTable \u547d\u4ee4"},{"location":"backup/oss/jindotable/jindotable_archive/#_4","text":"\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help unarchiveTable","title":"\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f"},{"location":"backup/oss/jindotable/jindotable_archive/#_5","text":"jindotable -unarchiveTable -t <dbName.tableName> [-i/-a/-o/-cr] [-notWait] [-c \"<condition>\" | -fullTable] [-d/-restoreDays <restore days>] [-b/-before <before days>] [-p/-parallel <parallelism>] [-mr/-mapReduce] [-e/-explain] [-w/-workingDir <working directory>][-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u5f85\u89e3\u51bb\u7684\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a\u6570\u636e\u5e93\u540d.\u8868\u540d\u3002\u6570\u636e\u5e93\u548c\u8868\u540d\u4e4b\u95f4\u4ee5\u534a\u89d2\u53e5\u53f7\uff08.\uff09\u5206\u9694\u3002\u8868\u53ef\u4ee5\u662f\u5206\u533a\u8868\u6216\u975e\u5206\u533a\u8868\u3002 \u662f -i/-a/-o/-cr \u5982\u679c\u4e0d\u6307\u5b9a\u53c2\u6570\uff0c\u5219\u8f6c\u6362\u5b58\u50a8\u683c\u5f0f\u4e3a\u6807\u51c6\uff08Standard\uff09\u5b58\u50a8\u3002\u652f\u6301\u5982\u4e0b\u65b9\u5f0f\uff1a-i\uff1a\u5219\u8f6c\u6362\u5b58\u50a8\u683c\u5f0f\u4e3a\u4f4e\u9891\uff08Infrequent Access\uff0cIA\uff09\u5b58\u50a8\uff0c\u539f\u672c\u4e3a\u6807\u51c6\u5b58\u50a8\u7684\u6587\u4ef6\u88ab\u8df3\u8fc7\u3002-a\uff1a\u5219\u8f6c\u6362\u5b58\u50a8\u683c\u5f0f\u5f52\u6863\uff08Archive\uff09\u5b58\u50a8\u3002-o: \u4ec5\u505a\u89e3\u51bb\uff08Restore\uff09\u64cd\u4f5c\u3002\u539f\u672c\u4e3a\u6807\u51c6\u5b58\u50a8\u6216\u4f4e\u9891\u5b58\u50a8\u7684\u6587\u4ef6\u5747\u88ab\u8df3\u8fc7\u3002\u5df2\u7ecf\u5904\u4e8e\u89e3\u51bb\u72b6\u6001\u7684\u6587\u4ef6\u4e5f\u4f1a\u88ab\u8df3\u8fc7\uff0c\u5373\u4e0d\u4f1a\u91cd\u590d\u89e3\u51bb\u3002-cr\uff1a\u68c0\u67e5\u662f\u5426\u6240\u6709 restore \u4efb\u52a1\u5df2\u5b8c\u6210\u3002 \u5426 -notWait \u53ea\u5bf9 Restore \u64cd\u4f5c\u751f\u6548\u3002\u5982\u679c\u51fa\u73b0\u8be5\u9009\u9879\uff0c\u5219\u4e0d\u4f1a\u7b49 OSS \u670d\u52a1\u7aef\u5b8c\u6210 Restore \u64cd\u4f5c\u3002\u5426\u5219\uff0c\u4f1a\u7b49 Restore \u64cd\u4f5c\u5b8c\u6210\u6216\u8005\u8d85\u65f6\uff0810\u5206\u949f\uff09\u624d\u9000\u51fa \u5426 -c \"<condition>\" / -fullTable -fullTable \u548c -c \"<condition>\" \u53ea\u9700\u63d0\u4f9b\u4e00\u4e2a\uff0c\u5373\u8981\u4e48\u6307\u5b9a -c \"<condition>\" \uff0c\u8981\u4e48\u6307\u5b9a -fullTable \u3002\u6307\u5b9a -fullTable \u65f6\uff0c\u5219\u4e3a\u79fb\u52a8\u6574\u8868\uff0c\u65e2\u53ef\u4ee5\u662f\u975e\u5206\u533a\u8868\u4e5f\u53ef\u4ee5\u662f\u5206\u533a\u8868\u3002\u6307\u5b9a -c \"<condition>\" \u65f6\uff0c\u5219\u63d0\u4f9b\u4e86\u4e00\u4e2a\u8fc7\u6ee4\u6761\u4ef6\uff0c\u7528\u6765\u9009\u62e9\u5e0c\u671b\u79fb\u52a8\u7684\u5206\u533a\uff0c\u652f\u6301\u5e38\u89c1\u8fd0\u7b97\u7b26\uff0c\u4f8b\u5982\u5927\u4e8e\u53f7\uff08>\uff09\u3002\u4f8b\u5982\uff0c\u6570\u636e\u7c7b\u578b\u4e3aString\u7684\u5206\u533ads\uff0c\u5e0c\u671b\u5206\u533a\u540d\u5927\u4e8e 'd'\uff0c\u5219\u4ee3\u7801\u4e3a-c \" ds > 'd' \"\u3002 \u5426 -d/-restoreDays <restore days> \u5f53\u6d89\u53ca\u4ec5\u89e3\u51bb\uff08Restore\uff09\u64cd\u4f5c\u65f6\uff0c\u7ef4\u6301\u89e3\u51bb\u72b6\u6001\u7684\u5929\u6570\uff0c\u9ed8\u8ba4\u4e3a\u4e00\u5929\u3002\u4f8b\u5982\uff0c\u5bf9\u4e00\u4e2a\u51b7\u5f52\u6863\u7684\u8868\u505a\u4ec5\u89e3\u51bb\u64cd\u4f5c\u65f6\uff08-o \u53c2\u6570\uff09\uff0c\u8be5\u53c2\u6570\u5c06\u4ea7\u751f\u4f5c\u7528\uff1b\u5bf9\u4e00\u4e2a\u4ece\u672a\u89e3\u51bb\u8fc7\u7684\u5f52\u6863\u72b6\u6001\u7684\u8868\u8f6c\u4e3a\u6807\u51c6\u5b58\u50a8\u65f6\uff0c\u8be5\u53c2\u6570\u5c06\u5728\u4e2d\u95f4\u73af\u8282\u53d1\u6325\u4f5c\u7528\uff1b\u800c\u5bf9\u4e00\u4e2a\u4f4e\u9891\u72b6\u6001\u7684\u8868\u8f6c\u4e3a\u6807\u51c6\u5b58\u50a8\u65f6\uff0c\u8be5\u53c2\u6570\u4e0d\u4f1a\u4ea7\u751f\u4f5c\u7528\u3002 \u5426 -b/-before <before days> \u53ea\u6709\u521b\u5efa\u65f6\u95f4\u8ddd\u79bb\u73b0\u5728\u8d85\u8fc7\u4e00\u5b9a\u5929\u6570\u7684\u8868\u6216\u5206\u533a\u624d\u4f1a\u88ab\u89e3\u51bb\u3002 \u5426 -p/-parallel <parallelism> \u89e3\u51bb\u64cd\u4f5c\u7684\u5e76\u884c\u5ea6\u3002 \u5426 -mr/-mapReduce \u4f7f\u7528Hadoop MapReduce\u800c\u975e\u672c\u5730\u591a\u7ebf\u7a0b\u6765\u89e3\u51bb\u6570\u636e\u3002 \u5426 -e/-explain \u5982\u679c\u51fa\u73b0\u8be5\u9009\u9879\uff0c\u5219\u4e3a\u89e3\u91ca\uff08explain\uff09\u6a21\u5f0f\uff0c\u53ea\u4f1a\u663e\u793a\u5f85\u79fb\u52a8\u7684\u5206\u533a\u5217\u8868\uff0c\u800c\u4e0d\u4f1a\u771f\u6b63\u79fb\u52a8\u6570\u636e\u3002 \u5426 -w/-workingDir \u53ea\u5728MapReduce\u4f5c\u4e1a\u65f6\u4f7f\u7528\uff0c\u4e3aMapReduce\u4f5c\u4e1a\u7684\u5de5\u4f5c\u76ee\u5f55\u3002\u5fc5\u987b\u6709\u8bfb\u5199\u6743\u9650\uff0c\u5de5\u4f5c\u76ee\u5f55\u53ef\u4ee5\u975e\u7a7a\uff08\u4f5c\u4e1a\u6267\u884c\u8fc7\u7a0b\u4e2d\u4f1a\u521b\u5efa\u4e34\u65f6\u6587\u4ef6\uff0c\u6267\u884c\u5b8c\u6bd5\u4f1a\u6e05\u7406\u4e34\u65f6\u6587\u4ef6\uff09\u3002 \u5426 -l/-logDir <log directory> \u6307\u5b9aLog\u6587\u4ef6\u7684\u76ee\u5f55\u3002 \u5426","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"backup/oss/jindotable/jindotable_data_migration/","text":"JindoTable \u6570\u4ed3\u8fc1\u79fb \u00a7 JindoTable \u63d0\u4f9b\u4e86\u9ad8\u6548\u4fbf\u6377\u7684\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u8fc1\u79fb\uff0c\u90e8\u7f72 JindoSDK\uff0c\u60a8\u5c31\u53ef\u4ee5\u4f7f\u7528\u8fd9\u4e9b\u529f\u80fd\u3002 \u4f7f\u7528 JindoTable \u5c06 Hive \u6570\u636e\u5206\u533a\u5f52\u6863\u548c\u5206\u5c42\u5230 OSS","title":"JindoTable \u6570\u4ed3\u8fc1\u79fb"},{"location":"backup/oss/jindotable/jindotable_data_migration/#jindotable","text":"JindoTable \u63d0\u4f9b\u4e86\u9ad8\u6548\u4fbf\u6377\u7684\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u8fc1\u79fb\uff0c\u90e8\u7f72 JindoSDK\uff0c\u60a8\u5c31\u53ef\u4ee5\u4f7f\u7528\u8fd9\u4e9b\u529f\u80fd\u3002 \u4f7f\u7528 JindoTable \u5c06 Hive \u6570\u636e\u5206\u533a\u5f52\u6863\u548c\u5206\u5c42\u5230 OSS","title":"JindoTable \u6570\u4ed3\u8fc1\u79fb"},{"location":"backup/oss/jindotable/table_moveto/","text":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS \u00a7 HDFS \u53d7\u9650\u4e8e\u96c6\u7fa4\u89c4\u6a21\u548c\u6210\u672c\u56e0\u7d20\uff0c\u65e0\u6cd5\u65e0\u9650\u6269\u5c55\uff0c\u5bb9\u91cf\u5b58\u5728\u74f6\u9888\u3002\u4e91\u5382\u5546\u63d0\u4f9b\u7684\u5bf9\u8c61\u5b58\u50a8\u53ef\u4ee5\u4f5c\u4e3a HDFS \u7684\u66ff\u4ee3\u6216\u8865\u5145\uff0c\u6269\u5c55\u4e91\u4e0a Hadoop \u5e73\u53f0\u7684\u5b58\u50a8\u80fd\u529b\u3002 \u672c\u5de5\u5177\u7528\u4e8e\u628a Hive \u6570\u636e\u6839\u636e\u5206\u533a\u952e\u89c4\u5219\u7b5b\u9009\uff0c\u5728 HDFS \u548c OSS \u4e4b\u95f4\u8f6c\u79fb\u5206\u533a\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u90e8\u7f72 JindoSDK \u00a7 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 JindoSDK + \u963f\u91cc\u4e91 OSS \u5feb\u901f\u5165\u95e8 \u3002 \u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883 \u00a7 \u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e \u914d\u7f6e MoveTo \u5de5\u5177\u5728 HDFS \u4e0b\u7684\u9501\u76ee\u5f55\uff1a \u00a7 \u5728 Hadoop \u914d\u7f6e\u6587\u4ef6 core-site.xml \u6216 hdfs-site.xml \uff08\u4efb\u4e00\u5373\u53ef\uff0c\u5728 $HADOOP_CONF_DIR \u76ee\u5f55\u4e0b\uff09\u65b0\u589e\u914d\u7f6e\u9879\uff1a jindotable.moveto.tablelock.base.dir \u8be5\u914d\u7f6e\u7684\u503c\u5e94\u6307\u5411\u4e00\u4e2a HDFS \u76ee\u5f55\uff0c\u76ee\u7684\u662f\u5b58\u653e MoveTo \u5de5\u5177\u5728\u8fd0\u884c\u65f6\u81ea\u52a8\u521b\u5efa\u7684\u9501\u6587\u4ef6\u3002\u9700\u786e\u4fdd\u8be5\u76ee\u5f55\u53ea\u4f1a\u88ab MoveTo \u5de5\u5177\u8bbf\u95ee\uff0c\u5e76\u4e14\u6709\u8bbf\u95ee\u6743\u9650\u3002\u5982\u679c\u4e0d\u914d\u7f6e\uff0c\u5219\u4f7f\u7528\u7f3a\u7701\u503c hdfs:///tmp/jindotable-lock/ \uff0c\u65e0\u6743\u9650\u5219\u62a5\u9519\u3002 \u4f7f\u7528\u8bf4\u660e \u00a7 \u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f \u00a7 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help moveTo \u53c2\u6570\u8bf4\u660e \u00a7 jindotable -moveTo -t <dbName.tableName> -d <destination path> [-c \"<condition>\" | -fullTable] [-b/-before <before days>] [-p/-parallel <parallelism>] [-s/-storagePolicy <OSS storage policy>] [-o/-overWrite] [-r/-removeSource] [-skipTrash] [-e/-explain] [-q/-queue <yarn queue>] [-w/-workingDir <working directory>][-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u8981\u79fb\u52a8\u7684\u8868\u3002 \u662f -d <destination path> \u76ee\u6807\u8def\u5f84\uff0c\u4e3a\u8868\u7ea7\u522b\u7684\u8def\u5f84\uff0c\u5206\u533a\u8def\u5f84\u4f1a\u5728\u8fd9\u4e2a\u8def\u5f84\u4e0b\u81ea\u52a8\u521b\u5efa\u3002 \u662f -c \"<condition>\" / -fullTable \u5206\u533a\u8fc7\u6ee4\u6761\u4ef6\u8868\u8fbe\u5f0f\uff0c\u652f\u6301\u57fa\u672c\u8fd0\u7b97\u7b26\uff0c\u4e0d\u652f\u6301udf\u3002 \u5426 -b/before <before days> \u6839\u636e\u5206\u533a\u521b\u5efa\u65f6\u95f4\uff0c\u521b\u5efa\u65f6\u95f4\u8d85\u8fc7\u7ed9\u5b9a\u5929\u6570\u7684\u5206\u533a\u624d\u8fdb\u884c\u79fb\u52a8\u3002 \u5426 -p/-parallel <parallelism> \u6574\u4e2amoveTo\u4efb\u52a1\u7684\u6700\u5927task\u5e76\u53d1\u5ea6\uff0c\u9ed8\u8ba4\u4e3a1\u3002 \u5426 -s/-storagePolicy <OSS storage policy> \u62f7\u8d1d\u81f3 OSS \u65f6\u6570\u636e\u6587\u4ef6\u7684\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301 Standard\uff08\u9ed8\u8ba4\uff09\u3001IA\u3001Archive\u3001ColdArchive\u3002 \u5426 -o/-overWrite \u662f\u5426\u8986\u76d6\u6700\u7ec8\u76ee\u5f55\u3002\u5206\u533a\u7ea7\u522b\u8986\u76d6\uff0c\u4e0d\u4f1a\u8986\u76d6\u672c\u6b21\u79fb\u52a8\u4e0d\u6d89\u53ca\u7684\u5206\u533a\u3002 \u5426 -r/-removeSource \u662f\u5426\u5728\u79fb\u52a8\u5b8c\u6210\u540e\u5220\u9664\u6e90\u8def\u5f84\u3002 \u5426 -skipTrash \u5982\u679c\u5220\u9664\u6e90\u8def\u5f84\uff0c\u662f\u5426\u8df3\u8fc7Trash\u3002 \u5426 -e/-explain \u5982\u679c\u6307\u5b9aexplain\u6a21\u5f0f\uff0c\u4e0d\u4f1a\u89e6\u53d1\u5b9e\u9645\u64cd\u4f5c\uff0c\u4ec5\u6253\u5370\u4f1a\u540c\u6b65\u7684\u5206\u533a\u3002 \u5426 -q/-queue <yarn queue \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684yarn\u961f\u5217\u3002 \u5426 -w/-workingDir \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684\u5de5\u4f5c\u4e34\u65f6\u76ee\u5f55\u3002 \u5426 -l/-logDir <log directory> \u672c\u5730\u65e5\u5fd7\u76ee\u5f55\uff0c\u9ed8\u8ba4\u4e3a /tmp/<current user>/ \u5426 \u4f7f\u7528\u793a\u4f8b \u00a7 \u6709\u4e00\u4e2a HDFS \u4e0a\u7684 Hive \u5206\u533a\u8868\uff0c\u5982\u4e0b\u56fe\u6240\u793a\uff1a \u60f3\u628a bbb \u548c ccc \u5206\u533a\u79fb\u52a8\u5230 OSS\uff0c\u5148\u7528 explain \u6a21\u5f0f\u770b\u770b\u4f1a\u79fb\u52a8\u7684\u5206\u533a\u662f\u5426\u7b26\u5408\u9884\u671f\uff0c\u53c2\u6570\u4e3a -e \u6216 -explain\uff1a \u53bb\u6389\u53c2\u6570 -e\uff0c\u771f\u6b63\u79fb\u52a8\u5206\u533a\uff1a \u6267\u884c\u5b8c\u6210\u540e\uff0c\u68c0\u67e5\u6570\u636e\uff0c\u6570\u636e\u5df2\u7ecf\u5728 OSS\uff1a \u518d\u79fb\u56de HDFS\uff0c\u7ed3\u679c\u5931\u8d25\u4e86\uff0c\u4f1a\u663e\u793a\u5931\u8d25\u539f\u56e0\uff0c\u539f\u6765\u662f\u76ee\u6807\u76ee\u5f55\u975e\u7a7a\uff0c\u63d0\u793a\u53ef\u4ee5\u4f7f\u7528 -overWrite \u6e05\u7a7a\u76ee\u6807\u76ee\u5f55\uff1a \u4f7f\u7528 -overWrite \u5f3a\u5236\u8986\u76d6\uff1a \u5f02\u5e38\u72b6\u6001\u5904\u7406 \u00a7 \u4e3a\u4e86\u5c3d\u53ef\u80fd\u4fdd\u8bc1\u6570\u636e\u5b89\u5168\uff0c\u9632\u6b62\u6c61\u67d3\uff0c\u8be5\u547d\u4ee4\u4f1a\u81ea\u52a8\u68c0\u67e5\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u786e\u4fdd\u4e0d\u5b58\u5728\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u76ee\u5f55\u62f7\u8d1d\u3002 \u5982\u679c\u65e0\u6cd5\u6392\u9664\uff0c\u5219\u8be5\u8868\u6216\u5206\u533a\u7684\u62f7\u8d1d\u547d\u4ee4\u5c06\u6267\u884c\u5931\u8d25\u3002 \u6b64\u65f6\uff0c\u9700\u8981\u4e3b\u52a8\u5e72\u9884\uff0c\u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u6b63\u5728\u6267\u884c\u62f7\u8d1d\u4e4b\u540e\uff0c\u6e05\u7406\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u5219\u53ef\u4ee5\u91cd\u65b0\u6267\u884c\u8be5\u547d\u4ee4\u3002 \u5bf9\u4e8e\u975e\u5206\u533a\u8868\u7684\u62f7\u8d1d\u6216\u8fc1\u79fb\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u5373\u4e3a\u8868\u7ea7\u522b\u76ee\u5f55\uff1b\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u4e3a\u5f85\u62f7\u8d1d\u6216\u79fb\u52a8\u5206\u533a\u7684\u5206\u533a\u7ea7\u76ee\u5f55\uff0c \u53ea\u9700\u5bf9\u5f85\u62f7\u8d1d\u7684\u5206\u533a\u8fdb\u884c\u6e05\u7406\u3002 \u51fa\u73b0\u539f\u56e0 \u00a7 \u5982\u679c\u8be5\u547d\u4ee4\u88ab\u5f02\u5e38\u4e2d\u6b62\uff0c\u5219\u53ef\u80fd\u51fa\u73b0\u9700\u8981\u4e3b\u52a8\u5e72\u9884\u7684\u60c5\u51b5\u3002\u6b64\u65f6\u62f7\u8d1d\u8fd8\u672a\u5b8c\u6210\uff0c\u6e90\u6570\u636e\u4e0e\u8868\u7684\u5143\u4fe1\u606f\u5747\u672a\u6539\u53d8\uff0c \u6570\u636e\u4ecd\u5904\u4e8e\u5b89\u5168\u72b6\u6001\u3002\u5e38\u89c1\u7684\u5f02\u5e38\u4e2d\u6b62\u6709\u4e24\u79cd\u60c5\u51b5\uff1a * \u7528\u6237\u5728\u547d\u4ee4\u5c1a\u672a\u7ed3\u675f\u65f6\uff0c\u4e3b\u52a8\u6740\u6389\u547d\u4ee4\u8fdb\u7a0b * \u7531\u4e8e\u5185\u5b58\u6ea2\u51fa\u7b49\u5f02\u5e38\uff0c\u8fdb\u7a0b\u5f02\u5e38\u4e2d\u6b62 \u5982\u679c\u62f7\u8d1d\u8868\u6216\u5206\u533a\u65f6\u62f7\u8d1d\u5931\u8d25\u5e76\u663e\u793a Conflicts found \uff0c\u5219\u5c5e\u4e8e\u5f02\u5e38\u72b6\u6001\uff0c\u9700\u4e3b\u52a8\u5e72\u9884\u3002 \u5904\u7406\u65b9\u6cd5 \u00a7 \u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u7684\u76ee\u6807\u8def\u5f84\u62f7\u8d1d\u6570\u636e\u3002\u7279\u522b\u662f\uff0c\u786e\u4fdd\u6ca1\u6709 DistCp/JindoDistCp \u7b49\u5206\u5e03\u5f0f\u62f7\u8d1d\u547d\u4ee4\u3002 \u5220\u9664\u76ee\u7684\u5730\u76ee\u5f55\u3002\u5bf9\u4e8e\u975e\u5206\u533a\u8868\uff0c\u5220\u9664\u8868\u4e00\u7ea7\u76ee\u5f55\u3002\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u5220\u9664\u5b58\u5728\u51b2\u7a81\u7684\u5206\u533a\u7ea7\u76ee\u5f55\u3002\u5207\u52ff\u5220\u9664\u6e90\u76ee\u5f55\u3002","title":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS"},{"location":"backup/oss/jindotable/table_moveto/#jindotable-hive-oss","text":"HDFS \u53d7\u9650\u4e8e\u96c6\u7fa4\u89c4\u6a21\u548c\u6210\u672c\u56e0\u7d20\uff0c\u65e0\u6cd5\u65e0\u9650\u6269\u5c55\uff0c\u5bb9\u91cf\u5b58\u5728\u74f6\u9888\u3002\u4e91\u5382\u5546\u63d0\u4f9b\u7684\u5bf9\u8c61\u5b58\u50a8\u53ef\u4ee5\u4f5c\u4e3a HDFS \u7684\u66ff\u4ee3\u6216\u8865\u5145\uff0c\u6269\u5c55\u4e91\u4e0a Hadoop \u5e73\u53f0\u7684\u5b58\u50a8\u80fd\u529b\u3002 \u672c\u5de5\u5177\u7528\u4e8e\u628a Hive \u6570\u636e\u6839\u636e\u5206\u533a\u952e\u89c4\u5219\u7b5b\u9009\uff0c\u5728 HDFS \u548c OSS \u4e4b\u95f4\u8f6c\u79fb\u5206\u533a\u3002","title":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS"},{"location":"backup/oss/jindotable/table_moveto/#_1","text":"","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/oss/jindotable/table_moveto/#jindosdk","text":"\u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 JindoSDK + \u963f\u91cc\u4e91 OSS \u5feb\u901f\u5165\u95e8 \u3002","title":"\u5df2\u90e8\u7f72 JindoSDK"},{"location":"backup/oss/jindotable/table_moveto/#hadoop-hive","text":"\u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e","title":"\u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883"},{"location":"backup/oss/jindotable/table_moveto/#moveto-hdfs","text":"\u5728 Hadoop \u914d\u7f6e\u6587\u4ef6 core-site.xml \u6216 hdfs-site.xml \uff08\u4efb\u4e00\u5373\u53ef\uff0c\u5728 $HADOOP_CONF_DIR \u76ee\u5f55\u4e0b\uff09\u65b0\u589e\u914d\u7f6e\u9879\uff1a jindotable.moveto.tablelock.base.dir \u8be5\u914d\u7f6e\u7684\u503c\u5e94\u6307\u5411\u4e00\u4e2a HDFS \u76ee\u5f55\uff0c\u76ee\u7684\u662f\u5b58\u653e MoveTo \u5de5\u5177\u5728\u8fd0\u884c\u65f6\u81ea\u52a8\u521b\u5efa\u7684\u9501\u6587\u4ef6\u3002\u9700\u786e\u4fdd\u8be5\u76ee\u5f55\u53ea\u4f1a\u88ab MoveTo \u5de5\u5177\u8bbf\u95ee\uff0c\u5e76\u4e14\u6709\u8bbf\u95ee\u6743\u9650\u3002\u5982\u679c\u4e0d\u914d\u7f6e\uff0c\u5219\u4f7f\u7528\u7f3a\u7701\u503c hdfs:///tmp/jindotable-lock/ \uff0c\u65e0\u6743\u9650\u5219\u62a5\u9519\u3002","title":"\u914d\u7f6e MoveTo \u5de5\u5177\u5728 HDFS \u4e0b\u7684\u9501\u76ee\u5f55\uff1a"},{"location":"backup/oss/jindotable/table_moveto/#_2","text":"","title":"\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/oss/jindotable/table_moveto/#_3","text":"\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help moveTo","title":"\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f"},{"location":"backup/oss/jindotable/table_moveto/#_4","text":"jindotable -moveTo -t <dbName.tableName> -d <destination path> [-c \"<condition>\" | -fullTable] [-b/-before <before days>] [-p/-parallel <parallelism>] [-s/-storagePolicy <OSS storage policy>] [-o/-overWrite] [-r/-removeSource] [-skipTrash] [-e/-explain] [-q/-queue <yarn queue>] [-w/-workingDir <working directory>][-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u8981\u79fb\u52a8\u7684\u8868\u3002 \u662f -d <destination path> \u76ee\u6807\u8def\u5f84\uff0c\u4e3a\u8868\u7ea7\u522b\u7684\u8def\u5f84\uff0c\u5206\u533a\u8def\u5f84\u4f1a\u5728\u8fd9\u4e2a\u8def\u5f84\u4e0b\u81ea\u52a8\u521b\u5efa\u3002 \u662f -c \"<condition>\" / -fullTable \u5206\u533a\u8fc7\u6ee4\u6761\u4ef6\u8868\u8fbe\u5f0f\uff0c\u652f\u6301\u57fa\u672c\u8fd0\u7b97\u7b26\uff0c\u4e0d\u652f\u6301udf\u3002 \u5426 -b/before <before days> \u6839\u636e\u5206\u533a\u521b\u5efa\u65f6\u95f4\uff0c\u521b\u5efa\u65f6\u95f4\u8d85\u8fc7\u7ed9\u5b9a\u5929\u6570\u7684\u5206\u533a\u624d\u8fdb\u884c\u79fb\u52a8\u3002 \u5426 -p/-parallel <parallelism> \u6574\u4e2amoveTo\u4efb\u52a1\u7684\u6700\u5927task\u5e76\u53d1\u5ea6\uff0c\u9ed8\u8ba4\u4e3a1\u3002 \u5426 -s/-storagePolicy <OSS storage policy> \u62f7\u8d1d\u81f3 OSS \u65f6\u6570\u636e\u6587\u4ef6\u7684\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301 Standard\uff08\u9ed8\u8ba4\uff09\u3001IA\u3001Archive\u3001ColdArchive\u3002 \u5426 -o/-overWrite \u662f\u5426\u8986\u76d6\u6700\u7ec8\u76ee\u5f55\u3002\u5206\u533a\u7ea7\u522b\u8986\u76d6\uff0c\u4e0d\u4f1a\u8986\u76d6\u672c\u6b21\u79fb\u52a8\u4e0d\u6d89\u53ca\u7684\u5206\u533a\u3002 \u5426 -r/-removeSource \u662f\u5426\u5728\u79fb\u52a8\u5b8c\u6210\u540e\u5220\u9664\u6e90\u8def\u5f84\u3002 \u5426 -skipTrash \u5982\u679c\u5220\u9664\u6e90\u8def\u5f84\uff0c\u662f\u5426\u8df3\u8fc7Trash\u3002 \u5426 -e/-explain \u5982\u679c\u6307\u5b9aexplain\u6a21\u5f0f\uff0c\u4e0d\u4f1a\u89e6\u53d1\u5b9e\u9645\u64cd\u4f5c\uff0c\u4ec5\u6253\u5370\u4f1a\u540c\u6b65\u7684\u5206\u533a\u3002 \u5426 -q/-queue <yarn queue \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684yarn\u961f\u5217\u3002 \u5426 -w/-workingDir \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684\u5de5\u4f5c\u4e34\u65f6\u76ee\u5f55\u3002 \u5426 -l/-logDir <log directory> \u672c\u5730\u65e5\u5fd7\u76ee\u5f55\uff0c\u9ed8\u8ba4\u4e3a /tmp/<current user>/ \u5426","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"backup/oss/jindotable/table_moveto/#_5","text":"\u6709\u4e00\u4e2a HDFS \u4e0a\u7684 Hive \u5206\u533a\u8868\uff0c\u5982\u4e0b\u56fe\u6240\u793a\uff1a \u60f3\u628a bbb \u548c ccc \u5206\u533a\u79fb\u52a8\u5230 OSS\uff0c\u5148\u7528 explain \u6a21\u5f0f\u770b\u770b\u4f1a\u79fb\u52a8\u7684\u5206\u533a\u662f\u5426\u7b26\u5408\u9884\u671f\uff0c\u53c2\u6570\u4e3a -e \u6216 -explain\uff1a \u53bb\u6389\u53c2\u6570 -e\uff0c\u771f\u6b63\u79fb\u52a8\u5206\u533a\uff1a \u6267\u884c\u5b8c\u6210\u540e\uff0c\u68c0\u67e5\u6570\u636e\uff0c\u6570\u636e\u5df2\u7ecf\u5728 OSS\uff1a \u518d\u79fb\u56de HDFS\uff0c\u7ed3\u679c\u5931\u8d25\u4e86\uff0c\u4f1a\u663e\u793a\u5931\u8d25\u539f\u56e0\uff0c\u539f\u6765\u662f\u76ee\u6807\u76ee\u5f55\u975e\u7a7a\uff0c\u63d0\u793a\u53ef\u4ee5\u4f7f\u7528 -overWrite \u6e05\u7a7a\u76ee\u6807\u76ee\u5f55\uff1a \u4f7f\u7528 -overWrite \u5f3a\u5236\u8986\u76d6\uff1a","title":"\u4f7f\u7528\u793a\u4f8b"},{"location":"backup/oss/jindotable/table_moveto/#_6","text":"\u4e3a\u4e86\u5c3d\u53ef\u80fd\u4fdd\u8bc1\u6570\u636e\u5b89\u5168\uff0c\u9632\u6b62\u6c61\u67d3\uff0c\u8be5\u547d\u4ee4\u4f1a\u81ea\u52a8\u68c0\u67e5\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u786e\u4fdd\u4e0d\u5b58\u5728\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u76ee\u5f55\u62f7\u8d1d\u3002 \u5982\u679c\u65e0\u6cd5\u6392\u9664\uff0c\u5219\u8be5\u8868\u6216\u5206\u533a\u7684\u62f7\u8d1d\u547d\u4ee4\u5c06\u6267\u884c\u5931\u8d25\u3002 \u6b64\u65f6\uff0c\u9700\u8981\u4e3b\u52a8\u5e72\u9884\uff0c\u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u6b63\u5728\u6267\u884c\u62f7\u8d1d\u4e4b\u540e\uff0c\u6e05\u7406\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u5219\u53ef\u4ee5\u91cd\u65b0\u6267\u884c\u8be5\u547d\u4ee4\u3002 \u5bf9\u4e8e\u975e\u5206\u533a\u8868\u7684\u62f7\u8d1d\u6216\u8fc1\u79fb\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u5373\u4e3a\u8868\u7ea7\u522b\u76ee\u5f55\uff1b\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u4e3a\u5f85\u62f7\u8d1d\u6216\u79fb\u52a8\u5206\u533a\u7684\u5206\u533a\u7ea7\u76ee\u5f55\uff0c \u53ea\u9700\u5bf9\u5f85\u62f7\u8d1d\u7684\u5206\u533a\u8fdb\u884c\u6e05\u7406\u3002","title":"\u5f02\u5e38\u72b6\u6001\u5904\u7406"},{"location":"backup/oss/jindotable/table_moveto/#_7","text":"\u5982\u679c\u8be5\u547d\u4ee4\u88ab\u5f02\u5e38\u4e2d\u6b62\uff0c\u5219\u53ef\u80fd\u51fa\u73b0\u9700\u8981\u4e3b\u52a8\u5e72\u9884\u7684\u60c5\u51b5\u3002\u6b64\u65f6\u62f7\u8d1d\u8fd8\u672a\u5b8c\u6210\uff0c\u6e90\u6570\u636e\u4e0e\u8868\u7684\u5143\u4fe1\u606f\u5747\u672a\u6539\u53d8\uff0c \u6570\u636e\u4ecd\u5904\u4e8e\u5b89\u5168\u72b6\u6001\u3002\u5e38\u89c1\u7684\u5f02\u5e38\u4e2d\u6b62\u6709\u4e24\u79cd\u60c5\u51b5\uff1a * \u7528\u6237\u5728\u547d\u4ee4\u5c1a\u672a\u7ed3\u675f\u65f6\uff0c\u4e3b\u52a8\u6740\u6389\u547d\u4ee4\u8fdb\u7a0b * \u7531\u4e8e\u5185\u5b58\u6ea2\u51fa\u7b49\u5f02\u5e38\uff0c\u8fdb\u7a0b\u5f02\u5e38\u4e2d\u6b62 \u5982\u679c\u62f7\u8d1d\u8868\u6216\u5206\u533a\u65f6\u62f7\u8d1d\u5931\u8d25\u5e76\u663e\u793a Conflicts found \uff0c\u5219\u5c5e\u4e8e\u5f02\u5e38\u72b6\u6001\uff0c\u9700\u4e3b\u52a8\u5e72\u9884\u3002","title":"\u51fa\u73b0\u539f\u56e0"},{"location":"backup/oss/jindotable/table_moveto/#_8","text":"\u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u7684\u76ee\u6807\u8def\u5f84\u62f7\u8d1d\u6570\u636e\u3002\u7279\u522b\u662f\uff0c\u786e\u4fdd\u6ca1\u6709 DistCp/JindoDistCp \u7b49\u5206\u5e03\u5f0f\u62f7\u8d1d\u547d\u4ee4\u3002 \u5220\u9664\u76ee\u7684\u5730\u76ee\u5f55\u3002\u5bf9\u4e8e\u975e\u5206\u533a\u8868\uff0c\u5220\u9664\u8868\u4e00\u7ea7\u76ee\u5f55\u3002\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u5220\u9664\u5b58\u5728\u51b2\u7a81\u7684\u5206\u533a\u7ea7\u76ee\u5f55\u3002\u5207\u52ff\u5220\u9664\u6e90\u76ee\u5f55\u3002","title":"\u5904\u7406\u65b9\u6cd5"},{"location":"backup/oss/jindotable/table_shell/","text":"\u4f7f\u7528 JindoTable \u547d\u4ee4\u5b9e\u73b0 Hive \u8868\u548c\u5206\u533a\u7684\u6570\u636e\u6cbb\u7406 \u00a7 JindoTable \u63d0\u4f9b\u591a\u4e2a\u547d\u4ee4\u884c\u547d\u4ee4\uff0c\u53ef\u4ee5\u4fbf\u6377\u67e5\u8be2 Hive \u8868\u548c\u5206\u533a\u7684\u57fa\u672c\u4fe1\u606f\u6216\u7a7a\u95f4\u5360\u7528\uff0c \u6216\u8005\u5b9e\u73b0\u6570\u636e\u7684\u8fc1\u79fb\u3001\u5f52\u6863\u3002\u672c\u6587\u4ecb\u7ecd\u8fd9\u4e9b\u547d\u4ee4\u884c\u547d\u4ee4\u7684\u7528\u6cd5\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u90e8\u7f72 JindoSDK \u00a7 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8 \u3002 \u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883 \u00a7 \u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e \u4f7f\u7528\u8bf4\u660e \u00a7 \u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f \u00a7 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u67e5\u770b JindoTable \u652f\u6301\u7684\u6240\u6709\u547d\u4ee4\uff1a jindotable -help \u5982\u679c\u60f3\u67e5\u770b\u547d\u4ee4\u7684\u5177\u4f53\u7528\u6cd5\uff0c\u6267\u884c\uff1a jindotable -help <cmd> \u5176\u4e2d\uff0c <cmd> \u53ef\u4ee5\u662f showTable \u3001 moveTo \u3001 archiveTable \u7b49\u3002 showTable \u547d\u4ee4 \u00a7 jindotable -showTable -t <dbName.tableName> \u5176\u4e2d <dbName.tableName> \u4e3a\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a \u5e93\u540d.\u8868\u540d \u529f\u80fd\uff1a\u5982\u679c\u662f\u5206\u533a\u8868\uff0c\u5219\u5c55\u793a\u6240\u6709\u5206\u533a\uff1b\u5982\u679c\u662f\u975e\u5206\u533a\u8868\uff0c\u5219\u8fd4\u56de\u8868\u7684\u5b58\u50a8\u60c5\u51b5\u3002 \u793a\u4f8b\uff1a\u5c55\u793a db1.t1 \u5206\u533a\u8868\u7684\u6240\u6709\u5206\u533a\u3002 jindo table -showTable -t db1.t1 showPartition \u547d\u4ee4 \u00a7 jindotable -showPartition -t <dbName.tableName> -p <partitionSpec> \u5176\u4e2d <dbName.tableName> \u4e3a\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a \u5e93\u540d.\u8868\u540d \uff1b <partitionSpec> \u4e3a\u5206\u533a\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a col1=v1,col2=v2 \u529f\u80fd\uff1a\u8fd4\u56de\u5206\u533a\u7684\u5b58\u50a8\u60c5\u51b5\u3002 \u793a\u4f8b\uff1a\u8fd4\u56de\u5206\u533a\u8868 db1.t1 \u5728 2020-10-12 \u65e5\u7684\u5b58\u50a8\u60c5\u51b5\u3002 jindo table -showPartition -t db1.t1 -p date=2020-10-12 listTables \u547d\u4ee4 \u00a7 jindotable -listTables [-db] <dbName> \u5176\u4e2d <dbName> \u4e3a\u6570\u636e\u5e93\u540d\u79f0\uff0c\u53ef\u7701\u7565\uff0c\u7701\u7565\u5219\u4e3a\u9ed8\u8ba4\u6570\u636e\u5e93\u3002 \u529f\u80fd\uff1a\u5c55\u793a\u6307\u5b9a\u6570\u636e\u5e93\u4e2d\u7684\u6240\u6709\u8868\u3002\u4e0d\u6307\u5b9a -db \u65f6\u9ed8\u8ba4\u5c55\u793a default \u5e93\u4e2d\u7684\u8868\u3002 \u793a\u4f8b\uff1a\u5217\u51fa\u6570\u636e\u5e93 db1 \u4e2d\u7684\u8868\u3002 jindo table -listTables -db db1 moveTo \u547d\u4ee4 \u00a7 \u8bf7\u53c2\u8003 \u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1 archiveTable/unarchiveTable \u547d\u4ee4 \u00a7 \u8bf7\u53c2\u8003 JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a cacheTable/uncacheTable \u547d\u4ee4 \u00a7 \u4ec5\u652f\u6301\u914d\u5907 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7684\u573a\u666f\uff0c\u8bf7\u53c2\u8003 \u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e","title":"\u4f7f\u7528 JindoTable \u547d\u4ee4\u5b9e\u73b0 Hive \u8868\u548c\u5206\u533a\u7684\u6570\u636e\u6cbb\u7406"},{"location":"backup/oss/jindotable/table_shell/#jindotable-hive","text":"JindoTable \u63d0\u4f9b\u591a\u4e2a\u547d\u4ee4\u884c\u547d\u4ee4\uff0c\u53ef\u4ee5\u4fbf\u6377\u67e5\u8be2 Hive \u8868\u548c\u5206\u533a\u7684\u57fa\u672c\u4fe1\u606f\u6216\u7a7a\u95f4\u5360\u7528\uff0c \u6216\u8005\u5b9e\u73b0\u6570\u636e\u7684\u8fc1\u79fb\u3001\u5f52\u6863\u3002\u672c\u6587\u4ecb\u7ecd\u8fd9\u4e9b\u547d\u4ee4\u884c\u547d\u4ee4\u7684\u7528\u6cd5\u3002","title":"\u4f7f\u7528 JindoTable \u547d\u4ee4\u5b9e\u73b0 Hive \u8868\u548c\u5206\u533a\u7684\u6570\u636e\u6cbb\u7406"},{"location":"backup/oss/jindotable/table_shell/#_1","text":"","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/oss/jindotable/table_shell/#jindosdk","text":"\u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8 \u3002","title":"\u5df2\u90e8\u7f72 JindoSDK"},{"location":"backup/oss/jindotable/table_shell/#hadoop-hive","text":"\u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e","title":"\u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883"},{"location":"backup/oss/jindotable/table_shell/#_2","text":"","title":"\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/oss/jindotable/table_shell/#_3","text":"\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u67e5\u770b JindoTable \u652f\u6301\u7684\u6240\u6709\u547d\u4ee4\uff1a jindotable -help \u5982\u679c\u60f3\u67e5\u770b\u547d\u4ee4\u7684\u5177\u4f53\u7528\u6cd5\uff0c\u6267\u884c\uff1a jindotable -help <cmd> \u5176\u4e2d\uff0c <cmd> \u53ef\u4ee5\u662f showTable \u3001 moveTo \u3001 archiveTable \u7b49\u3002","title":"\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f"},{"location":"backup/oss/jindotable/table_shell/#showtable","text":"jindotable -showTable -t <dbName.tableName> \u5176\u4e2d <dbName.tableName> \u4e3a\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a \u5e93\u540d.\u8868\u540d \u529f\u80fd\uff1a\u5982\u679c\u662f\u5206\u533a\u8868\uff0c\u5219\u5c55\u793a\u6240\u6709\u5206\u533a\uff1b\u5982\u679c\u662f\u975e\u5206\u533a\u8868\uff0c\u5219\u8fd4\u56de\u8868\u7684\u5b58\u50a8\u60c5\u51b5\u3002 \u793a\u4f8b\uff1a\u5c55\u793a db1.t1 \u5206\u533a\u8868\u7684\u6240\u6709\u5206\u533a\u3002 jindo table -showTable -t db1.t1","title":"showTable \u547d\u4ee4"},{"location":"backup/oss/jindotable/table_shell/#showpartition","text":"jindotable -showPartition -t <dbName.tableName> -p <partitionSpec> \u5176\u4e2d <dbName.tableName> \u4e3a\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a \u5e93\u540d.\u8868\u540d \uff1b <partitionSpec> \u4e3a\u5206\u533a\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a col1=v1,col2=v2 \u529f\u80fd\uff1a\u8fd4\u56de\u5206\u533a\u7684\u5b58\u50a8\u60c5\u51b5\u3002 \u793a\u4f8b\uff1a\u8fd4\u56de\u5206\u533a\u8868 db1.t1 \u5728 2020-10-12 \u65e5\u7684\u5b58\u50a8\u60c5\u51b5\u3002 jindo table -showPartition -t db1.t1 -p date=2020-10-12","title":"showPartition \u547d\u4ee4"},{"location":"backup/oss/jindotable/table_shell/#listtables","text":"jindotable -listTables [-db] <dbName> \u5176\u4e2d <dbName> \u4e3a\u6570\u636e\u5e93\u540d\u79f0\uff0c\u53ef\u7701\u7565\uff0c\u7701\u7565\u5219\u4e3a\u9ed8\u8ba4\u6570\u636e\u5e93\u3002 \u529f\u80fd\uff1a\u5c55\u793a\u6307\u5b9a\u6570\u636e\u5e93\u4e2d\u7684\u6240\u6709\u8868\u3002\u4e0d\u6307\u5b9a -db \u65f6\u9ed8\u8ba4\u5c55\u793a default \u5e93\u4e2d\u7684\u8868\u3002 \u793a\u4f8b\uff1a\u5217\u51fa\u6570\u636e\u5e93 db1 \u4e2d\u7684\u8868\u3002 jindo table -listTables -db db1","title":"listTables \u547d\u4ee4"},{"location":"backup/oss/jindotable/table_shell/#moveto","text":"\u8bf7\u53c2\u8003 \u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1","title":"moveTo \u547d\u4ee4"},{"location":"backup/oss/jindotable/table_shell/#archivetableunarchivetable","text":"\u8bf7\u53c2\u8003 JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a","title":"archiveTable/unarchiveTable \u547d\u4ee4"},{"location":"backup/oss/jindotable/table_shell/#cachetableuncachetable","text":"\u4ec5\u652f\u6301\u914d\u5907 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7684\u573a\u666f\uff0c\u8bf7\u53c2\u8003 \u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e","title":"cacheTable/uncacheTable \u547d\u4ee4"},{"location":"backup/oss/presto/jindosdk_on_presto/","text":"Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS \u4e0a\u7684\u6570\u636e \u00a7 Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002 \u6b65\u9aa4 \u00a7 1. \u5b89\u88c5 jar \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/ 2. \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> 3. \u914d\u7f6e OSS Access Key \u00a7 \u5c06 OSS \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 4. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 \u4f7f\u7528\u793a\u4f8b \u00a7 \u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='oss://<bucket>/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS \u4e0a\u7684\u6570\u636e"},{"location":"backup/oss/presto/jindosdk_on_presto/#presto-jindosdk-oss","text":"Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002","title":"Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS \u4e0a\u7684\u6570\u636e"},{"location":"backup/oss/presto/jindosdk_on_presto/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/oss/presto/jindosdk_on_presto/#1-jar","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/","title":"1. \u5b89\u88c5 jar \u5305"},{"location":"backup/oss/presto/jindosdk_on_presto/#2-jindosdk-oss","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration>","title":"2. \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b"},{"location":"backup/oss/presto/jindosdk_on_presto/#3-oss-access-key","text":"\u5c06 OSS \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002","title":"3. \u914d\u7f6e OSS Access Key"},{"location":"backup/oss/presto/jindosdk_on_presto/#4-presto","text":"","title":"4. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"backup/oss/presto/jindosdk_on_presto/#_2","text":"\u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='oss://<bucket>/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"\u4f7f\u7528\u793a\u4f8b"},{"location":"backup/oss/security/jindosdk_credential_provider_bucket_oss/","text":"JindoSDK OSS Credential Provider \u6309 OSS bucket \u914d\u7f6e\u4f7f\u7528\u8bf4\u660e \u00a7 \u6309\u7167 bucket \u914d\u7f6e Credential Provider \u00a7 <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleAliyunCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.AliyunCredentialsProvider\u7684\u5b9e\u73b0\u7c7b\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u8bf4\u660e XXX \u4e3a OSS bucket \u540d\u79f0\u3002 Credential Provider \u7c7b\u578b \u00a7 1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS bucket\u7684AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS bucket\u7684AccessKey Secret</value> </property> <property> <name>fs.oss.bucket.XXX.securityToken</name> <value>OSS bucket\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration> 2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS bucket\u7684AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS bucket\u7684AccessKey Secret</value> </property> </configuration> 3. EnvironmentVariableCredentialsProvider \u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6 AccessKey\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS bucket\u7684AccessKey Id OSS_ACCESS_KEY_SECRET OSS bucket\u7684AccessKey Secret OSS_SECURITY_TOKEN OSS bucket\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548Token\u65f6\u9700\u8981\u3002 4. CommonCredentialsProvider\u4e3a\u901a\u7528\u914d\u7f6e\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS bucket \u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS bucket \u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS bucket \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration>","title":"JindoSDK OSS Credential Provider \u6309 OSS bucket \u914d\u7f6e\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/oss/security/jindosdk_credential_provider_bucket_oss/#jindosdk-oss-credential-provider-oss-bucket","text":"","title":"JindoSDK OSS Credential Provider \u6309 OSS bucket \u914d\u7f6e\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/oss/security/jindosdk_credential_provider_bucket_oss/#bucket-credential-provider","text":"<configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleAliyunCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.AliyunCredentialsProvider\u7684\u5b9e\u73b0\u7c7b\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u8bf4\u660e XXX \u4e3a OSS bucket \u540d\u79f0\u3002","title":"\u6309\u7167 bucket \u914d\u7f6e Credential Provider"},{"location":"backup/oss/security/jindosdk_credential_provider_bucket_oss/#credential-provider","text":"","title":"Credential Provider \u7c7b\u578b"},{"location":"backup/oss/security/jindosdk_credential_provider_bucket_oss/#1-temporarycredentialsprovider-accesskey-securitytoken-oss","text":"\u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS bucket\u7684AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS bucket\u7684AccessKey Secret</value> </property> <property> <name>fs.oss.bucket.XXX.securityToken</name> <value>OSS bucket\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration>","title":"1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002"},{"location":"backup/oss/security/jindosdk_credential_provider_bucket_oss/#2-simplecredentialsprovider-accesskey-oss","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS bucket\u7684AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS bucket\u7684AccessKey Secret</value> </property> </configuration>","title":"2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002"},{"location":"backup/oss/security/jindosdk_credential_provider_bucket_oss/#3-environmentvariablecredentialsprovider-accesskey","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS bucket\u7684AccessKey Id OSS_ACCESS_KEY_SECRET OSS bucket\u7684AccessKey Secret OSS_SECURITY_TOKEN OSS bucket\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548Token\u65f6\u9700\u8981\u3002","title":"3. EnvironmentVariableCredentialsProvider \u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6 AccessKey\u3002"},{"location":"backup/oss/security/jindosdk_credential_provider_bucket_oss/#4-commoncredentialsprovider","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS bucket \u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS bucket \u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS bucket \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration>","title":"4. CommonCredentialsProvider\u4e3a\u901a\u7528\u914d\u7f6e\u3002"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/","text":"JindoSDK OSS Credential Provider \u914d\u7f6e \u00a7 \u57fa\u672c\u914d\u7f6e\u65b9\u5f0f \u00a7 \u60a8\u53ef\u4ee5\u5c06 OSS \u7684 Access Key ID \u3001 Access Secret \u3001 Endpoint \u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \uff0c\u914d\u7f6e\u9879\u5982\u4e0b\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> \u4f7f\u7528 Hadoop Credential Providers \u5b58\u50a8 AccessKey \u4fe1\u606f \u00a7 \u4e0a\u9762\u7684 fs.oss.accessKeyId \u548c fs.oss.accessKeySecret \u5c06\u4f1a\u660e\u6587\u914d\u7f6e\u5728 core-site.xml \u4e2d\uff0c\u53ef\u4ee5\u5c06\u5176\u4ee5\u52a0\u5bc6\u5bf9\u65b9\u5f0f\u5b58\u50a8\u81f3 Hadoop Credential Providers\u6587\u4ef6\u4e2d\u3002 \u4f7f\u7528Hadoop\u63d0\u4f9b\u7684\u547d\u4ee4\uff0c\u5b58\u50a8AccessKey\u548cSecurityToken\u4fe1\u606f\u81f3Credential\u6587\u4ef6\u4e2d\u3002\u547d\u4ee4\u683c\u5f0f\u5982\u4e0b\u3002 hadoop credential <subcommand> [options] \u4f8b\u5982\uff0c\u5b58\u50a8AccessKey\u548cToken\u4fe1\u606f\u81f3JECKS\u6587\u4ef6\u4e2d\uff0c\u9664\u4e86\u4f7f\u7528\u6587\u4ef6\u6743\u9650\u4fdd\u62a4\u8be5\u6587\u4ef6\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u6307\u5b9a\u5bc6\u7801\u52a0\u5bc6\u5b58\u50a8\u4fe1\u606f\uff0c\u5982\u679c\u4e0d\u6307\u5b9a\u5bc6\u7801\u5219\u4f7f\u7528\u9ed8\u8ba4\u5b57\u7b26\u4e32\u52a0\u5bc6\u3002 hadoop credential create fs.oss.accessKeyId -value AAA -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.accessKeySecret -value BBB -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.securityToken -value CCC -provider jceks://file/root/oss.jceks \u751f\u6210Credential\u6587\u4ef6\u540e\uff0c\u60a8\u9700\u8981\u914d\u7f6e\u4e0b\u9762\u7684\u53c2\u6570\u6765\u6307\u5b9aProvider\u7684\u7c7b\u578b\u548c\u4f4d\u7f6e\u3002 <configuration> <property> <name>fs.oss.security.credential.provider.path</name> <value>jceks://file/root/oss.jceks</value> <description>\u914d\u7f6e\u5b58\u50a8AK\u7684Credential\u6587\u4ef6\u3002\u4f8b\u5982\uff0cjceks://file/${user.home}/oss.jceks\u4e3aHOME\u4e0b\u7684oss.jceks\u6587\u4ef6</description> </property> </configuration> \u4f7f\u7528 JindoSDK OSS Credential Provider \u00a7 \u9ed8\u8ba4\u4f1a\u914d\u7f6e SimpleCredentialsProvider\uff0cEnvironmentVariableCredentialsProvider\uff0cCommonCredentialsProvider \u8fd9\u4e09\u4e2aCredential Provider, \u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002 <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.JindoCredentialsProvider\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u60a8\u53ef\u4ee5\u6839\u636e\u60c5\u51b5\uff0c\u9009\u62e9\u4e0d\u540c\u7684Credential Provider\uff0c\u652f\u6301\u5982\u4e0bProvider\uff1a 1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS\u7684AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS\u7684AccessKey Secret</value> </property> <property> <name>fs.oss.securityToken</name> <value>OSS\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration> 2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS \u7684AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS \u7684AccessKey Secret</value> </property> </configuration> 3. EnvironmentVariableCredentialsProvider\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6AK\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS \u7684 AccessKey Id OSS_ACCESS_KEY_SECRET OSS \u7684 AccessKey Secret OSS_SECURITY_TOKEN OSS \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002 4. CommonCredentialsProvider \u4e3a\u901a\u7528\u914d\u7f6e\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS \u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS \u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS \u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration> 5. CustomCredentialsProvider \u5bf9\u63a5\u5b9a\u5236\u7684\u514d\u5bc6\u670d\u52a1\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CustomCredentialsProvider</value> </property> </configuration> \u914d\u7f6e\u514d\u5bc6\u670d\u52a1\u5730\u5740 <configuration> <property> <name>aliyun.oss.provider.url</name> <value>\u514d\u5bc6\u670d\u52a1\u5730\u5740</value> </property> </configuration> aliyun.oss.provider.url \u652f\u6301 http(s) \u534f\u8bae \u548c secrets \u534f\u8bae\uff1a a. http(s) \u534f\u8bae http(s) \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a http://localhost:1234/sts , http \u514d\u5bc6\u534f\u8bae\u8981\u6c42\u8fd4\u56de\u7ed3\u679c\u4e3a Json \u683c\u5f0f\uff0c\u5982\u679c\u60a8\u9700\u8981\u5bf9\u63a5\u60a8\u7684 http \u514d\u5bc6\u670d\u52a1\uff0c\u514d\u5bc6\u670d\u52a1\u7684\u8bbe\u8ba1\u53c2\u8003 \u300a\u901a\u8fc7API\u4f7f\u7528\u5b9e\u4f8bRAM\u89d2\u8272#\u4e34\u65f6\u6388\u6743Token\u300b { \"AccessKeyId\" : \"XXXXXXXXX\", \"AccessKeySecret\" : \"XXXXXXXXX\", \"Expiration\" : \"2020-11-01T05:20:01Z\", \"SecurityToken\" : \"XXXXXXXXX\", \"LastUpdated\" : \"2020-10-31T23:20:01Z\", \"Code\" : \"Success\" } b. Secrets \u534f\u8bae Secrets \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a secrets:///local_path_prefix \uff0c\u5e38\u89c1\u4f7f\u7528\u4e8e k8s \u573a\u666f \uff0c \u5176\u4e2d local_path_prefix \u4e3a\u8def\u5f84\u524d\u7f00\uff0c\u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOssAccessKeyId /secret/JindoOssAccessKeySecret /secret/JindoOssSecurityToken \u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss/ \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOss/AccessKeyId /secret/JindoOss/AccessKeySecret /secret/JindoOss/SecurityToken JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 OSS bucket \u914d\u7f6e\u4e0d\u540c\u7684 Credential Provider \u00a7 \u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u6309 OSS bucket \u914d\u7f6e\u4f7f\u7528\u8bf4\u660e \u3002","title":"JindoSDK OSS Credential Provider \u914d\u7f6e"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/#jindosdk-oss-credential-provider","text":"","title":"JindoSDK OSS Credential Provider \u914d\u7f6e"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/#_1","text":"\u60a8\u53ef\u4ee5\u5c06 OSS \u7684 Access Key ID \u3001 Access Secret \u3001 Endpoint \u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \uff0c\u914d\u7f6e\u9879\u5982\u4e0b\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration>","title":"\u57fa\u672c\u914d\u7f6e\u65b9\u5f0f"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/#hadoop-credential-providers-accesskey","text":"\u4e0a\u9762\u7684 fs.oss.accessKeyId \u548c fs.oss.accessKeySecret \u5c06\u4f1a\u660e\u6587\u914d\u7f6e\u5728 core-site.xml \u4e2d\uff0c\u53ef\u4ee5\u5c06\u5176\u4ee5\u52a0\u5bc6\u5bf9\u65b9\u5f0f\u5b58\u50a8\u81f3 Hadoop Credential Providers\u6587\u4ef6\u4e2d\u3002 \u4f7f\u7528Hadoop\u63d0\u4f9b\u7684\u547d\u4ee4\uff0c\u5b58\u50a8AccessKey\u548cSecurityToken\u4fe1\u606f\u81f3Credential\u6587\u4ef6\u4e2d\u3002\u547d\u4ee4\u683c\u5f0f\u5982\u4e0b\u3002 hadoop credential <subcommand> [options] \u4f8b\u5982\uff0c\u5b58\u50a8AccessKey\u548cToken\u4fe1\u606f\u81f3JECKS\u6587\u4ef6\u4e2d\uff0c\u9664\u4e86\u4f7f\u7528\u6587\u4ef6\u6743\u9650\u4fdd\u62a4\u8be5\u6587\u4ef6\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u6307\u5b9a\u5bc6\u7801\u52a0\u5bc6\u5b58\u50a8\u4fe1\u606f\uff0c\u5982\u679c\u4e0d\u6307\u5b9a\u5bc6\u7801\u5219\u4f7f\u7528\u9ed8\u8ba4\u5b57\u7b26\u4e32\u52a0\u5bc6\u3002 hadoop credential create fs.oss.accessKeyId -value AAA -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.accessKeySecret -value BBB -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.securityToken -value CCC -provider jceks://file/root/oss.jceks \u751f\u6210Credential\u6587\u4ef6\u540e\uff0c\u60a8\u9700\u8981\u914d\u7f6e\u4e0b\u9762\u7684\u53c2\u6570\u6765\u6307\u5b9aProvider\u7684\u7c7b\u578b\u548c\u4f4d\u7f6e\u3002 <configuration> <property> <name>fs.oss.security.credential.provider.path</name> <value>jceks://file/root/oss.jceks</value> <description>\u914d\u7f6e\u5b58\u50a8AK\u7684Credential\u6587\u4ef6\u3002\u4f8b\u5982\uff0cjceks://file/${user.home}/oss.jceks\u4e3aHOME\u4e0b\u7684oss.jceks\u6587\u4ef6</description> </property> </configuration>","title":"\u4f7f\u7528 Hadoop Credential Providers \u5b58\u50a8 AccessKey \u4fe1\u606f"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/#jindosdk-oss-credential-provider_1","text":"\u9ed8\u8ba4\u4f1a\u914d\u7f6e SimpleCredentialsProvider\uff0cEnvironmentVariableCredentialsProvider\uff0cCommonCredentialsProvider \u8fd9\u4e09\u4e2aCredential Provider, \u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002 <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.JindoCredentialsProvider\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u60a8\u53ef\u4ee5\u6839\u636e\u60c5\u51b5\uff0c\u9009\u62e9\u4e0d\u540c\u7684Credential Provider\uff0c\u652f\u6301\u5982\u4e0bProvider\uff1a","title":"\u4f7f\u7528 JindoSDK OSS Credential Provider"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/#1-temporarycredentialsprovider-accesskey-securitytoken-oss","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS\u7684AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS\u7684AccessKey Secret</value> </property> <property> <name>fs.oss.securityToken</name> <value>OSS\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration>","title":"1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/#2-simplecredentialsprovider-accesskey-oss","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS \u7684AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS \u7684AccessKey Secret</value> </property> </configuration>","title":"2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/#3-environmentvariablecredentialsproviderak","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS \u7684 AccessKey Id OSS_ACCESS_KEY_SECRET OSS \u7684 AccessKey Secret OSS_SECURITY_TOKEN OSS \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002","title":"3. EnvironmentVariableCredentialsProvider\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6AK\u3002"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/#4-commoncredentialsprovider","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS \u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS \u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS \u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration>","title":"4. CommonCredentialsProvider \u4e3a\u901a\u7528\u914d\u7f6e\u3002"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/#5-customcredentialsprovider","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CustomCredentialsProvider</value> </property> </configuration> \u914d\u7f6e\u514d\u5bc6\u670d\u52a1\u5730\u5740 <configuration> <property> <name>aliyun.oss.provider.url</name> <value>\u514d\u5bc6\u670d\u52a1\u5730\u5740</value> </property> </configuration> aliyun.oss.provider.url \u652f\u6301 http(s) \u534f\u8bae \u548c secrets \u534f\u8bae\uff1a a. http(s) \u534f\u8bae http(s) \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a http://localhost:1234/sts , http \u514d\u5bc6\u534f\u8bae\u8981\u6c42\u8fd4\u56de\u7ed3\u679c\u4e3a Json \u683c\u5f0f\uff0c\u5982\u679c\u60a8\u9700\u8981\u5bf9\u63a5\u60a8\u7684 http \u514d\u5bc6\u670d\u52a1\uff0c\u514d\u5bc6\u670d\u52a1\u7684\u8bbe\u8ba1\u53c2\u8003 \u300a\u901a\u8fc7API\u4f7f\u7528\u5b9e\u4f8bRAM\u89d2\u8272#\u4e34\u65f6\u6388\u6743Token\u300b { \"AccessKeyId\" : \"XXXXXXXXX\", \"AccessKeySecret\" : \"XXXXXXXXX\", \"Expiration\" : \"2020-11-01T05:20:01Z\", \"SecurityToken\" : \"XXXXXXXXX\", \"LastUpdated\" : \"2020-10-31T23:20:01Z\", \"Code\" : \"Success\" } b. Secrets \u534f\u8bae Secrets \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a secrets:///local_path_prefix \uff0c\u5e38\u89c1\u4f7f\u7528\u4e8e k8s \u573a\u666f \uff0c \u5176\u4e2d local_path_prefix \u4e3a\u8def\u5f84\u524d\u7f00\uff0c\u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOssAccessKeyId /secret/JindoOssAccessKeySecret /secret/JindoOssSecurityToken \u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss/ \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOss/AccessKeyId /secret/JindoOss/AccessKeySecret /secret/JindoOss/SecurityToken","title":"5. CustomCredentialsProvider \u5bf9\u63a5\u5b9a\u5236\u7684\u514d\u5bc6\u670d\u52a1\u3002"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/#jindosdk-oss-bucket-credential-provider","text":"\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u6309 OSS bucket \u914d\u7f6e\u4f7f\u7528\u8bf4\u660e \u3002","title":"JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 OSS bucket \u914d\u7f6e\u4e0d\u540c\u7684 Credential Provider"},{"location":"backup/oss/spark/jindosdk_ide_spark/","text":"Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 \u00a7 \u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindosdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop with provided scope, your runtime should install hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.spark.sql.SparkSession; public class TestJindoSDK { public static void main(String[] args) throws Exception { SparkSession spark = SparkSession .builder() .config(\"spark.hadoop.fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\") .config(\"spark.hadoop.fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\") .config(\"spark.hadoop.fs.oss.accessKeyId\", \"xxx\") .config(\"spark.hadoop.fs.oss.accessKeySecret\", \"xxx\") // set accessKey, secret, endpoint and so on. .appName(\"TestJindoSDK\") .getOrCreate(); spark.read().parquet(\"oss://xxx\").count(); spark.stop(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency>","title":"Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5"},{"location":"backup/oss/spark/jindosdk_ide_spark/#spark-jindosdk-ide","text":"\u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindosdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop with provided scope, your runtime should install hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.spark.sql.SparkSession; public class TestJindoSDK { public static void main(String[] args) throws Exception { SparkSession spark = SparkSession .builder() .config(\"spark.hadoop.fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\") .config(\"spark.hadoop.fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\") .config(\"spark.hadoop.fs.oss.accessKeyId\", \"xxx\") .config(\"spark.hadoop.fs.oss.accessKeySecret\", \"xxx\") // set accessKey, secret, endpoint and so on. .appName(\"TestJindoSDK\") .getOrCreate(); spark.read().parquet(\"oss://xxx\").count(); spark.stop(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency>","title":"Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5"},{"location":"backup/oss/spark/jindosdk_on_spark/","text":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e \u00a7 JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411Hadoop/Spark\u751f\u6001\u7684OSS\u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684Hadoop FileSystem\u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fddSpark\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/ 3. \u914d\u7f6e JindoSDK \u00a7 \u5168\u5c40\u914d\u7f6e,\u4fee\u6539 Spark \u4f7f\u7528\u7684 core-site.xml \u00a7 \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u4efb\u52a1\u7ea7\u522b\u914d\u7f6e \u00a7 \u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx --conf spark.hadoop.fs.oss.endpoint=oss-cn-xxx.aliyuncs.com 4. \u4f7f\u7528 Spark \u8bbf\u95ee OSS \u00a7 \u521b\u5efa\u8868 create table test_oss (c1 string) location \"oss://bucket/dir\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss; 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e"},{"location":"backup/oss/spark/jindosdk_on_spark/#spark-jindosdk-oss","text":"JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411Hadoop/Spark\u751f\u6001\u7684OSS\u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684Hadoop FileSystem\u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e"},{"location":"backup/oss/spark/jindosdk_on_spark/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/oss/spark/jindosdk_on_spark/#1-spark-hadoop","text":"\u786e\u4fddSpark\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"backup/oss/spark/jindosdk_on_spark/#2-spark-classpath-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/","title":"2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK"},{"location":"backup/oss/spark/jindosdk_on_spark/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/oss/spark/jindosdk_on_spark/#spark-core-sitexml","text":"\u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002","title":"\u5168\u5c40\u914d\u7f6e,\u4fee\u6539 Spark \u4f7f\u7528\u7684 core-site.xml"},{"location":"backup/oss/spark/jindosdk_on_spark/#_2","text":"\u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx --conf spark.hadoop.fs.oss.endpoint=oss-cn-xxx.aliyuncs.com","title":"\u4efb\u52a1\u7ea7\u522b\u914d\u7f6e"},{"location":"backup/oss/spark/jindosdk_on_spark/#4-spark-oss","text":"\u521b\u5efa\u8868 create table test_oss (c1 string) location \"oss://bucket/dir\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss;","title":"4. \u4f7f\u7528 Spark \u8bbf\u95ee OSS"},{"location":"backup/oss/spark/jindosdk_on_spark/#5","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/oss/sqoop/kitesdk_on_sqoop/","text":"Sqoop \u4f7f\u7528 Kite SDK \u8bfb\u5199 OSS \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 \u5728\u96c6\u7fa4\u4e0a\u6709\u5f00\u6e90\u7248\u672c Sqoop \u8f6f\u4ef6\uff0c\u7248\u672c\u4e0d\u4f4e\u4e8e 1.4.7\u3002 Sqoop \u4f9d\u8d56\u7684 Hadoop \u73af\u5883\u53ef\u8bbf\u95ee OSS\uff0c\u53ef\u53c2\u8003 Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u4f7f\u7528 Kite SDK \u8bfb\u5199 OSS \u00a7 Sqoop \u672c\u8eab\u5e76\u4e0d\u652f\u6301\u5bf9 OSS \u7684\u8bfb\u5199\uff0c\u9700\u8981\u4f7f\u7528\u7b2c\u4e09\u65b9 Kite SDK \u8fdb\u884c URI \u7684\u8f6c\u6362\u4ece\u800c\u8fdb\u884c\u6570\u636e\u4ea4\u4e92\u3002 \u6b65\u9aa4 \u00a7 1. \u5b89\u88c5 jar \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 jar \u5305 kite-data-oss-3.4.0.jar ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u5c06 sdk \u5305\u5b89\u88c5\u5230 Sqoop \u7684 classpath \u76ee\u5f55\u4e0b\u3002 cp ./kite-data-oss-3.4.0.jar <SQOOP_HOME>/lib/kite-data-oss-3.4.0.jar 2.\u6388\u4e88\u5bf9 JAR \u7684\u6743\u9650 \u00a7 sudo chmod 755 kite-data-oss-3.4.0.jar 3.\u4f7f\u7528 oss \u8fde\u63a5\u5668\u5bfc\u5165 mysql \u4e0a\u7684\u6570\u636e \u00a7 sqoop import --connect jdbc:mysql://<host>:<port>/database --username username --password password --table yourtable --target-dir \"oss://yourbucket/dir/\" --as-parquetfile -m 5","title":"Sqoop \u4f7f\u7528 Kite SDK \u8bfb\u5199 OSS"},{"location":"backup/oss/sqoop/kitesdk_on_sqoop/#sqoop-kite-sdk-oss","text":"","title":"Sqoop \u4f7f\u7528 Kite SDK \u8bfb\u5199 OSS"},{"location":"backup/oss/sqoop/kitesdk_on_sqoop/#_1","text":"\u5728\u96c6\u7fa4\u4e0a\u6709\u5f00\u6e90\u7248\u672c Sqoop \u8f6f\u4ef6\uff0c\u7248\u672c\u4e0d\u4f4e\u4e8e 1.4.7\u3002 Sqoop \u4f9d\u8d56\u7684 Hadoop \u73af\u5883\u53ef\u8bbf\u95ee OSS\uff0c\u53ef\u53c2\u8003 Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/oss/sqoop/kitesdk_on_sqoop/#kite-sdk-oss","text":"Sqoop \u672c\u8eab\u5e76\u4e0d\u652f\u6301\u5bf9 OSS \u7684\u8bfb\u5199\uff0c\u9700\u8981\u4f7f\u7528\u7b2c\u4e09\u65b9 Kite SDK \u8fdb\u884c URI \u7684\u8f6c\u6362\u4ece\u800c\u8fdb\u884c\u6570\u636e\u4ea4\u4e92\u3002","title":"\u4f7f\u7528 Kite SDK \u8bfb\u5199 OSS"},{"location":"backup/oss/sqoop/kitesdk_on_sqoop/#_2","text":"","title":"\u6b65\u9aa4"},{"location":"backup/oss/sqoop/kitesdk_on_sqoop/#1-jar","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 jar \u5305 kite-data-oss-3.4.0.jar ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u5c06 sdk \u5305\u5b89\u88c5\u5230 Sqoop \u7684 classpath \u76ee\u5f55\u4e0b\u3002 cp ./kite-data-oss-3.4.0.jar <SQOOP_HOME>/lib/kite-data-oss-3.4.0.jar","title":"1. \u5b89\u88c5 jar \u5305"},{"location":"backup/oss/sqoop/kitesdk_on_sqoop/#2-jar","text":"sudo chmod 755 kite-data-oss-3.4.0.jar","title":"2.\u6388\u4e88\u5bf9 JAR \u7684\u6743\u9650"},{"location":"backup/oss/sqoop/kitesdk_on_sqoop/#3-oss-mysql","text":"sqoop import --connect jdbc:mysql://<host>:<port>/database --username username --password password --table yourtable --target-dir \"oss://yourbucket/dir/\" --as-parquetfile -m 5","title":"3.\u4f7f\u7528 oss \u8fde\u63a5\u5668\u5bfc\u5165 mysql \u4e0a\u7684\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_aliyun_sls_example/","text":"\u4f7f\u7528\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u6536\u96c6\u670d\u52a1 \u00a7 JindoRuntime \u9ed8\u8ba4\u65e5\u5fd7\u6253\u5728\u5bb9\u5668 std \u8f93\u51fa\u91cc\u9762\uff0c\u53ef\u4ee5\u4f7f\u7528\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u6536\u96c6\u670d\u52a1\u5c06\u65e5\u5fd7\u6536\u96c6\u8d77\u6765\u3002 1\u3001ack \u96c6\u7fa4\u7ed1\u5b9a sls \u670d\u52a1 \u00a7 \u60a8\u53ef\u4ee5\u5728\u521b\u5efa\u96c6\u7fa4\u65f6\u7ed1\u5b9a sls \u670d\u52a1 \u4e5f\u53ef\u4ee5\u5728\u521b\u5efa\u96c6\u7fa4\u540e\u7ed1\u5b9a\u963f\u91cc\u4e91 sls \u670d\u52a1 \u9009\u62e9\u96c6\u7fa4\u4fe1\u606f > \u96c6\u7fa4\u8d44\u6e90 > \u65e5\u5fd7\u670d\u52a1 Project 2\u3001\u521b\u5efa JindoRuntime \u96c6\u7fa4 \u00a7 \u53ef\u53c2\u8003\u6587\u7ae0 \u52a0\u901fOSS\u4e0a\u6570\u636e \u521b\u5efa JindoRuntime \u96c6\u7fa4\u3002 3\u3001\u914d\u7f6e sls \u6536\u96c6 \u00a7 \u60a8\u53ef\u4ee5\u9009\u62e9\u5728 ack \u63a7\u5236\u53f0\u64cd\u4f5c\uff0c\u6216\u8005\u76f4\u63a5\u64cd\u4f5c\u96c6\u7fa4 K8S \u8d44\u6e90 3.1 ack \u63a7\u5236\u53f0\u64cd\u4f5c \u00a7 \u672c\u6587\u4ee5 JindoRuntime \u540d\u5b57\u4e3a test \u8fdb\u884c\u4e3e\u4f8b \u60a8\u53ef\u4ee5\u767b\u9646 ack \u63a7\u5236\u53f0 > \u5de5\u4f5c\u8d1f\u8f7d * \u6709\u72b6\u6001\uff08master/worker\uff09 \u5b88\u62a4\u8fdb\u7a0b\u96c6\uff08fuse\uff09 \u70b9\u51fb \u7f16\u8f91 \u6dfb\u52a0\u65e5\u5fd7\u6536\u96c6 \u5176\u4e2d\u65e5\u5fd7\u5e93\u540d\u79f0\u53ef\u81ea\u5b9a\u4e49\u586b\u5199\uff0c\u8be5\u540d\u79f0\u5373\u4e3a sls \u4e0a\u65e5\u5fd7\u5e93\u7684\u540d\u5b57\uff0c\u5bb9\u5668\u5185\u65e5\u5fd7\u8def\u5f84\u4e3a stdout \u70b9\u51fb\u53f3\u4fa7\u66f4\u65b0 3.2\u3001\u76f4\u63a5\u64cd\u4f5c K8S \u8d44\u6e90 \u00a7 \u4ee5\u6536\u96c6 fuse \u65e5\u5fd7\u4e3a\u4f8b\uff0c\u6267\u884c kubectl edit daemonset test-jindofs-fuse \u4fee\u6539 spec.env \u589e\u52a0 - name: aliyun_logs_jindo-fuse value: stdout name \u683c\u5f0f\u4e3a\uff1a aliyun_logs_ \uff0c\u524d\u7f00\u4e3a aliyun_logs_ value \u4e3a\u56fa\u5b9a\u503c stdout \u5982\u679c\u60f3\u4fee\u6539 master/worker\uff0c\u53ef\u6267\u884c kubectl edit statefulset test-jindofs-master kubectl edit statefulset test-jindofs-worker \u8fdb\u884c\u5bf9\u5e94\u7684\u7f16\u8f91 4\u3001\u767b\u9646\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u670d\u52a1\u67e5\u770b\u76f8\u5173\u65e5\u5fd7\u60c5\u51b5 \u00a7 \u4ee5 jindofuse \u65e5\u5fd7\u6536\u96c6\u4e3a\u4f8b\uff0c\u65e5\u5fd7\u5e93\u540d\u79f0\u4e3a jindo-fuse ,\u5728sls\u4e0a\u627e\u5230\u5bf9\u5e94\u7684\u65e5\u5fd7\u5e93","title":"\u4f7f\u7528\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u6536\u96c6\u670d\u52a1"},{"location":"fluid-jindoruntime/jindo_fluid_aliyun_sls_example/#sls","text":"JindoRuntime \u9ed8\u8ba4\u65e5\u5fd7\u6253\u5728\u5bb9\u5668 std \u8f93\u51fa\u91cc\u9762\uff0c\u53ef\u4ee5\u4f7f\u7528\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u6536\u96c6\u670d\u52a1\u5c06\u65e5\u5fd7\u6536\u96c6\u8d77\u6765\u3002","title":"\u4f7f\u7528\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u6536\u96c6\u670d\u52a1"},{"location":"fluid-jindoruntime/jindo_fluid_aliyun_sls_example/#1ack-sls","text":"\u60a8\u53ef\u4ee5\u5728\u521b\u5efa\u96c6\u7fa4\u65f6\u7ed1\u5b9a sls \u670d\u52a1 \u4e5f\u53ef\u4ee5\u5728\u521b\u5efa\u96c6\u7fa4\u540e\u7ed1\u5b9a\u963f\u91cc\u4e91 sls \u670d\u52a1 \u9009\u62e9\u96c6\u7fa4\u4fe1\u606f > \u96c6\u7fa4\u8d44\u6e90 > \u65e5\u5fd7\u670d\u52a1 Project","title":"1\u3001ack \u96c6\u7fa4\u7ed1\u5b9a sls \u670d\u52a1"},{"location":"fluid-jindoruntime/jindo_fluid_aliyun_sls_example/#2-jindoruntime","text":"\u53ef\u53c2\u8003\u6587\u7ae0 \u52a0\u901fOSS\u4e0a\u6570\u636e \u521b\u5efa JindoRuntime \u96c6\u7fa4\u3002","title":"2\u3001\u521b\u5efa JindoRuntime \u96c6\u7fa4"},{"location":"fluid-jindoruntime/jindo_fluid_aliyun_sls_example/#3-sls","text":"\u60a8\u53ef\u4ee5\u9009\u62e9\u5728 ack \u63a7\u5236\u53f0\u64cd\u4f5c\uff0c\u6216\u8005\u76f4\u63a5\u64cd\u4f5c\u96c6\u7fa4 K8S \u8d44\u6e90","title":"3\u3001\u914d\u7f6e sls \u6536\u96c6"},{"location":"fluid-jindoruntime/jindo_fluid_aliyun_sls_example/#31-ack","text":"\u672c\u6587\u4ee5 JindoRuntime \u540d\u5b57\u4e3a test \u8fdb\u884c\u4e3e\u4f8b \u60a8\u53ef\u4ee5\u767b\u9646 ack \u63a7\u5236\u53f0 > \u5de5\u4f5c\u8d1f\u8f7d * \u6709\u72b6\u6001\uff08master/worker\uff09 \u5b88\u62a4\u8fdb\u7a0b\u96c6\uff08fuse\uff09 \u70b9\u51fb \u7f16\u8f91 \u6dfb\u52a0\u65e5\u5fd7\u6536\u96c6 \u5176\u4e2d\u65e5\u5fd7\u5e93\u540d\u79f0\u53ef\u81ea\u5b9a\u4e49\u586b\u5199\uff0c\u8be5\u540d\u79f0\u5373\u4e3a sls \u4e0a\u65e5\u5fd7\u5e93\u7684\u540d\u5b57\uff0c\u5bb9\u5668\u5185\u65e5\u5fd7\u8def\u5f84\u4e3a stdout \u70b9\u51fb\u53f3\u4fa7\u66f4\u65b0","title":"3.1 ack \u63a7\u5236\u53f0\u64cd\u4f5c"},{"location":"fluid-jindoruntime/jindo_fluid_aliyun_sls_example/#32-k8s","text":"\u4ee5\u6536\u96c6 fuse \u65e5\u5fd7\u4e3a\u4f8b\uff0c\u6267\u884c kubectl edit daemonset test-jindofs-fuse \u4fee\u6539 spec.env \u589e\u52a0 - name: aliyun_logs_jindo-fuse value: stdout name \u683c\u5f0f\u4e3a\uff1a aliyun_logs_ \uff0c\u524d\u7f00\u4e3a aliyun_logs_ value \u4e3a\u56fa\u5b9a\u503c stdout \u5982\u679c\u60f3\u4fee\u6539 master/worker\uff0c\u53ef\u6267\u884c kubectl edit statefulset test-jindofs-master kubectl edit statefulset test-jindofs-worker \u8fdb\u884c\u5bf9\u5e94\u7684\u7f16\u8f91","title":"3.2\u3001\u76f4\u63a5\u64cd\u4f5c K8S \u8d44\u6e90"},{"location":"fluid-jindoruntime/jindo_fluid_aliyun_sls_example/#4-sls","text":"\u4ee5 jindofuse \u65e5\u5fd7\u6536\u96c6\u4e3a\u4f8b\uff0c\u65e5\u5fd7\u5e93\u540d\u79f0\u4e3a jindo-fuse ,\u5728sls\u4e0a\u627e\u5230\u5bf9\u5e94\u7684\u65e5\u5fd7\u5e93","title":"4\u3001\u767b\u9646\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u670d\u52a1\u67e5\u770b\u76f8\u5173\u65e5\u5fd7\u60c5\u51b5"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/","text":"\u6570\u636e\u9884\u52a0\u8f7d \u00a7 \u914d\u7f6e\u5f85\u521b\u5efa\u7684DataLoad\u5bf9\u8c61 \u521b\u5efaDataLoad\u5bf9\u8c61 \u67e5\u770b\u521b\u5efa\u7684DataLoad\u5bf9\u8c61\u72b6\u6001 \u7b49\u5f85\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5b8c\u6210 DataLoad\u8fdb\u9636\u914d\u7f6e \u6307\u5b9a\u4e00\u4e2a\u6216\u591a\u4e2a\u6570\u636e\u96c6\u5b50\u76ee\u5f55\u8fdb\u884c\u52a0\u8f7d \u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf \u539f\u5b50\u6027\u7f13\u5b58 \u5355\u72ec\u8fdb\u884c\u5143\u6570\u636e\u7f13\u5b58\uff0c\u4e0d\u505a\u6570\u636e\u7f13\u5b58 \u7f13\u5b58\u8fdb\u5ea6\u53caLOG\u67e5\u770b \u73af\u5883\u6e05\u7406 \u4e3a\u4e86\u4fdd\u8bc1\u5e94\u7528\u5728\u8bbf\u95ee\u6570\u636e\u65f6\u7684\u6027\u80fd\uff0c\u53ef\u4ee5\u901a\u8fc7 \u6570\u636e\u9884\u52a0\u8f7d \u63d0\u524d\u5c06\u8fdc\u7a0b\u5b58\u50a8\u7cfb\u7edf\u4e2d\u7684\u6570\u636e\u62c9\u53d6\u5230\u9760\u8fd1\u8ba1\u7b97\u7ed3\u70b9\u7684\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d\uff0c\u4f7f\u5f97\u6d88\u8d39\u8be5\u6570\u636e\u96c6\u7684\u5e94\u7528\u80fd\u591f\u5728\u9996\u6b21\u8fd0\u884c\u65f6\u5373\u53ef\u4eab\u53d7\u5230\u7f13\u5b58\u5e26\u6765\u7684\u52a0\u901f\u6548\u679c\u3002 \u4e3a\u6b64\uff0c\u6211\u4eec\u63d0\u4f9b\u4e86DataLoad CRD, \u8be5CRD\u8ba9\u4f60\u53ef\u901a\u8fc7\u7b80\u5355\u7684\u914d\u7f6e\u5c31\u80fd\u5b8c\u6210\u6574\u4e2a\u6570\u636e\u9884\u52a0\u8f7d\u8fc7\u7a0b\uff0c\u5e76\u5bf9\u6570\u636e\u9884\u52a0\u8f7d\u7684\u8bb8\u591a\u884c\u4e3a\u8fdb\u884c\u81ea\u5b9a\u4e49\u63a7\u5236\u3002 \u5047\u8bbe\u6211\u4eec\u5df2\u7ecf\u6709\u5982\u4e0b\u4e00\u4e2aname\u4e3a spark \u7684dataset\uff1a apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: spark spec: mounts: - mountPoint: oss://test-bucket/spark name: sparkV1 accessModes: - ReadWriteMany \u63a5\u4e0b\u6765\u5bf9\u8fd9\u4e2adataset\u8fdb\u884c\u9884\u70ed\uff0c\u8bf7\u6ce8\u610f\u4e0b\u6587\u4e2d\u7684 /sparkV1 \u4e3a mounts[].name \u7684\u503c\uff0c\u5982\u679c\u60a8\u7684 mounts[].path \u4e3a / , \u90a3\u4e48\u4e0b\u6587\u4e2d\u7684 /sparkV1 \u53ef\u4ee5\u76f4\u63a5\u6539\u4e3a / \u914d\u7f6e\u5f85\u521b\u5efa\u7684DataLoad\u5bf9\u8c61 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default target: - path: /sparkV1/ spec.dataset \u6307\u660e\u4e86\u9700\u8981\u8fdb\u884c\u6570\u636e\u9884\u52a0\u8f7d\u7684\u76ee\u6807\u6570\u636e\u96c6\uff0c\u5728\u8be5\u4f8b\u5b50\u4e2d\uff0c\u6211\u4eec\u7684\u6570\u636e\u9884\u52a0\u8f7d\u76ee\u6807\u4e3a default \u547d\u540d\u7a7a\u95f4\u4e0b\u540d\u4e3a spark \u7684\u6570\u636e\u96c6\uff0c\u5982\u679c\u8be5\u914d\u7f6e\u4e0e\u4f60\u6240\u5728\u7684\u5b9e\u9645\u73af\u5883\u4e0d\u7b26\uff0c\u8bf7\u6839\u636e\u4f60\u7684\u5b9e\u9645\u73af\u5883\u5bf9\u5176\u8fdb\u884c\u8c03\u6574\u3002 \u521b\u5efaDataLoad\u5bf9\u8c61 \u00a7 kubectl create -f dataload.yaml \u67e5\u770b\u521b\u5efa\u7684DataLoad\u5bf9\u8c61\u72b6\u6001 \u00a7 kubectl get dataload spark-dataload \u4e0a\u8ff0\u547d\u4ee4\u4f1a\u5f97\u5230\u7c7b\u4f3c\u4ee5\u4e0b\u7ed3\u679c\uff1a NAME DATASET PHASE AGE spark-dataload spark Loading 2m13s \u4f60\u4e5f\u53ef\u4ee5\u901a\u8fc7 kubectl describe \u83b7\u53d6\u6709\u5173\u8be5DataLoad\u7684\u66f4\u591a\u8be6\u7ec6\u4fe1\u606f\uff1a kubectl describe dataload spark-dataload \u5f97\u5230\u4ee5\u4e0b\u7ed3\u679c\uff1a Name: spark-dataload Namespace: default Labels: <none> Annotations: <none> API Version: data.fluid.io/v1alpha1 Kind: DataLoad ... Spec: Dataset: Name: spark Namespace: default Status: Conditions: Phase: Loading Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal DataLoadJobStarted 80s DataLoad The DataLoad job spark-dataload-loader-job started \u4e0a\u8ff0\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u6839\u636e\u4f60\u6240\u5728\u7684\u7f51\u7edc\u73af\u5883\u4e0d\u540c\uff0c\u53ef\u80fd\u4f1a\u8017\u8d39\u6570\u5206\u949f \u7b49\u5f85\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5b8c\u6210 \u00a7 kubectl get dataload spark-dataload \u4f60\u4f1a\u770b\u5230\u8be5DataLoad\u7684 Phase \u72b6\u6001\u5df2\u7ecf\u4ece Loading \u53d8\u4e3a Complete \uff0c\u8fd9\u8868\u660e\u6574\u4e2a\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5df2\u7ecf\u5b8c\u6210 NAME DATASET PHASE AGE spark-dataload spark Complete 5m17s \u6b64\u65f6\u518d\u6b21\u67e5\u770bDataset\u5bf9\u8c61\u7684\u7f13\u5b58\u72b6\u6001\uff1a kubectl get dataset spark \u53ef\u53d1\u73b0\uff0c\u8fdc\u7a0b\u5b58\u50a8\u7cfb\u7edf\u4e2d\u7684\u5168\u90e8\u6570\u636e\u5747\u5df2\u6210\u529f\u7f13\u5b58\u5230\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE spark 1.92GiB 1.92GiB 4.00GiB 100.0% Bound 7m41s DataLoad\u8fdb\u9636\u914d\u7f6e \u00a7 \u6307\u5b9a\u4e00\u4e2a\u6216\u591a\u4e2a\u6570\u636e\u96c6\u5b50\u76ee\u5f55\u8fdb\u884c\u52a0\u8f7d \u00a7 \u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6\u53ef\u4ee5\u52a0\u8f7d\u6307\u5b9a\u7684\u5b50\u76ee\u5f55(\u6216\u6587\u4ef6)\uff0c\u800c\u4e0d\u662f\u6574\u4e2a\u6570\u636e\u96c6\uff0c\u4f8b\u5982\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 - path: /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \u4e0a\u8ff0DataLoad\u4ec5\u4f1a\u52a0\u8f7d /sparkV1/spark-2.4.7 \u76ee\u5f55\u4e0b\u7684\u5168\u90e8\u6587\u4ef6\uff0c\u4ee5\u53ca /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \u6587\u4ef6 \u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf \u00a7 \u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6,\u4f60\u4e5f\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u63a7\u5236\u52a0\u8f7d\u7684\u6570\u636e\u526f\u672c\u6570\u91cf\uff0c\u4f8b\u5982\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 replicas: 1 - path: /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz replicas: 2 \u4e0a\u8ff0DataLoad\u5728\u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6\uff0c\u5bf9\u4e8e /sparkV1/spark-2.4.7 \u76ee\u5f55\u4e0b\u7684\u6587\u4ef6\u4ec5\u4f1a\u5728\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d\u4fdd\u7559 1\u4efd \u6570\u636e\u7f13\u5b58\u526f\u672c\uff0c\u800c\u5bf9\u4e8e\u6587\u4ef6 /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \uff0c\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u5c06\u4f1a\u4fdd\u7559 2\u4efd \u7f13\u5b58\u526f\u672c\u3002 \u539f\u5b50\u6027\u7f13\u5b58 \u00a7 \u5982\u60a8\u60f3\u4fdd\u6301\u7f13\u5b58\u4e00\u81f4\u6027\u548c\u539f\u5b50\u6027\uff0c\u90a3\u4e48\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b yaml \u6765\u5b8c\u6210\u8fd9\u4e2a\u529f\u80fd apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default options: atomicCache: \"true\" \u5355\u72ec\u8fdb\u884c\u5143\u6570\u636e\u7f13\u5b58\uff0c\u4e0d\u505a\u6570\u636e\u7f13\u5b58 \u00a7 \u5728\u8fdb\u884c dataload \u9884\u70ed\u65f6\uff0c\u9ed8\u8ba4\u662f\u4f1a\u505a\u6570\u636e\u7f13\u5b58\uff0c\u539f\u6570\u636e\u7f13\u5b58\u662f\u53ef\u9009\uff0c\u4f46\u5982\u679c\u60a8\u53ea\u60f3\u5143\u6570\u636e\u7f13\u5b58\u4e0d\u60f3\u505a\u6570\u636e\u7f13\u5b58\u7684\u8bdd\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b yaml \u6253\u5f00\u53ea\u505a\u5143\u6570\u636e\u7f13\u5b58\u7684\u5f00\u5173 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default options: loadMetadataOnly: \"true\" \u901a\u8fc7\u901a\u914d\u7b26\u8fdb\u884c\u7f13\u5b58\u8fc7\u6ee4 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 replicas: 1 options: filter: \"*.tgz\" cacheThread: \"2\" dryrun: \"false\" \u4f7f\u7528\u5217\u8868\u6587\u4ef6\u8fdb\u884c\u7f13\u5b58 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true options: accessKeyId: \"xxx\" accessKeySecret: \"xxx\" endpoint: \"oss-cn-shanghai-internal.aliyuncs.com\" url: \"oss://test-bucket/filelist\" dryrun: \"false\" \u7f13\u5b58\u8fdb\u5ea6\u53caLOG\u67e5\u770b \u00a7 $ kubectl get dataload spark-dataload NAME DATASET PHASE AGE spark-dataload spark Loading 2m13s \u67e5\u770b\u5f53\u524d\u7f13\u5b58\u7684\u76f8\u5173\u4fe1\u606f kubectl logs spark-dataload \u73af\u5883\u6e05\u7406 \u00a7 $ kubectl delete dataload spark-dataload","title":"\u6570\u636e\u9884\u52a0\u8f7d"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#_1","text":"\u914d\u7f6e\u5f85\u521b\u5efa\u7684DataLoad\u5bf9\u8c61 \u521b\u5efaDataLoad\u5bf9\u8c61 \u67e5\u770b\u521b\u5efa\u7684DataLoad\u5bf9\u8c61\u72b6\u6001 \u7b49\u5f85\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5b8c\u6210 DataLoad\u8fdb\u9636\u914d\u7f6e \u6307\u5b9a\u4e00\u4e2a\u6216\u591a\u4e2a\u6570\u636e\u96c6\u5b50\u76ee\u5f55\u8fdb\u884c\u52a0\u8f7d \u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf \u539f\u5b50\u6027\u7f13\u5b58 \u5355\u72ec\u8fdb\u884c\u5143\u6570\u636e\u7f13\u5b58\uff0c\u4e0d\u505a\u6570\u636e\u7f13\u5b58 \u7f13\u5b58\u8fdb\u5ea6\u53caLOG\u67e5\u770b \u73af\u5883\u6e05\u7406 \u4e3a\u4e86\u4fdd\u8bc1\u5e94\u7528\u5728\u8bbf\u95ee\u6570\u636e\u65f6\u7684\u6027\u80fd\uff0c\u53ef\u4ee5\u901a\u8fc7 \u6570\u636e\u9884\u52a0\u8f7d \u63d0\u524d\u5c06\u8fdc\u7a0b\u5b58\u50a8\u7cfb\u7edf\u4e2d\u7684\u6570\u636e\u62c9\u53d6\u5230\u9760\u8fd1\u8ba1\u7b97\u7ed3\u70b9\u7684\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d\uff0c\u4f7f\u5f97\u6d88\u8d39\u8be5\u6570\u636e\u96c6\u7684\u5e94\u7528\u80fd\u591f\u5728\u9996\u6b21\u8fd0\u884c\u65f6\u5373\u53ef\u4eab\u53d7\u5230\u7f13\u5b58\u5e26\u6765\u7684\u52a0\u901f\u6548\u679c\u3002 \u4e3a\u6b64\uff0c\u6211\u4eec\u63d0\u4f9b\u4e86DataLoad CRD, \u8be5CRD\u8ba9\u4f60\u53ef\u901a\u8fc7\u7b80\u5355\u7684\u914d\u7f6e\u5c31\u80fd\u5b8c\u6210\u6574\u4e2a\u6570\u636e\u9884\u52a0\u8f7d\u8fc7\u7a0b\uff0c\u5e76\u5bf9\u6570\u636e\u9884\u52a0\u8f7d\u7684\u8bb8\u591a\u884c\u4e3a\u8fdb\u884c\u81ea\u5b9a\u4e49\u63a7\u5236\u3002 \u5047\u8bbe\u6211\u4eec\u5df2\u7ecf\u6709\u5982\u4e0b\u4e00\u4e2aname\u4e3a spark \u7684dataset\uff1a apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: spark spec: mounts: - mountPoint: oss://test-bucket/spark name: sparkV1 accessModes: - ReadWriteMany \u63a5\u4e0b\u6765\u5bf9\u8fd9\u4e2adataset\u8fdb\u884c\u9884\u70ed\uff0c\u8bf7\u6ce8\u610f\u4e0b\u6587\u4e2d\u7684 /sparkV1 \u4e3a mounts[].name \u7684\u503c\uff0c\u5982\u679c\u60a8\u7684 mounts[].path \u4e3a / , \u90a3\u4e48\u4e0b\u6587\u4e2d\u7684 /sparkV1 \u53ef\u4ee5\u76f4\u63a5\u6539\u4e3a /","title":"\u6570\u636e\u9884\u52a0\u8f7d"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#dataload","text":"apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default target: - path: /sparkV1/ spec.dataset \u6307\u660e\u4e86\u9700\u8981\u8fdb\u884c\u6570\u636e\u9884\u52a0\u8f7d\u7684\u76ee\u6807\u6570\u636e\u96c6\uff0c\u5728\u8be5\u4f8b\u5b50\u4e2d\uff0c\u6211\u4eec\u7684\u6570\u636e\u9884\u52a0\u8f7d\u76ee\u6807\u4e3a default \u547d\u540d\u7a7a\u95f4\u4e0b\u540d\u4e3a spark \u7684\u6570\u636e\u96c6\uff0c\u5982\u679c\u8be5\u914d\u7f6e\u4e0e\u4f60\u6240\u5728\u7684\u5b9e\u9645\u73af\u5883\u4e0d\u7b26\uff0c\u8bf7\u6839\u636e\u4f60\u7684\u5b9e\u9645\u73af\u5883\u5bf9\u5176\u8fdb\u884c\u8c03\u6574\u3002","title":"\u914d\u7f6e\u5f85\u521b\u5efa\u7684DataLoad\u5bf9\u8c61"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#dataload_1","text":"kubectl create -f dataload.yaml","title":"\u521b\u5efaDataLoad\u5bf9\u8c61"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#dataload_2","text":"kubectl get dataload spark-dataload \u4e0a\u8ff0\u547d\u4ee4\u4f1a\u5f97\u5230\u7c7b\u4f3c\u4ee5\u4e0b\u7ed3\u679c\uff1a NAME DATASET PHASE AGE spark-dataload spark Loading 2m13s \u4f60\u4e5f\u53ef\u4ee5\u901a\u8fc7 kubectl describe \u83b7\u53d6\u6709\u5173\u8be5DataLoad\u7684\u66f4\u591a\u8be6\u7ec6\u4fe1\u606f\uff1a kubectl describe dataload spark-dataload \u5f97\u5230\u4ee5\u4e0b\u7ed3\u679c\uff1a Name: spark-dataload Namespace: default Labels: <none> Annotations: <none> API Version: data.fluid.io/v1alpha1 Kind: DataLoad ... Spec: Dataset: Name: spark Namespace: default Status: Conditions: Phase: Loading Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal DataLoadJobStarted 80s DataLoad The DataLoad job spark-dataload-loader-job started \u4e0a\u8ff0\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u6839\u636e\u4f60\u6240\u5728\u7684\u7f51\u7edc\u73af\u5883\u4e0d\u540c\uff0c\u53ef\u80fd\u4f1a\u8017\u8d39\u6570\u5206\u949f","title":"\u67e5\u770b\u521b\u5efa\u7684DataLoad\u5bf9\u8c61\u72b6\u6001"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#_2","text":"kubectl get dataload spark-dataload \u4f60\u4f1a\u770b\u5230\u8be5DataLoad\u7684 Phase \u72b6\u6001\u5df2\u7ecf\u4ece Loading \u53d8\u4e3a Complete \uff0c\u8fd9\u8868\u660e\u6574\u4e2a\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5df2\u7ecf\u5b8c\u6210 NAME DATASET PHASE AGE spark-dataload spark Complete 5m17s \u6b64\u65f6\u518d\u6b21\u67e5\u770bDataset\u5bf9\u8c61\u7684\u7f13\u5b58\u72b6\u6001\uff1a kubectl get dataset spark \u53ef\u53d1\u73b0\uff0c\u8fdc\u7a0b\u5b58\u50a8\u7cfb\u7edf\u4e2d\u7684\u5168\u90e8\u6570\u636e\u5747\u5df2\u6210\u529f\u7f13\u5b58\u5230\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE spark 1.92GiB 1.92GiB 4.00GiB 100.0% Bound 7m41s","title":"\u7b49\u5f85\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5b8c\u6210"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#dataload_3","text":"","title":"DataLoad\u8fdb\u9636\u914d\u7f6e"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#_3","text":"\u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6\u53ef\u4ee5\u52a0\u8f7d\u6307\u5b9a\u7684\u5b50\u76ee\u5f55(\u6216\u6587\u4ef6)\uff0c\u800c\u4e0d\u662f\u6574\u4e2a\u6570\u636e\u96c6\uff0c\u4f8b\u5982\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 - path: /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \u4e0a\u8ff0DataLoad\u4ec5\u4f1a\u52a0\u8f7d /sparkV1/spark-2.4.7 \u76ee\u5f55\u4e0b\u7684\u5168\u90e8\u6587\u4ef6\uff0c\u4ee5\u53ca /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \u6587\u4ef6","title":"\u6307\u5b9a\u4e00\u4e2a\u6216\u591a\u4e2a\u6570\u636e\u96c6\u5b50\u76ee\u5f55\u8fdb\u884c\u52a0\u8f7d"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#_4","text":"\u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6,\u4f60\u4e5f\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u63a7\u5236\u52a0\u8f7d\u7684\u6570\u636e\u526f\u672c\u6570\u91cf\uff0c\u4f8b\u5982\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 replicas: 1 - path: /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz replicas: 2 \u4e0a\u8ff0DataLoad\u5728\u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6\uff0c\u5bf9\u4e8e /sparkV1/spark-2.4.7 \u76ee\u5f55\u4e0b\u7684\u6587\u4ef6\u4ec5\u4f1a\u5728\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d\u4fdd\u7559 1\u4efd \u6570\u636e\u7f13\u5b58\u526f\u672c\uff0c\u800c\u5bf9\u4e8e\u6587\u4ef6 /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \uff0c\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u5c06\u4f1a\u4fdd\u7559 2\u4efd \u7f13\u5b58\u526f\u672c\u3002","title":"\u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#_5","text":"\u5982\u60a8\u60f3\u4fdd\u6301\u7f13\u5b58\u4e00\u81f4\u6027\u548c\u539f\u5b50\u6027\uff0c\u90a3\u4e48\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b yaml \u6765\u5b8c\u6210\u8fd9\u4e2a\u529f\u80fd apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default options: atomicCache: \"true\"","title":"\u539f\u5b50\u6027\u7f13\u5b58"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#_6","text":"\u5728\u8fdb\u884c dataload \u9884\u70ed\u65f6\uff0c\u9ed8\u8ba4\u662f\u4f1a\u505a\u6570\u636e\u7f13\u5b58\uff0c\u539f\u6570\u636e\u7f13\u5b58\u662f\u53ef\u9009\uff0c\u4f46\u5982\u679c\u60a8\u53ea\u60f3\u5143\u6570\u636e\u7f13\u5b58\u4e0d\u60f3\u505a\u6570\u636e\u7f13\u5b58\u7684\u8bdd\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b yaml \u6253\u5f00\u53ea\u505a\u5143\u6570\u636e\u7f13\u5b58\u7684\u5f00\u5173 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default options: loadMetadataOnly: \"true\"","title":"\u5355\u72ec\u8fdb\u884c\u5143\u6570\u636e\u7f13\u5b58\uff0c\u4e0d\u505a\u6570\u636e\u7f13\u5b58"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#_7","text":"apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 replicas: 1 options: filter: \"*.tgz\" cacheThread: \"2\" dryrun: \"false\"","title":"\u901a\u8fc7\u901a\u914d\u7b26\u8fdb\u884c\u7f13\u5b58\u8fc7\u6ee4"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#_8","text":"apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true options: accessKeyId: \"xxx\" accessKeySecret: \"xxx\" endpoint: \"oss-cn-shanghai-internal.aliyuncs.com\" url: \"oss://test-bucket/filelist\" dryrun: \"false\"","title":"\u4f7f\u7528\u5217\u8868\u6587\u4ef6\u8fdb\u884c\u7f13\u5b58"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#log","text":"$ kubectl get dataload spark-dataload NAME DATASET PHASE AGE spark-dataload spark Loading 2m13s \u67e5\u770b\u5f53\u524d\u7f13\u5b58\u7684\u76f8\u5173\u4fe1\u606f kubectl logs spark-dataload","title":"\u7f13\u5b58\u8fdb\u5ea6\u53caLOG\u67e5\u770b"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#_9","text":"$ kubectl delete dataload spark-dataload","title":"\u73af\u5883\u6e05\u7406"},{"location":"fluid-jindoruntime/jindo_fluid_download/","text":"\u4e0b\u8f7d 1.0.0-20240112 \u4e0b\u8f7d \u00a7 1.0.0-20240112 \u00a7 Fluid 1.0.0 JindoRuntime\u76f8\u5173\u529f\u80fd\u5347\u7ea7\uff1a * JindoCache \u5347\u7ea7\u5230 6.2.0","title":"Jindo fluid download"},{"location":"fluid-jindoruntime/jindo_fluid_download/#_1","text":"","title":"\u4e0b\u8f7d"},{"location":"fluid-jindoruntime/jindo_fluid_download/#100-20240112","text":"Fluid 1.0.0 JindoRuntime\u76f8\u5173\u529f\u80fd\u5347\u7ea7\uff1a * JindoCache \u5347\u7ea7\u5230 6.2.0","title":"1.0.0-20240112"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/","text":"\u52a0\u901fHDFS\u4e0a\u6570\u636e \u00a7 JindoFSx \u63d0\u4f9b\u4e86\u8bbf\u95ee HDFS \u7684\u80fd\u529b\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e HDFS \u4f5c\u4e3a JindoFSx \u7684\u540e\u7aef\u5b58\u50a8\uff0c\u4f7f Fluid \u901a\u8fc7 JindoFSx \u6765\u8bbf\u95ee HDFS \u4e0a\u7684\u6570\u636e\uff0c\u540c\u65f6 JindoFSx \u4e5f\u63d0\u4f9b\u4e86\u5bf9 HDFS \u4e0a\u7684\u6570\u636e\u4ee5\u53ca\u5143\u6570\u636e\u7684\u7f13\u5b58\u52a0\u901f\u529f\u80fd\u3002 \u672c\u6587\u6863\u5c55\u793a\u4e86\u5982\u4f55\u5728 Fluid \u4e2d\u4ee5\u58f0\u660e\u5f0f\u7684\u65b9\u5f0f\u5b8c\u6210 JindoFSx \u90e8\u7f72\uff0c\u5bf9\u63a5 HDFS \u6570\u636e\u6e90\u3002 \u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 \u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz \u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 1\u3001\u521b\u5efa dataset \u548c JindoRuntime \u00a7 1.1\u3001HA \u96c6\u7fa4\u4ece HDFS \u914d\u7f6e\u6587\u4ef6\u4e2d\u76f8\u5173\u4fe1\u606f\uff0c\u975e HA \u96c6\u7fa4\u8bf7\u7565\u8fc7\u6b64\u6b65\u9aa4 \u00a7 \u4ece HDFS \u96c6\u7fa4\u4e0a\u627e\u5230 hdfs-site.xml \u914d\u7f6e\u6587\u4ef6 * \u963f\u91cc\u4e91EMR\u96c6\u7fa4\u5728 /etc/ecm/hadoop-conf \u8def\u5f84\u4e0b * CDH \u96c6\u7fa4\u53ef\u53c2\u8003 What-is-the-Path-of-hdfs-site-xml-core-xml \u9700\u8981\u5c06\u5176\u4e2d rpc \u7684\u8bbf\u95ee\u5730\u5740\u4e2d\u57df\u540d\u7684\u90e8\u5206\u53c2\u6570\uff0c\u4fee\u6539\u6210\u53ef\u8bbf\u95ee\u7684\u5185\u7f51 IP \u5730\u5740\uff0c\u4f8b\u5982\uff1a <property> <name>dfs.namenode.rpc-address.emr-cluster.nn1</name> <value>emr-header-1.cluster-xxx:8020</value> </property> <property> <name>dfs.namenode.rpc-address.emr-cluster.nn2</name> <value>emr-header-2.cluster-xxx:8020</value> </property> \u4fee\u6539\u4e3a <property> <name>dfs.namenode.rpc-address.emr-cluster.nn1</name> <value>192.168.0.1:8020</value> </property> <property> <name>dfs.namenode.rpc-address.emr-cluster.nn2</name> <value>192.168.0.2:8020</value> </property> \u4ece\u914d\u7f6e\u6587\u4ef6\u91cc\u6211\u4eec\u53ef\u4ee5\u62ff\u5230\u5982\u4e0b\u51e0\u4e2a\u4fe1\u606f \u96c6\u7fa4namespaceService\u540d\u79f0\uff1aemr-cluster \u4e24\u4e2anamenode\u540d\u5b57\uff1ann1 \u548c nn2 \u4e24\u4e2anamenode\u7684\u8bbf\u95ee\u5730\u5740\uff1a192.168.0.1:8020 \u548c 192.168.0.2:8020 1.2\u3001\u521b\u5efa Dataset \u8d44\u6e90\u5bf9\u8c61 \u00a7 \u9996\u5148\u521b\u5efa\u4e00\u4e2a\u540d\u4e3a dataset.yaml \u7684\u6587\u4ef6\uff0c\u9488\u5bf9 HDFS \u96c6\u7fa4\u5206\u4e3a HA \u96c6\u7fa4\u548c\u975e HA \u96c6\u7fa4\u4e24\u79cd\u4e0d\u540c\u7684\u60c5\u51b5\u3002 HA \u96c6\u7fa4 \u00a7 \u5c061.1\u4e2d\u62ff\u5230\u7684\u51e0\u4e2a\u4fe1\u606f\u653e\u5230 mounts.options \u91cc\u9762\uff0cuser \u4ee3\u8868\u7528\u6237\u6743\u9650 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hdfs spec: mounts: - mountPoint: hdfs://emr-cluster/ name: hdfs options: fs.hdfs.emr-cluster.dfs.ha.namenodes = nn1,nn2 fs.hdfs.emr-cluster.dfs.namenode.rpc-address.nn1 = 192.168.0.1:8020 fs.hdfs.emr-cluster.dfs.namenode.rpc-address.nn2 = 192.168.0.2:8020 fs.hdfs.user = hadoop path: / accessModes: - ReadOnlyMany \u5176\u4e2d mountPoint \u4e3a\u9700\u8981\u8bbf\u95ee\u7684 HDFS \u8def\u5f84\uff08\u963f\u91cc\u4e91EMR\u96c6\u7fa4\u9ed8\u8ba4\u540d\u5b57\u4e3a emr-cluster\uff0c\u5982\u4e0b\u6240\u793a\uff09 - accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 <property> <name>fs.defaultFS</name> <value>hdfs://emr-cluster</value> </property> \u975e HA \u96c6\u7fa4 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hdfs spec: mounts: - mountPoint: hdfs://<namenodeIP>:<port>/ name: hdfs path: / accessModes: - ReadOnlyMany \u5176\u4e2d mountPoint \u4e3a HDFS \u96c6\u7fa4\u7684 namenode \u7684 IP \u5730\u5740\u548c\u7aef\u53e3\u7684\u7ec4\u5408 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 \u00a7 1.3\u3001\u521b\u5efa Dataset \u00a7 $ kubectl create -f dataset.yaml 1.4 \u521b\u5efa JindoRuntime \u8d44\u6e90\u5bf9\u8c61 \u00a7 \u521b\u5efa\u4e00\u4e2a JindoRuntime \u5bf9\u5e94\u7684 JindoFSx \u96c6\u7fa4 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hdfs spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /mnt/disk1/ quota: 290Gi high: \"0.9\" low: \"0.2\" \u591a\u5757\u76d8\u591a\u4e2aquota\u8bf7\u53c2\u8003 path: /mnt/disk1/,/mnt/disk2/,/mnt/disk3/ quotaList: 290G,290G,290G \u5176\u4e2d path \u548c quota \u7684\u6570\u91cf\u5e94\u8be5\u76f8\u540c\u3002 1.5\u3001\u521b\u5efa JindoRuntime \u00a7 \u6267\u884c create \u547d\u4ee4 $ kubectl create -f runtime.yaml \u67e5\u770b dataset \u72b6\u6001 \u00a7 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hdfs 180.01GiB 0.00B 261.00GiB 0.0% Bound 62m \u68c0\u67e5 PV\uff0cPVC \u521b\u5efa\u60c5\u51b5 \u00a7 $ kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hdfs 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hdfs Bound hadoop 100Gi RWX 58m \u68c0\u67e5 jindoruntime \u90e8\u7f72\u60c5\u51b5 \u00a7 $ kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hdfs Ready Ready 62m","title":"\u52a0\u901fHDFS\u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#hdfs","text":"JindoFSx \u63d0\u4f9b\u4e86\u8bbf\u95ee HDFS \u7684\u80fd\u529b\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e HDFS \u4f5c\u4e3a JindoFSx \u7684\u540e\u7aef\u5b58\u50a8\uff0c\u4f7f Fluid \u901a\u8fc7 JindoFSx \u6765\u8bbf\u95ee HDFS \u4e0a\u7684\u6570\u636e\uff0c\u540c\u65f6 JindoFSx \u4e5f\u63d0\u4f9b\u4e86\u5bf9 HDFS \u4e0a\u7684\u6570\u636e\u4ee5\u53ca\u5143\u6570\u636e\u7684\u7f13\u5b58\u52a0\u901f\u529f\u80fd\u3002 \u672c\u6587\u6863\u5c55\u793a\u4e86\u5982\u4f55\u5728 Fluid \u4e2d\u4ee5\u58f0\u660e\u5f0f\u7684\u65b9\u5f0f\u5b8c\u6210 JindoFSx \u90e8\u7f72\uff0c\u5bf9\u63a5 HDFS \u6570\u636e\u6e90\u3002","title":"\u52a0\u901fHDFS\u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#_1","text":"kubectl create ns fluid-system","title":"\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#fluid","text":"","title":"\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#fluid_1","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#1-dataset-jindoruntime","text":"","title":"1\u3001\u521b\u5efa dataset \u548c JindoRuntime"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#11ha-hdfs-ha","text":"\u4ece HDFS \u96c6\u7fa4\u4e0a\u627e\u5230 hdfs-site.xml \u914d\u7f6e\u6587\u4ef6 * \u963f\u91cc\u4e91EMR\u96c6\u7fa4\u5728 /etc/ecm/hadoop-conf \u8def\u5f84\u4e0b * CDH \u96c6\u7fa4\u53ef\u53c2\u8003 What-is-the-Path-of-hdfs-site-xml-core-xml \u9700\u8981\u5c06\u5176\u4e2d rpc \u7684\u8bbf\u95ee\u5730\u5740\u4e2d\u57df\u540d\u7684\u90e8\u5206\u53c2\u6570\uff0c\u4fee\u6539\u6210\u53ef\u8bbf\u95ee\u7684\u5185\u7f51 IP \u5730\u5740\uff0c\u4f8b\u5982\uff1a <property> <name>dfs.namenode.rpc-address.emr-cluster.nn1</name> <value>emr-header-1.cluster-xxx:8020</value> </property> <property> <name>dfs.namenode.rpc-address.emr-cluster.nn2</name> <value>emr-header-2.cluster-xxx:8020</value> </property> \u4fee\u6539\u4e3a <property> <name>dfs.namenode.rpc-address.emr-cluster.nn1</name> <value>192.168.0.1:8020</value> </property> <property> <name>dfs.namenode.rpc-address.emr-cluster.nn2</name> <value>192.168.0.2:8020</value> </property> \u4ece\u914d\u7f6e\u6587\u4ef6\u91cc\u6211\u4eec\u53ef\u4ee5\u62ff\u5230\u5982\u4e0b\u51e0\u4e2a\u4fe1\u606f \u96c6\u7fa4namespaceService\u540d\u79f0\uff1aemr-cluster \u4e24\u4e2anamenode\u540d\u5b57\uff1ann1 \u548c nn2 \u4e24\u4e2anamenode\u7684\u8bbf\u95ee\u5730\u5740\uff1a192.168.0.1:8020 \u548c 192.168.0.2:8020","title":"1.1\u3001HA \u96c6\u7fa4\u4ece HDFS \u914d\u7f6e\u6587\u4ef6\u4e2d\u76f8\u5173\u4fe1\u606f\uff0c\u975e HA \u96c6\u7fa4\u8bf7\u7565\u8fc7\u6b64\u6b65\u9aa4"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#12-dataset","text":"\u9996\u5148\u521b\u5efa\u4e00\u4e2a\u540d\u4e3a dataset.yaml \u7684\u6587\u4ef6\uff0c\u9488\u5bf9 HDFS \u96c6\u7fa4\u5206\u4e3a HA \u96c6\u7fa4\u548c\u975e HA \u96c6\u7fa4\u4e24\u79cd\u4e0d\u540c\u7684\u60c5\u51b5\u3002","title":"1.2\u3001\u521b\u5efa Dataset \u8d44\u6e90\u5bf9\u8c61"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#ha","text":"\u5c061.1\u4e2d\u62ff\u5230\u7684\u51e0\u4e2a\u4fe1\u606f\u653e\u5230 mounts.options \u91cc\u9762\uff0cuser \u4ee3\u8868\u7528\u6237\u6743\u9650 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hdfs spec: mounts: - mountPoint: hdfs://emr-cluster/ name: hdfs options: fs.hdfs.emr-cluster.dfs.ha.namenodes = nn1,nn2 fs.hdfs.emr-cluster.dfs.namenode.rpc-address.nn1 = 192.168.0.1:8020 fs.hdfs.emr-cluster.dfs.namenode.rpc-address.nn2 = 192.168.0.2:8020 fs.hdfs.user = hadoop path: / accessModes: - ReadOnlyMany \u5176\u4e2d mountPoint \u4e3a\u9700\u8981\u8bbf\u95ee\u7684 HDFS \u8def\u5f84\uff08\u963f\u91cc\u4e91EMR\u96c6\u7fa4\u9ed8\u8ba4\u540d\u5b57\u4e3a emr-cluster\uff0c\u5982\u4e0b\u6240\u793a\uff09 - accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 <property> <name>fs.defaultFS</name> <value>hdfs://emr-cluster</value> </property>","title":"HA \u96c6\u7fa4"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#ha_1","text":"apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hdfs spec: mounts: - mountPoint: hdfs://<namenodeIP>:<port>/ name: hdfs path: / accessModes: - ReadOnlyMany \u5176\u4e2d mountPoint \u4e3a HDFS \u96c6\u7fa4\u7684 namenode \u7684 IP \u5730\u5740\u548c\u7aef\u53e3\u7684\u7ec4\u5408 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199","title":"\u975e HA \u96c6\u7fa4"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#_2","text":"","title":""},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#13-dataset","text":"$ kubectl create -f dataset.yaml","title":"1.3\u3001\u521b\u5efa Dataset"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#14-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a JindoRuntime \u5bf9\u5e94\u7684 JindoFSx \u96c6\u7fa4 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hdfs spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /mnt/disk1/ quota: 290Gi high: \"0.9\" low: \"0.2\" \u591a\u5757\u76d8\u591a\u4e2aquota\u8bf7\u53c2\u8003 path: /mnt/disk1/,/mnt/disk2/,/mnt/disk3/ quotaList: 290G,290G,290G \u5176\u4e2d path \u548c quota \u7684\u6570\u91cf\u5e94\u8be5\u76f8\u540c\u3002","title":"1.4 \u521b\u5efa JindoRuntime \u8d44\u6e90\u5bf9\u8c61"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#15-jindoruntime","text":"\u6267\u884c create \u547d\u4ee4 $ kubectl create -f runtime.yaml","title":"1.5\u3001\u521b\u5efa JindoRuntime"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#dataset","text":"$ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hdfs 180.01GiB 0.00B 261.00GiB 0.0% Bound 62m","title":"\u67e5\u770b dataset \u72b6\u6001"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#pvpvc","text":"$ kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hdfs 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hdfs Bound hadoop 100Gi RWX 58m","title":"\u68c0\u67e5 PV\uff0cPVC \u521b\u5efa\u60c5\u51b5"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#jindoruntime","text":"$ kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hdfs Ready Ready 62m","title":"\u68c0\u67e5 jindoruntime \u90e8\u7f72\u60c5\u51b5"},{"location":"fluid-jindoruntime/jindo_fluid_monitor_metric/","text":"\u76d1\u63a7\u548c\u53ef\u89c2\u6d4b\u6027\u529f\u80fd \u00a7 \u672c\u6587\u4ecb\u7ecd\u5728\u542f\u52a8 Fluid JindoRuntime \u7684\u524d\u63d0\u4e0b\uff0c\u901a\u8fc7 Prometheus + Grafana \u6536\u96c6\u7f13\u5b58\u7cfb\u7edf\u76f8\u5173\u7684 metrics \u6307\u6807\uff0c\u8fdb\u884c\u53ef\u89c6\u5316\u5c55\u793a\u7684\u4f7f\u7528\u65b9\u6cd5\u3002 \u4f7f\u7528 In-Cluster \u6a21\u5f0f\u90e8\u7f72 prometheus \u548c grafana \u76d1\u63a7 Fluid \u5e94\u7528 \u00a7 1. \u90e8\u7f72\u6216\u914d\u7f6e Prometheus \u00a7 \u5982\u679c\u96c6\u7fa4\u5185\u65e0 prometheus\uff0c\u8bf7\u70b9\u51fb\u4e0b\u8f7d prometheus-jindofs.yaml \u6587\u4ef6 $ kubectl apply -f prometheus-jindofs.yaml \u5982\u96c6\u7fa4\u5185\u6709 prometheus,\u53ef\u5c06\u4ee5\u4e0b\u914d\u7f6e\u5199\u5230 prometheus \u914d\u7f6e\u6587\u4ef6\u4e2d: scrape_configs: - job_name: 'jindo runtime' metrics_path: /jindometrics/prometheus kubernetes_sd_configs: - role: endpoints relabel_configs: - source_labels: [__meta_kubernetes_service_label_role] regex: jindofs-master action: keep - source_labels: [__meta_kubernetes_endpoint_port_name] regex: rpc action: keep - source_labels: [__meta_kubernetes_namespace] target_label: namespace replacement: $1 action: replace - source_labels: [__meta_kubernetes_service_label_release] target_label: fluid_runtime replacement: $1 action: replace - source_labels: [__meta_kubernetes_endpoint_address_target_name] target_label: pod replacement: $1 action: replace 2. \u90e8\u7f72 grafana \u00a7 \u5982\u679c\u96c6\u7fa4\u5185\u65e0 grafana\uff0c\u8bf7\u70b9\u51fb\u4e0b\u8f7d grafana.yaml \u6587\u4ef6 $ kubectl apply -f grafana.yaml 3. \u914d\u7f6e grafana \u00a7 \u767b\u5f55 grafana \u627e\u5230 grafana \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\u548c\u6539\u8282\u70b9\u5bf9\u5916\u53ef\u8bbf\u95ee grafana-node-ip \uff0c\u5982\u679c\u60f3\u8981\u5728\u516c\u7f51\u8bbf\u95ee\uff0c\u53ef\u4ee5\u7ed1\u5b9a\u516c\u7f51IP\uff0c $ kubectl get pod -n kube-system -o wide | grep grafana monitoring-grafana-7dfcf6d5cd-5k8qw 1/1 Running 0 108m 10.54.1.11 cn-shanghai.192.168.1.1 <none> <none> \u627e\u5230 grafana \u670d\u52a1\u5bf9\u5e94\u7684 NodePort $ kubectl describe svc monitoring-grafana -n kube-system Name: monitoring-grafana Namespace: kube-system Labels: <none> Annotations: <none> Selector: app=grafana Type: NodePort IP Families: <none> IP: 172.16.83.188 IPs: 172.16.83.188 Port: <unset> 80/TCP TargetPort: 3000/TCP NodePort: <unset> 30396/TCP Endpoints: 10.54.1.75:3000 Session Affinity: None External Traffic Policy: Cluster Events: <none> \u53ef\u4ee5\u770b\u5230\u5f53\u524d\u66b4\u9732\u7684 NodePort \u4e3a 30396 \u8bbf\u95ee http://$grafana-node-ip:NodePort \uff0c\u9ed8\u8ba4\u8d26\u53f7\u5bc6\u7801 admin:admin \u67e5\u770b prometheus svc \u7aef\u53e3 \u540c\u6837\u5148\u627e\u5230 prometheus \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\uff0c\u627e\u5230\u8be5\u8282\u70b9\u7684 IP $ kubectl get pod -n kube-system -o wide | grep grafana prometheus-deployment-d6d8554b4-wfl98 1/1 Running 0 17d 10.54.1.204 cn-shanghai.192.168.1.1 <none> <none> kubectl get no cn-shanghai.192.168.31.107 -o wide NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP cn-shanghai.192.168.1.1 Ready <none> 5h51m v1.20.4-aliyun.1 192.168.1.1 101.132.66.11 \u901a\u8fc7\u67e5\u770b\u8282\u70b9\u7684 INTERNAL-IP \uff0c\u53ef\u4e86\u89e3\u8be5\u8282\u70b9\u7684\u5185\u90e8\u8bbf\u95ee IP \u4e3a 192.168.1.1 $ kubectl get svc -n kube-system | grep prometheus-svc prometheus-svc NodePort 10.100.0.144 <none> 9090:31225/TCP 22h $ kubectl describe svc prometheus-svc -n kube-system Name: prometheus-svc Namespace: kube-system Labels: kubernetes.io/name=Prometheus name=prometheus-svc Annotations: kubectl.kubernetes.io/last-applied-configuration: {\"apiVersion\":\"v1\",\"kind\":\"Service\",\"metadata\":{\"annotations\":{},\"labels\":{\"kubernetes.io/name\":\"Prometheus\",\"name\":\"prometheus-svc\"},\"nam... Selector: app=prometheus Type: NodePort IP: 10.100.0.144 Port: prometheus 9090/TCP TargetPort: 9090/TCP NodePort: prometheus 31225/TCP Endpoints: 10.99.224.138:9090 Session Affinity: None External Traffic Policy: Cluster Events: <none> \u53ef\u4ee5\u770b\u5230\u5f53\u524d prometheus svc \u66b4\u9732\u7684\u7aef\u53e3\u662f 31225 \uff0c\u540e\u7eed\u4f7f\u7528 http://IP:NodePort \u7684\u5f62\u5f0f\u6765\u53d1\u73b0 prometheus \u670d\u52a1 \u914d\u7f6e prometheus data source \u4f7f\u7528 http://IP:NodePort \u7684\u5f62\u5f0f\u6765\u53d1\u73b0 prometheus \u670d\u52a1 \u5bfc\u5165\u5b8c\u6210\u540e\u70b9\u51fbSave & Test \u663e\u793a Data source is working \u5373\u53ef \u5bfc\u5165\u6a21\u677f\u6587\u4ef6 grafana \u9009\u62e9\u5bfc\u5165\u6a21\u677f Json \u6587\u4ef6\uff0c\u70b9\u51fb\u6b64\u5904\u4e0b\u8f7d fluid-prometheus-grafana-monitor-jindofs.json \u67e5\u770b\u76d1\u63a7 \u5728 DashBoards \u4e2d\u627e\u5230\u5bfc\u5165\u7684\u6a21\u7248\u89c6\u56fe \u6ce8\uff1afluid_runtime \u5bf9\u5e94Fluid runtime name; namespace \u5bf9\u5e94 Fluid runtime namespace JindoRuntime Prometheus \u6307\u6807\u9884\u89c8(\u66f4\u65b0\u4e2d) \u00a7 \u4e0b\u9762\u5bf9 JindoRuntime \u5f53\u524d\u6536\u96c6\u7684\u6307\u6807\u8fdb\u884c\u9884\u89c8\u548c\u63cf\u8ff0\uff0c\u60a8\u53ef\u4ee5\u57fa\u4e8e\u5df2\u6536\u96c6\u6307\u6807\u5728 Grafana \u4e2d\u8fdb\u884c\u7ec4\u5408\u548c\u56fe\u8868\u5c55\u793a\uff0c\u5982\u60a8\u9700\u8981\u989d\u5916\u7684\u57fa\u672c metrics \u5c55\u793a\uff0c\u8bf7\u5f00 issue \u8054\u7cfb\u6211\u4eec # HELP jindofsx_ns_backend_read_bytes_time_total_window # TYPE jindofsx_ns_backend_read_bytes_time_total_window gauge jindofsx_ns_backend_read_bytes_time_total_window 20367553 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_read_bytes_total # TYPE jindofsx_ns_backend_read_bytes_total gauge jindofsx_ns_backend_read_bytes_total 166703347021 #\u603b\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_bytes_total_window # TYPE jindofsx_ns_backend_read_bytes_total_window gauge jindofsx_ns_backend_read_bytes_total_window 1174671374 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_time_total # TYPE jindofsx_ns_backend_read_time_total gauge jindofsx_ns_backend_read_time_total 8717632094 #\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_readop_num_total # TYPE jindofsx_ns_backend_readop_num_total gauge jindofsx_ns_backend_readop_num_total 181951 #\u5e95\u5c42\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_local_read_bytes_time_total_window # TYPE jindofsx_ns_local_read_bytes_time_total_window gauge jindofsx_ns_local_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_bytes_total # TYPE jindofsx_ns_local_read_bytes_total gauge jindofsx_ns_local_read_bytes_total 0 #\u603b\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_local_read_bytes_total_window # TYPE jindofsx_ns_local_read_bytes_total_window gauge jindofsx_ns_local_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_time_total # TYPE jindofsx_ns_local_read_time_total gauge jindofsx_ns_local_read_time_total 0 #\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_readop_num_total # TYPE jindofsx_ns_local_readop_num_total gauge jindofsx_ns_local_readop_num_total 0 #\u77ed\u8def\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_remote_read_bytes_time_total_window # TYPE jindofsx_ns_remote_read_bytes_time_total_window gauge jindofsx_ns_remote_read_bytes_time_total_window 73714663 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_read_bytes_total # TYPE jindofsx_ns_remote_read_bytes_total gauge jindofsx_ns_remote_read_bytes_total 112108045498 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_bytes_total_window # TYPE jindofsx_ns_remote_read_bytes_total_window gauge jindofsx_ns_remote_read_bytes_total_window 13380884043 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_time_total # TYPE jindofsx_ns_remote_read_time_total gauge jindofsx_ns_remote_read_time_total 765272359 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u65f6\u95f4\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_readop_num_total # TYPE jindofsx_ns_remote_readop_num_total gauge jindofsx_ns_remote_readop_num_total 107314 #\u8fdc\u7aef\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_slicelet_read_bytes_time_total_window # TYPE jindofsx_ns_slicelet_read_bytes_time_total_window gauge jindofsx_ns_slicelet_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_read_bytes_total # TYPE jindofsx_ns_slicelet_read_bytes_total gauge jindofsx_ns_slicelet_read_bytes_total 0 #\u603b\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_bytes_total_window # TYPE jindofsx_ns_slicelet_read_bytes_total_window gauge jindofsx_ns_slicelet_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_time_total # TYPE jindofsx_ns_slicelet_read_time_total gauge jindofsx_ns_slicelet_read_time_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_readop_num_total # TYPE jindofsx_ns_slicelet_readop_num_total gauge jindofsx_ns_slicelet_readop_num_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_total_disk_cap # TYPE jindofsx_ns_total_disk_cap gauge jindofsx_ns_total_disk_cap 840739848192 #\u78c1\u76d8\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_mem_cap # TYPE jindofsx_ns_total_mem_cap gauge jindofsx_ns_total_mem_cap 0 #\u5185\u5b58\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_stsnodes_num # TYPE jindofsx_ns_total_stsnodes_num gauge jindofsx_ns_total_stsnodes_num 3 #alive worker \u8282\u70b9\u7684\u6570\u91cf # HELP jindofsx_ns_total_used_disk_cap # TYPE jindofsx_ns_total_used_disk_cap gauge jindofsx_ns_total_used_disk_cap 153423446016 #\u5df2\u7528\u78c1\u76d8\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_used_mem_cap # TYPE jindofsx_ns_total_used_mem_cap gauge jindofsx_ns_total_used_mem_cap 0 #\u5df2\u7528\u5185\u5b58\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte","title":"\u76d1\u63a7\u548c\u53ef\u89c2\u6d4b\u6027\u529f\u80fd"},{"location":"fluid-jindoruntime/jindo_fluid_monitor_metric/#_1","text":"\u672c\u6587\u4ecb\u7ecd\u5728\u542f\u52a8 Fluid JindoRuntime \u7684\u524d\u63d0\u4e0b\uff0c\u901a\u8fc7 Prometheus + Grafana \u6536\u96c6\u7f13\u5b58\u7cfb\u7edf\u76f8\u5173\u7684 metrics \u6307\u6807\uff0c\u8fdb\u884c\u53ef\u89c6\u5316\u5c55\u793a\u7684\u4f7f\u7528\u65b9\u6cd5\u3002","title":"\u76d1\u63a7\u548c\u53ef\u89c2\u6d4b\u6027\u529f\u80fd"},{"location":"fluid-jindoruntime/jindo_fluid_monitor_metric/#in-cluster-prometheus-grafana-fluid","text":"","title":"\u4f7f\u7528 In-Cluster \u6a21\u5f0f\u90e8\u7f72 prometheus \u548c grafana \u76d1\u63a7 Fluid \u5e94\u7528"},{"location":"fluid-jindoruntime/jindo_fluid_monitor_metric/#1-prometheus","text":"\u5982\u679c\u96c6\u7fa4\u5185\u65e0 prometheus\uff0c\u8bf7\u70b9\u51fb\u4e0b\u8f7d prometheus-jindofs.yaml \u6587\u4ef6 $ kubectl apply -f prometheus-jindofs.yaml \u5982\u96c6\u7fa4\u5185\u6709 prometheus,\u53ef\u5c06\u4ee5\u4e0b\u914d\u7f6e\u5199\u5230 prometheus \u914d\u7f6e\u6587\u4ef6\u4e2d: scrape_configs: - job_name: 'jindo runtime' metrics_path: /jindometrics/prometheus kubernetes_sd_configs: - role: endpoints relabel_configs: - source_labels: [__meta_kubernetes_service_label_role] regex: jindofs-master action: keep - source_labels: [__meta_kubernetes_endpoint_port_name] regex: rpc action: keep - source_labels: [__meta_kubernetes_namespace] target_label: namespace replacement: $1 action: replace - source_labels: [__meta_kubernetes_service_label_release] target_label: fluid_runtime replacement: $1 action: replace - source_labels: [__meta_kubernetes_endpoint_address_target_name] target_label: pod replacement: $1 action: replace","title":"1. \u90e8\u7f72\u6216\u914d\u7f6e Prometheus"},{"location":"fluid-jindoruntime/jindo_fluid_monitor_metric/#2-grafana","text":"\u5982\u679c\u96c6\u7fa4\u5185\u65e0 grafana\uff0c\u8bf7\u70b9\u51fb\u4e0b\u8f7d grafana.yaml \u6587\u4ef6 $ kubectl apply -f grafana.yaml","title":"2. \u90e8\u7f72 grafana"},{"location":"fluid-jindoruntime/jindo_fluid_monitor_metric/#3-grafana","text":"\u767b\u5f55 grafana \u627e\u5230 grafana \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\u548c\u6539\u8282\u70b9\u5bf9\u5916\u53ef\u8bbf\u95ee grafana-node-ip \uff0c\u5982\u679c\u60f3\u8981\u5728\u516c\u7f51\u8bbf\u95ee\uff0c\u53ef\u4ee5\u7ed1\u5b9a\u516c\u7f51IP\uff0c $ kubectl get pod -n kube-system -o wide | grep grafana monitoring-grafana-7dfcf6d5cd-5k8qw 1/1 Running 0 108m 10.54.1.11 cn-shanghai.192.168.1.1 <none> <none> \u627e\u5230 grafana \u670d\u52a1\u5bf9\u5e94\u7684 NodePort $ kubectl describe svc monitoring-grafana -n kube-system Name: monitoring-grafana Namespace: kube-system Labels: <none> Annotations: <none> Selector: app=grafana Type: NodePort IP Families: <none> IP: 172.16.83.188 IPs: 172.16.83.188 Port: <unset> 80/TCP TargetPort: 3000/TCP NodePort: <unset> 30396/TCP Endpoints: 10.54.1.75:3000 Session Affinity: None External Traffic Policy: Cluster Events: <none> \u53ef\u4ee5\u770b\u5230\u5f53\u524d\u66b4\u9732\u7684 NodePort \u4e3a 30396 \u8bbf\u95ee http://$grafana-node-ip:NodePort \uff0c\u9ed8\u8ba4\u8d26\u53f7\u5bc6\u7801 admin:admin \u67e5\u770b prometheus svc \u7aef\u53e3 \u540c\u6837\u5148\u627e\u5230 prometheus \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\uff0c\u627e\u5230\u8be5\u8282\u70b9\u7684 IP $ kubectl get pod -n kube-system -o wide | grep grafana prometheus-deployment-d6d8554b4-wfl98 1/1 Running 0 17d 10.54.1.204 cn-shanghai.192.168.1.1 <none> <none> kubectl get no cn-shanghai.192.168.31.107 -o wide NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP cn-shanghai.192.168.1.1 Ready <none> 5h51m v1.20.4-aliyun.1 192.168.1.1 101.132.66.11 \u901a\u8fc7\u67e5\u770b\u8282\u70b9\u7684 INTERNAL-IP \uff0c\u53ef\u4e86\u89e3\u8be5\u8282\u70b9\u7684\u5185\u90e8\u8bbf\u95ee IP \u4e3a 192.168.1.1 $ kubectl get svc -n kube-system | grep prometheus-svc prometheus-svc NodePort 10.100.0.144 <none> 9090:31225/TCP 22h $ kubectl describe svc prometheus-svc -n kube-system Name: prometheus-svc Namespace: kube-system Labels: kubernetes.io/name=Prometheus name=prometheus-svc Annotations: kubectl.kubernetes.io/last-applied-configuration: {\"apiVersion\":\"v1\",\"kind\":\"Service\",\"metadata\":{\"annotations\":{},\"labels\":{\"kubernetes.io/name\":\"Prometheus\",\"name\":\"prometheus-svc\"},\"nam... Selector: app=prometheus Type: NodePort IP: 10.100.0.144 Port: prometheus 9090/TCP TargetPort: 9090/TCP NodePort: prometheus 31225/TCP Endpoints: 10.99.224.138:9090 Session Affinity: None External Traffic Policy: Cluster Events: <none> \u53ef\u4ee5\u770b\u5230\u5f53\u524d prometheus svc \u66b4\u9732\u7684\u7aef\u53e3\u662f 31225 \uff0c\u540e\u7eed\u4f7f\u7528 http://IP:NodePort \u7684\u5f62\u5f0f\u6765\u53d1\u73b0 prometheus \u670d\u52a1 \u914d\u7f6e prometheus data source \u4f7f\u7528 http://IP:NodePort \u7684\u5f62\u5f0f\u6765\u53d1\u73b0 prometheus \u670d\u52a1 \u5bfc\u5165\u5b8c\u6210\u540e\u70b9\u51fbSave & Test \u663e\u793a Data source is working \u5373\u53ef \u5bfc\u5165\u6a21\u677f\u6587\u4ef6 grafana \u9009\u62e9\u5bfc\u5165\u6a21\u677f Json \u6587\u4ef6\uff0c\u70b9\u51fb\u6b64\u5904\u4e0b\u8f7d fluid-prometheus-grafana-monitor-jindofs.json \u67e5\u770b\u76d1\u63a7 \u5728 DashBoards \u4e2d\u627e\u5230\u5bfc\u5165\u7684\u6a21\u7248\u89c6\u56fe \u6ce8\uff1afluid_runtime \u5bf9\u5e94Fluid runtime name; namespace \u5bf9\u5e94 Fluid runtime namespace","title":"3. \u914d\u7f6e grafana"},{"location":"fluid-jindoruntime/jindo_fluid_monitor_metric/#jindoruntime-prometheus","text":"\u4e0b\u9762\u5bf9 JindoRuntime \u5f53\u524d\u6536\u96c6\u7684\u6307\u6807\u8fdb\u884c\u9884\u89c8\u548c\u63cf\u8ff0\uff0c\u60a8\u53ef\u4ee5\u57fa\u4e8e\u5df2\u6536\u96c6\u6307\u6807\u5728 Grafana \u4e2d\u8fdb\u884c\u7ec4\u5408\u548c\u56fe\u8868\u5c55\u793a\uff0c\u5982\u60a8\u9700\u8981\u989d\u5916\u7684\u57fa\u672c metrics \u5c55\u793a\uff0c\u8bf7\u5f00 issue \u8054\u7cfb\u6211\u4eec # HELP jindofsx_ns_backend_read_bytes_time_total_window # TYPE jindofsx_ns_backend_read_bytes_time_total_window gauge jindofsx_ns_backend_read_bytes_time_total_window 20367553 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_read_bytes_total # TYPE jindofsx_ns_backend_read_bytes_total gauge jindofsx_ns_backend_read_bytes_total 166703347021 #\u603b\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_bytes_total_window # TYPE jindofsx_ns_backend_read_bytes_total_window gauge jindofsx_ns_backend_read_bytes_total_window 1174671374 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_time_total # TYPE jindofsx_ns_backend_read_time_total gauge jindofsx_ns_backend_read_time_total 8717632094 #\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_readop_num_total # TYPE jindofsx_ns_backend_readop_num_total gauge jindofsx_ns_backend_readop_num_total 181951 #\u5e95\u5c42\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_local_read_bytes_time_total_window # TYPE jindofsx_ns_local_read_bytes_time_total_window gauge jindofsx_ns_local_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_bytes_total # TYPE jindofsx_ns_local_read_bytes_total gauge jindofsx_ns_local_read_bytes_total 0 #\u603b\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_local_read_bytes_total_window # TYPE jindofsx_ns_local_read_bytes_total_window gauge jindofsx_ns_local_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_time_total # TYPE jindofsx_ns_local_read_time_total gauge jindofsx_ns_local_read_time_total 0 #\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_readop_num_total # TYPE jindofsx_ns_local_readop_num_total gauge jindofsx_ns_local_readop_num_total 0 #\u77ed\u8def\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_remote_read_bytes_time_total_window # TYPE jindofsx_ns_remote_read_bytes_time_total_window gauge jindofsx_ns_remote_read_bytes_time_total_window 73714663 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_read_bytes_total # TYPE jindofsx_ns_remote_read_bytes_total gauge jindofsx_ns_remote_read_bytes_total 112108045498 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_bytes_total_window # TYPE jindofsx_ns_remote_read_bytes_total_window gauge jindofsx_ns_remote_read_bytes_total_window 13380884043 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_time_total # TYPE jindofsx_ns_remote_read_time_total gauge jindofsx_ns_remote_read_time_total 765272359 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u65f6\u95f4\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_readop_num_total # TYPE jindofsx_ns_remote_readop_num_total gauge jindofsx_ns_remote_readop_num_total 107314 #\u8fdc\u7aef\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_slicelet_read_bytes_time_total_window # TYPE jindofsx_ns_slicelet_read_bytes_time_total_window gauge jindofsx_ns_slicelet_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_read_bytes_total # TYPE jindofsx_ns_slicelet_read_bytes_total gauge jindofsx_ns_slicelet_read_bytes_total 0 #\u603b\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_bytes_total_window # TYPE jindofsx_ns_slicelet_read_bytes_total_window gauge jindofsx_ns_slicelet_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_time_total # TYPE jindofsx_ns_slicelet_read_time_total gauge jindofsx_ns_slicelet_read_time_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_readop_num_total # TYPE jindofsx_ns_slicelet_readop_num_total gauge jindofsx_ns_slicelet_readop_num_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_total_disk_cap # TYPE jindofsx_ns_total_disk_cap gauge jindofsx_ns_total_disk_cap 840739848192 #\u78c1\u76d8\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_mem_cap # TYPE jindofsx_ns_total_mem_cap gauge jindofsx_ns_total_mem_cap 0 #\u5185\u5b58\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_stsnodes_num # TYPE jindofsx_ns_total_stsnodes_num gauge jindofsx_ns_total_stsnodes_num 3 #alive worker \u8282\u70b9\u7684\u6570\u91cf # HELP jindofsx_ns_total_used_disk_cap # TYPE jindofsx_ns_total_used_disk_cap gauge jindofsx_ns_total_used_disk_cap 153423446016 #\u5df2\u7528\u78c1\u76d8\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_used_mem_cap # TYPE jindofsx_ns_total_used_mem_cap gauge jindofsx_ns_total_used_mem_cap 0 #\u5df2\u7528\u5185\u5b58\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte","title":"JindoRuntime Prometheus \u6307\u6807\u9884\u89c8(\u66f4\u65b0\u4e2d)"},{"location":"fluid-jindoruntime/jindo_fluid_nas_ufs_example/","text":"\u52a0\u901fOSS\u4e0a\u6570\u636e \u00a7 \u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002 1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001 \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907 \u00a7 5.1 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: nas spec: mounts: - mountPoint: local:///mnt/nas name: data path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: nas spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d NAS \u7684\u8def\u5f84\uff0c local:// \u4e3a\u672c\u5730\u8def\u5f84\u524d\u7f00\uff0c /mnt/nas \u8868\u793a\u672c\u5730\u771f\u5b9e\u8def\u5f84\uff0c\u8be5\u8def\u5f84\u9700\u8981\u5728\u5404\u4e2a\u8282\u70b9\u4e0a\u90fd\u5b58\u5728\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JinRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime nas NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE nas Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset nas NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE nas 511MiB 0.00B 180.00GiB 0.0% Bound 1h \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/nas 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/nas Bound hadoop 100Gi RWX 58m 6\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime nas \u5220\u9664 dataset kubectl delete dataset nas","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_nas_ufs_example/#oss","text":"\u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_nas_ufs_example/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"fluid-jindoruntime/jindo_fluid_nas_ufs_example/#2-fluid","text":"","title":"2\u3001\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"fluid-jindoruntime/jindo_fluid_nas_ufs_example/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"fluid-jindoruntime/jindo_fluid_nas_ufs_example/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"fluid-jindoruntime/jindo_fluid_nas_ufs_example/#5","text":"","title":"5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907"},{"location":"fluid-jindoruntime/jindo_fluid_nas_ufs_example/#51-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: nas spec: mounts: - mountPoint: local:///mnt/nas name: data path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: nas spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d NAS \u7684\u8def\u5f84\uff0c local:// \u4e3a\u672c\u5730\u8def\u5f84\u524d\u7f00\uff0c /mnt/nas \u8868\u793a\u672c\u5730\u771f\u5b9e\u8def\u5f84\uff0c\u8be5\u8def\u5f84\u9700\u8981\u5728\u5404\u4e2a\u8282\u70b9\u4e0a\u90fd\u5b58\u5728\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JinRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime nas NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE nas Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset nas NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE nas 511MiB 0.00B 180.00GiB 0.0% Bound 1h \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/nas 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/nas Bound hadoop 100Gi RWX 58m","title":"5.1 \u521b\u5efa dataset \u548c JindoRuntime"},{"location":"fluid-jindoruntime/jindo_fluid_nas_ufs_example/#6","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime nas \u5220\u9664 dataset kubectl delete dataset nas","title":"6\u3001\u73af\u5883\u6e05\u7406"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/","text":"\u52a0\u901f OSS-HDFS \u4e0a\u6570\u636e \u00a7 \u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS-HDFS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002 1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001 \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907 \u00a7 5.1 \u51c6\u5907 OSS-HDFS \u670d\u52a1 \u00a7 \u60a8\u9700\u8981\u5f00\u901a\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1(Object Storage Service\uff0c\u7b80\u79f0OSS)\uff0c\u6309\u7167\u4f7f\u7528\u6587\u6863\u8fdb\u884cbucket\u521b\u5efa\uff0c\u5e76\u5728\u521b\u5efa\u65f6\u6307\u5b9abucket\u4e3a\u652f\u6301 OSS-HDFS \u6a21\u5f0f\uff0c\u8bbf\u95ee\u94fe\u63a5 https://github.com../../tree/master/docsdocs/user/4.x/4.6.x/4.6.12/jindofs/jindo_dls_quickstart.md 5.2 \u51c6\u5907\u6d4b\u8bd5\u6837\u4f8b\u6570\u636e \u00a7 \u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 Apache \u955c\u50cf\u7ad9\u70b9\u4e0a\u7684 Spark \u76f8\u5173\u8d44\u6e90\u4f5c\u4e3a\u6f14\u793a\u4e2d\u4f7f\u7528\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002\u8fd9\u4e2a\u9009\u62e9\u5e76\u6ca1\u6709\u4efb\u4f55\u7279\u6b8a\u4e4b\u5904\uff0c\u4f60\u53ef\u4ee5\u5c06\u8fd9\u4e2a\u8fdc\u7a0b\u6587\u4ef6\u4fee\u6539\u4e3a\u4efb\u610f\u4f60\u559c\u6b22\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002 \u73b0\u5728\u8fdc\u7a0b\u8d44\u6e90\u6587\u4ef6\u5230\u672c\u5730 mkdir tmp cd tmp wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz \u4e0a\u4f20\u672c\u5730\u6587\u4ef6\u5230 OSS-HDFS \u4e0a \u60a8\u53ef\u4ee5\u4f7f\u7528 JindoSDK \uff0c\u6309\u7167\u4f7f\u7528\u8bf4\u660e\u5c06\u672c\u5730\u4e0b\u8f7d\u7684\u6587\u4ef6\u4e0a\u4f20\u5230\u8fdc\u7a0b OSS-HDFS \u7684bucket\u4e0a\u3002\u5982\u60a8\u7684bucket\u540d\u5b57\u4e3a test-bucket \u4e14\u4f7f\u7528 JindoSDK \u4f5c\u4e3a\u5ba2\u6237\u7aef\u5de5\u5177\uff0c\u53ef\u6267\u884c\u5982\u4e0b\u547d\u4ee4\u4e0a\u4f20\u6587\u4ef6\u5230 OSS-HDFS \u4e0a\u9762\u3002 hadoop fs -put spark-2.4.7-bin-hadoop2.7.tgz oss://test-bucket/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz hadoop fs -put spark-3.1.1-bin-hadoop3.2.tgz oss://test-bucket/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz 5.3 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: v1 kind: Secret metadata: name: mysecret stringData: fs.oss.accessKeyId: <OSS_HDFS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_HDFS_ACCESS_KEY_SECRET> --- apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/dir/ options: fs.oss.endpoint: <OSS_HDFS_ENDPOINT> name: hadoop path: / encryptOptions: - name: fs.oss.accessKeyId valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeyId - name: fs.oss.accessKeySecret valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeySecret accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aOSS-HDFS bucket\u7684endpoint\u4fe1\u606f\uff0c\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08cn-xxx.oss-dls.aliyuncs.com\uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08oss-cn-xxx.aliyuncs.com\uff09\u4e0d\u540c\uff0c\u5982\u60a8\u7684\u670d\u52a1\u5728 hangzhou\uff0c\u90a3\u4e48\u5bf9\u5e94\u7684 endpoint \u4e3a\uff1acn-hangzhou.oss-dls.aliyuncs.com replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JindoRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hadoop Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE HCFS URL TOTAL FILES CACHE HIT RATIO AGE hadoop [Calculating] 0.00B 200.00GiB Bound hadoop-jindofs-master-0.default:18844 [Calculating] 6m28s \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m 6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: hadoop \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u67e5\u770b\u6587\u4ef6\u5927\u5c0f $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 \u8fdb\u884c\u6587\u4ef6\u7684cp\u89c2\u5bdf\u65f6\u95f4\u6d88\u8017\u4e8618s $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m18.386s user 0m0.002s sys 0m0.105s \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 oss \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: hadoop-dataload spec: loadMetadata: true dataset: name: hadoop namespace: default \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload spark-dataload NAME DATASET PHASE AGE hadoop-dataload hadoop Completed 2m13s \u67e5\u770b\u6b64\u65f6 dataset \u7684\u7f13\u5b58\u60c5\u51b5\uff0c\u53d1\u73b0210MB\u7684\u6570\u636e\u5df2\u7ecf\u90fd\u7f13\u5b58\u5230\u4e86\u672c\u5730\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hadoop 210.00MiB 210.00MiB 180.00GiB 100.0% Bound 1h \u4e3a\u4e86\u907f\u514d\u5176\u4ed6\u56e0\u7d20(\u6bd4\u5982page cache)\u5bf9\u7ed3\u679c\u9020\u6210\u5f71\u54cd\uff0c\u6211\u4eec\u5c06\u5220\u9664\u4e4b\u524d\u7684\u5bb9\u5668\uff0c\u65b0\u5efa\u76f8\u540c\u7684\u5e94\u7528\uff0c\u5c1d\u8bd5\u8bbf\u95ee\u540c\u6837\u7684\u6587\u4ef6\u3002\u7531\u4e8e\u6b64\u65f6\u6587\u4ef6\u5df2\u7ecf\u88ab JindoFSx \u7f13\u5b58\uff0c\u53ef\u4ee5\u770b\u5230\u7b2c\u4e8c\u6b21\u8bbf\u95ee\u6240\u9700\u65f6\u95f4\u8fdc\u5c0f\u4e8e\u7b2c\u4e00\u6b21\u3002 kubectl delete -f app.yaml && kubectl create -f app.yaml \u8fdb\u884c\u6587\u4ef6\u7684\u62f7\u8d1d\u89c2\u5bdf\u65f6\u95f4\uff0c\u53d1\u73b0\u6d88\u801748ms\uff0c\u6574\u4e2a\u62f7\u8d1d\u7684\u65f6\u95f4\u7f29\u77ed\u4e86300\u500d $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m0.048s user 0m0.001s sys 0m0.046s 7\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"\u52a0\u901f OSS-HDFS \u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#oss-hdfs","text":"\u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS-HDFS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002","title":"\u52a0\u901f OSS-HDFS \u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#2-fluid","text":"","title":"2\u3001\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#5","text":"","title":"5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#51-oss-hdfs","text":"\u60a8\u9700\u8981\u5f00\u901a\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1(Object Storage Service\uff0c\u7b80\u79f0OSS)\uff0c\u6309\u7167\u4f7f\u7528\u6587\u6863\u8fdb\u884cbucket\u521b\u5efa\uff0c\u5e76\u5728\u521b\u5efa\u65f6\u6307\u5b9abucket\u4e3a\u652f\u6301 OSS-HDFS \u6a21\u5f0f\uff0c\u8bbf\u95ee\u94fe\u63a5 https://github.com../../tree/master/docsdocs/user/4.x/4.6.x/4.6.12/jindofs/jindo_dls_quickstart.md","title":"5.1 \u51c6\u5907 OSS-HDFS \u670d\u52a1"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#52","text":"\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 Apache \u955c\u50cf\u7ad9\u70b9\u4e0a\u7684 Spark \u76f8\u5173\u8d44\u6e90\u4f5c\u4e3a\u6f14\u793a\u4e2d\u4f7f\u7528\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002\u8fd9\u4e2a\u9009\u62e9\u5e76\u6ca1\u6709\u4efb\u4f55\u7279\u6b8a\u4e4b\u5904\uff0c\u4f60\u53ef\u4ee5\u5c06\u8fd9\u4e2a\u8fdc\u7a0b\u6587\u4ef6\u4fee\u6539\u4e3a\u4efb\u610f\u4f60\u559c\u6b22\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002 \u73b0\u5728\u8fdc\u7a0b\u8d44\u6e90\u6587\u4ef6\u5230\u672c\u5730 mkdir tmp cd tmp wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz \u4e0a\u4f20\u672c\u5730\u6587\u4ef6\u5230 OSS-HDFS \u4e0a \u60a8\u53ef\u4ee5\u4f7f\u7528 JindoSDK \uff0c\u6309\u7167\u4f7f\u7528\u8bf4\u660e\u5c06\u672c\u5730\u4e0b\u8f7d\u7684\u6587\u4ef6\u4e0a\u4f20\u5230\u8fdc\u7a0b OSS-HDFS \u7684bucket\u4e0a\u3002\u5982\u60a8\u7684bucket\u540d\u5b57\u4e3a test-bucket \u4e14\u4f7f\u7528 JindoSDK \u4f5c\u4e3a\u5ba2\u6237\u7aef\u5de5\u5177\uff0c\u53ef\u6267\u884c\u5982\u4e0b\u547d\u4ee4\u4e0a\u4f20\u6587\u4ef6\u5230 OSS-HDFS \u4e0a\u9762\u3002 hadoop fs -put spark-2.4.7-bin-hadoop2.7.tgz oss://test-bucket/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz hadoop fs -put spark-3.1.1-bin-hadoop3.2.tgz oss://test-bucket/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz","title":"5.2 \u51c6\u5907\u6d4b\u8bd5\u6837\u4f8b\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#53-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: v1 kind: Secret metadata: name: mysecret stringData: fs.oss.accessKeyId: <OSS_HDFS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_HDFS_ACCESS_KEY_SECRET> --- apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/dir/ options: fs.oss.endpoint: <OSS_HDFS_ENDPOINT> name: hadoop path: / encryptOptions: - name: fs.oss.accessKeyId valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeyId - name: fs.oss.accessKeySecret valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeySecret accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aOSS-HDFS bucket\u7684endpoint\u4fe1\u606f\uff0c\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08cn-xxx.oss-dls.aliyuncs.com\uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08oss-cn-xxx.aliyuncs.com\uff09\u4e0d\u540c\uff0c\u5982\u60a8\u7684\u670d\u52a1\u5728 hangzhou\uff0c\u90a3\u4e48\u5bf9\u5e94\u7684 endpoint \u4e3a\uff1acn-hangzhou.oss-dls.aliyuncs.com replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JindoRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hadoop Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE HCFS URL TOTAL FILES CACHE HIT RATIO AGE hadoop [Calculating] 0.00B 200.00GiB Bound hadoop-jindofs-master-0.default:18844 [Calculating] 6m28s \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m","title":"5.3 \u521b\u5efa dataset \u548c JindoRuntime"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#6","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: hadoop \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u67e5\u770b\u6587\u4ef6\u5927\u5c0f $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 \u8fdb\u884c\u6587\u4ef6\u7684cp\u89c2\u5bdf\u65f6\u95f4\u6d88\u8017\u4e8618s $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m18.386s user 0m0.002s sys 0m0.105s \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 oss \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: hadoop-dataload spec: loadMetadata: true dataset: name: hadoop namespace: default \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload spark-dataload NAME DATASET PHASE AGE hadoop-dataload hadoop Completed 2m13s \u67e5\u770b\u6b64\u65f6 dataset \u7684\u7f13\u5b58\u60c5\u51b5\uff0c\u53d1\u73b0210MB\u7684\u6570\u636e\u5df2\u7ecf\u90fd\u7f13\u5b58\u5230\u4e86\u672c\u5730\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hadoop 210.00MiB 210.00MiB 180.00GiB 100.0% Bound 1h \u4e3a\u4e86\u907f\u514d\u5176\u4ed6\u56e0\u7d20(\u6bd4\u5982page cache)\u5bf9\u7ed3\u679c\u9020\u6210\u5f71\u54cd\uff0c\u6211\u4eec\u5c06\u5220\u9664\u4e4b\u524d\u7684\u5bb9\u5668\uff0c\u65b0\u5efa\u76f8\u540c\u7684\u5e94\u7528\uff0c\u5c1d\u8bd5\u8bbf\u95ee\u540c\u6837\u7684\u6587\u4ef6\u3002\u7531\u4e8e\u6b64\u65f6\u6587\u4ef6\u5df2\u7ecf\u88ab JindoFSx \u7f13\u5b58\uff0c\u53ef\u4ee5\u770b\u5230\u7b2c\u4e8c\u6b21\u8bbf\u95ee\u6240\u9700\u65f6\u95f4\u8fdc\u5c0f\u4e8e\u7b2c\u4e00\u6b21\u3002 kubectl delete -f app.yaml && kubectl create -f app.yaml \u8fdb\u884c\u6587\u4ef6\u7684\u62f7\u8d1d\u89c2\u5bdf\u65f6\u95f4\uff0c\u53d1\u73b0\u6d88\u801748ms\uff0c\u6574\u4e2a\u62f7\u8d1d\u7684\u65f6\u95f4\u7f29\u77ed\u4e86300\u500d $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m0.048s user 0m0.001s sys 0m0.046s","title":"6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#7","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"7\u3001\u73af\u5883\u6e05\u7406"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/","text":"\u52a0\u901fOSS\u4e0a\u6570\u636e \u00a7 \u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002 1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001 \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907 \u00a7 5.1 \u51c6\u5907 OSS \u670d\u52a1 \u00a7 \u60a8\u9700\u8981\u5f00\u901a\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1(Object Storage Service\uff0c\u7b80\u79f0OSS)\uff0c\u6309\u7167\u4f7f\u7528\u6587\u6863\u8fdb\u884cbucket\u521b\u5efa( https://www.aliyun.com/product/oss ) 5.2 \u51c6\u5907\u6d4b\u8bd5\u6837\u4f8b\u6570\u636e \u00a7 \u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 Apache \u955c\u50cf\u7ad9\u70b9\u4e0a\u7684 Spark \u76f8\u5173\u8d44\u6e90\u4f5c\u4e3a\u6f14\u793a\u4e2d\u4f7f\u7528\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002\u8fd9\u4e2a\u9009\u62e9\u5e76\u6ca1\u6709\u4efb\u4f55\u7279\u6b8a\u4e4b\u5904\uff0c\u4f60\u53ef\u4ee5\u5c06\u8fd9\u4e2a\u8fdc\u7a0b\u6587\u4ef6\u4fee\u6539\u4e3a\u4efb\u610f\u4f60\u559c\u6b22\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002 \u73b0\u5728\u8fdc\u7a0b\u8d44\u6e90\u6587\u4ef6\u5230\u672c\u5730 mkdir tmp cd tmp wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz \u4e0a\u4f20\u672c\u5730\u6587\u4ef6\u5230 OSS \u4e0a \u60a8\u53ef\u4ee5\u4f7f\u7528 OSS \u63d0\u4f9b\u7684\u5ba2\u6237\u7aef ossutil \u6216\u8005\u963f\u91cc\u4e91EMR\u56e2\u961f\u63d0\u4f9b\u7684 JindoSDK \uff0c\u6309\u7167\u4f7f\u7528\u8bf4\u660e\u5c06\u672c\u5730\u4e0b\u8f7d\u7684\u6587\u4ef6\u4e0a\u4f20\u5230\u8fdc\u7a0b OSS \u7684bucket\u4e0a\u3002\u5982\u60a8\u7684bucket\u540d\u5b57\u4e3a test-bucket \u4e14\u4f7f\u7528 JindoSDK \u4f5c\u4e3a\u5ba2\u6237\u7aef\u5de5\u5177\uff0c\u53ef\u6267\u884c\u5982\u4e0b\u547d\u4ee4\u4e0a\u4f20\u6587\u4ef6\u5230 OSS \u4e0a\u9762\u3002 hadoop fs -put spark-2.4.7-bin-hadoop2.7.tgz oss://test-bucket/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz hadoop fs -put spark-3.1.1-bin-hadoop3.2.tgz oss://test-bucket/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz 5.3 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: v1 kind: Secret metadata: name: mysecret stringData: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> --- apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/dir/ options: fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop path: / encryptOptions: - name: fs.oss.accessKeyId valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeyId - name: fs.oss.accessKeySecret valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeySecret accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aoss bucket\u7684endpoint\u4fe1\u606f\uff0c\u516c\u7f51\u6216\u5185\u7f51\u5730\u5740\u7686\u53ef\uff0c\u5982\u60a8\u7684 bucket \u5728\u676d\u5dde Region\uff0c\u90a3\u4e48\u516c\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou.aliyuncs.com\uff0c\u5185\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou-internal.aliyuncs.com(\u5185\u7f51\u5730\u5740\u4f7f\u7528\u6761\u4ef6\u4e3a\u60a8\u7684 K8S \u96c6\u7fa4\u6240\u5728\u533a\u57df\u548coss\u533a\u57df\u76f8\u540c)\u3002 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JindoRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hadoop Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE HCFS URL TOTAL FILES CACHE HIT RATIO AGE hadoop [Calculating] 0.00B 200.00GiB Bound hadoop-jindofs-master-0.default:18844 [Calculating] 6m28s \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m 6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: hadoop \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u67e5\u770b\u6587\u4ef6\u5927\u5c0f $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 \u8fdb\u884c\u6587\u4ef6\u7684cp\u89c2\u5bdf\u65f6\u95f4\u6d88\u8017\u4e8618s $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m18.386s user 0m0.002s sys 0m0.105s \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 oss \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: hadoop-dataload spec: loadMetadata: true dataset: name: hadoop namespace: default \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload spark-dataload NAME DATASET PHASE AGE hadoop-dataload hadoop Completed 2m13s \u67e5\u770b\u6b64\u65f6 dataset \u7684\u7f13\u5b58\u60c5\u51b5\uff0c\u53d1\u73b0210MB\u7684\u6570\u636e\u5df2\u7ecf\u90fd\u7f13\u5b58\u5230\u4e86\u672c\u5730\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hadoop 210.00MiB 210.00MiB 180.00GiB 100.0% Bound 1h \u4e3a\u4e86\u907f\u514d\u5176\u4ed6\u56e0\u7d20(\u6bd4\u5982page cache)\u5bf9\u7ed3\u679c\u9020\u6210\u5f71\u54cd\uff0c\u6211\u4eec\u5c06\u5220\u9664\u4e4b\u524d\u7684\u5bb9\u5668\uff0c\u65b0\u5efa\u76f8\u540c\u7684\u5e94\u7528\uff0c\u5c1d\u8bd5\u8bbf\u95ee\u540c\u6837\u7684\u6587\u4ef6\u3002\u7531\u4e8e\u6b64\u65f6\u6587\u4ef6\u5df2\u7ecf\u88ab JindoFSx \u7f13\u5b58\uff0c\u53ef\u4ee5\u770b\u5230\u7b2c\u4e8c\u6b21\u8bbf\u95ee\u6240\u9700\u65f6\u95f4\u8fdc\u5c0f\u4e8e\u7b2c\u4e00\u6b21\u3002 kubectl delete -f app.yaml && kubectl create -f app.yaml \u8fdb\u884c\u6587\u4ef6\u7684\u62f7\u8d1d\u89c2\u5bdf\u65f6\u95f4\uff0c\u53d1\u73b0\u6d88\u801748ms\uff0c\u6574\u4e2a\u62f7\u8d1d\u7684\u65f6\u95f4\u7f29\u77ed\u4e86300\u500d $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m0.048s user 0m0.001s sys 0m0.046s 7\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#oss","text":"\u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#2-fluid","text":"","title":"2\u3001\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#5","text":"","title":"5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#51-oss","text":"\u60a8\u9700\u8981\u5f00\u901a\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1(Object Storage Service\uff0c\u7b80\u79f0OSS)\uff0c\u6309\u7167\u4f7f\u7528\u6587\u6863\u8fdb\u884cbucket\u521b\u5efa( https://www.aliyun.com/product/oss )","title":"5.1 \u51c6\u5907 OSS \u670d\u52a1"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#52","text":"\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 Apache \u955c\u50cf\u7ad9\u70b9\u4e0a\u7684 Spark \u76f8\u5173\u8d44\u6e90\u4f5c\u4e3a\u6f14\u793a\u4e2d\u4f7f\u7528\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002\u8fd9\u4e2a\u9009\u62e9\u5e76\u6ca1\u6709\u4efb\u4f55\u7279\u6b8a\u4e4b\u5904\uff0c\u4f60\u53ef\u4ee5\u5c06\u8fd9\u4e2a\u8fdc\u7a0b\u6587\u4ef6\u4fee\u6539\u4e3a\u4efb\u610f\u4f60\u559c\u6b22\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002 \u73b0\u5728\u8fdc\u7a0b\u8d44\u6e90\u6587\u4ef6\u5230\u672c\u5730 mkdir tmp cd tmp wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz \u4e0a\u4f20\u672c\u5730\u6587\u4ef6\u5230 OSS \u4e0a \u60a8\u53ef\u4ee5\u4f7f\u7528 OSS \u63d0\u4f9b\u7684\u5ba2\u6237\u7aef ossutil \u6216\u8005\u963f\u91cc\u4e91EMR\u56e2\u961f\u63d0\u4f9b\u7684 JindoSDK \uff0c\u6309\u7167\u4f7f\u7528\u8bf4\u660e\u5c06\u672c\u5730\u4e0b\u8f7d\u7684\u6587\u4ef6\u4e0a\u4f20\u5230\u8fdc\u7a0b OSS \u7684bucket\u4e0a\u3002\u5982\u60a8\u7684bucket\u540d\u5b57\u4e3a test-bucket \u4e14\u4f7f\u7528 JindoSDK \u4f5c\u4e3a\u5ba2\u6237\u7aef\u5de5\u5177\uff0c\u53ef\u6267\u884c\u5982\u4e0b\u547d\u4ee4\u4e0a\u4f20\u6587\u4ef6\u5230 OSS \u4e0a\u9762\u3002 hadoop fs -put spark-2.4.7-bin-hadoop2.7.tgz oss://test-bucket/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz hadoop fs -put spark-3.1.1-bin-hadoop3.2.tgz oss://test-bucket/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz","title":"5.2 \u51c6\u5907\u6d4b\u8bd5\u6837\u4f8b\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#53-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: v1 kind: Secret metadata: name: mysecret stringData: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> --- apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/dir/ options: fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop path: / encryptOptions: - name: fs.oss.accessKeyId valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeyId - name: fs.oss.accessKeySecret valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeySecret accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aoss bucket\u7684endpoint\u4fe1\u606f\uff0c\u516c\u7f51\u6216\u5185\u7f51\u5730\u5740\u7686\u53ef\uff0c\u5982\u60a8\u7684 bucket \u5728\u676d\u5dde Region\uff0c\u90a3\u4e48\u516c\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou.aliyuncs.com\uff0c\u5185\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou-internal.aliyuncs.com(\u5185\u7f51\u5730\u5740\u4f7f\u7528\u6761\u4ef6\u4e3a\u60a8\u7684 K8S \u96c6\u7fa4\u6240\u5728\u533a\u57df\u548coss\u533a\u57df\u76f8\u540c)\u3002 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JindoRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hadoop Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE HCFS URL TOTAL FILES CACHE HIT RATIO AGE hadoop [Calculating] 0.00B 200.00GiB Bound hadoop-jindofs-master-0.default:18844 [Calculating] 6m28s \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m","title":"5.3 \u521b\u5efa dataset \u548c JindoRuntime"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#6","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: hadoop \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u67e5\u770b\u6587\u4ef6\u5927\u5c0f $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 \u8fdb\u884c\u6587\u4ef6\u7684cp\u89c2\u5bdf\u65f6\u95f4\u6d88\u8017\u4e8618s $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m18.386s user 0m0.002s sys 0m0.105s \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 oss \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: hadoop-dataload spec: loadMetadata: true dataset: name: hadoop namespace: default \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload spark-dataload NAME DATASET PHASE AGE hadoop-dataload hadoop Completed 2m13s \u67e5\u770b\u6b64\u65f6 dataset \u7684\u7f13\u5b58\u60c5\u51b5\uff0c\u53d1\u73b0210MB\u7684\u6570\u636e\u5df2\u7ecf\u90fd\u7f13\u5b58\u5230\u4e86\u672c\u5730\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hadoop 210.00MiB 210.00MiB 180.00GiB 100.0% Bound 1h \u4e3a\u4e86\u907f\u514d\u5176\u4ed6\u56e0\u7d20(\u6bd4\u5982page cache)\u5bf9\u7ed3\u679c\u9020\u6210\u5f71\u54cd\uff0c\u6211\u4eec\u5c06\u5220\u9664\u4e4b\u524d\u7684\u5bb9\u5668\uff0c\u65b0\u5efa\u76f8\u540c\u7684\u5e94\u7528\uff0c\u5c1d\u8bd5\u8bbf\u95ee\u540c\u6837\u7684\u6587\u4ef6\u3002\u7531\u4e8e\u6b64\u65f6\u6587\u4ef6\u5df2\u7ecf\u88ab JindoFSx \u7f13\u5b58\uff0c\u53ef\u4ee5\u770b\u5230\u7b2c\u4e8c\u6b21\u8bbf\u95ee\u6240\u9700\u65f6\u95f4\u8fdc\u5c0f\u4e8e\u7b2c\u4e00\u6b21\u3002 kubectl delete -f app.yaml && kubectl create -f app.yaml \u8fdb\u884c\u6587\u4ef6\u7684\u62f7\u8d1d\u89c2\u5bdf\u65f6\u95f4\uff0c\u53d1\u73b0\u6d88\u801748ms\uff0c\u6574\u4e2a\u62f7\u8d1d\u7684\u65f6\u95f4\u7f29\u77ed\u4e86300\u500d $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m0.048s user 0m0.001s sys 0m0.046s","title":"6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#7","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"7\u3001\u73af\u5883\u6e05\u7406"},{"location":"fluid-jindoruntime/jindo_fluid_overview/","text":"Fluid \u7ed3\u5408 JindoCache \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4ecb\u7ecd\u548c\u4f7f\u7528 \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 Kubernetes version > 1.14, \u652f\u6301CSI Golang 1.12+ Helm 3 \u5b89\u88c5\u5305\u4e0b\u8f7d \u00a7 \u4e0b\u8f7d\u9875\u9762 \u4f7f\u7528\u6587\u6863 \u00a7 \u52a0\u901fOSS\u4e0a\u6570\u636e \u52a0\u901fHDFS\u4e0a\u6570\u636e \u52a0\u901fOSS-HDFS\u4e0a\u6570\u636e \u52a0\u901fS3\u4e0a\u6570\u636e \u52a0\u901fNAS\u4e0a\u6570\u636e \u52a0\u901fPVC\u4e0a\u6570\u636e Fluid + JindoCache \u9ad8\u9636\u529f\u80fd\u4f7f\u7528 \u6570\u636e\u9884\u52a0\u8f7d \u76d1\u63a7\u53ca\u53ef\u89c2\u6d4b\u529f\u80fd \u95ee\u9898\u8bca\u65ad\u53ca\u5904\u7406 \u5bf9\u63a5\u963f\u91cc\u4e91SLS\u65e5\u5fd7\u670d\u52a1 \u6700\u4f73\u5b9e\u8df5\u7cfb\u5217 \u00a7 \u4e91\u539f\u751f AI \u5728\u81ea\u52a8\u9a7e\u9a76\u9886\u57df\u7684\u5e94\u7528\u63a2\u7d22\u4e0e\u6beb\u672b\u667a\u884c\u843d\u5730\u5b9e\u8df5 \u5fae\u535a\u6d77\u91cf\u6df1\u5ea6\u5b66\u4e60\u6a21\u578b\u8bad\u7ec3\u6548\u7387\u8dc3\u5347\u7684\u79d8\u5bc6 \u4f5c\u4e1a\u5e2e\u68c0\u7d22\u670d\u52a1\u57fa\u4e8e Fluid \u7684\u8ba1\u7b97\u5b58\u50a8\u5206\u79bb\u5b9e\u8df5","title":"Fluid JindoRuntime"},{"location":"fluid-jindoruntime/jindo_fluid_overview/#fluid-jindocache","text":"","title":"Fluid \u7ed3\u5408 JindoCache \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4ecb\u7ecd\u548c\u4f7f\u7528"},{"location":"fluid-jindoruntime/jindo_fluid_overview/#_1","text":"Kubernetes version > 1.14, \u652f\u6301CSI Golang 1.12+ Helm 3","title":"\u73af\u5883\u8981\u6c42"},{"location":"fluid-jindoruntime/jindo_fluid_overview/#_2","text":"\u4e0b\u8f7d\u9875\u9762","title":"\u5b89\u88c5\u5305\u4e0b\u8f7d"},{"location":"fluid-jindoruntime/jindo_fluid_overview/#_3","text":"\u52a0\u901fOSS\u4e0a\u6570\u636e \u52a0\u901fHDFS\u4e0a\u6570\u636e \u52a0\u901fOSS-HDFS\u4e0a\u6570\u636e \u52a0\u901fS3\u4e0a\u6570\u636e \u52a0\u901fNAS\u4e0a\u6570\u636e \u52a0\u901fPVC\u4e0a\u6570\u636e Fluid + JindoCache \u9ad8\u9636\u529f\u80fd\u4f7f\u7528 \u6570\u636e\u9884\u52a0\u8f7d \u76d1\u63a7\u53ca\u53ef\u89c2\u6d4b\u529f\u80fd \u95ee\u9898\u8bca\u65ad\u53ca\u5904\u7406 \u5bf9\u63a5\u963f\u91cc\u4e91SLS\u65e5\u5fd7\u670d\u52a1","title":"\u4f7f\u7528\u6587\u6863"},{"location":"fluid-jindoruntime/jindo_fluid_overview/#_4","text":"\u4e91\u539f\u751f AI \u5728\u81ea\u52a8\u9a7e\u9a76\u9886\u57df\u7684\u5e94\u7528\u63a2\u7d22\u4e0e\u6beb\u672b\u667a\u884c\u843d\u5730\u5b9e\u8df5 \u5fae\u535a\u6d77\u91cf\u6df1\u5ea6\u5b66\u4e60\u6a21\u578b\u8bad\u7ec3\u6548\u7387\u8dc3\u5347\u7684\u79d8\u5bc6 \u4f5c\u4e1a\u5e2e\u68c0\u7d22\u670d\u52a1\u57fa\u4e8e Fluid \u7684\u8ba1\u7b97\u5b58\u50a8\u5206\u79bb\u5b9e\u8df5","title":"\u6700\u4f73\u5b9e\u8df5\u7cfb\u5217"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/","text":"\u52a0\u901fPVC\u4e0a\u6570\u636e \u00a7 \u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c PVC \u6570\u636e\u5377\u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002 1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001 \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907 \u00a7 5.1 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u4e3a\u5df2\u5b58\u5728\u7684PVC\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 testPVC\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: pvc spec: mounts: - mountPoint: pvc://testPVC name: data path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: pvc spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d pvc \u7684\u8def\u5f84\uff0c pvc:// \u4e3a\u6302\u8f7d\u672c\u5730volume\u524d\u7f00\uff0c testPVC \u8868\u793a\u9700\u8981\u52a0\u901f\u7684PVC\u7684\u540d\u5b57\uff0c\u9700\u8981\u6709\u8bbf\u95ee\u6743\u9650\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JinRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime pvc NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE pvc Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset pvc NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE pvc 511MiB 0.00B 180.00GiB 0.0% Bound 1h \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/pvc 100Gi ROX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/pvc Bound hadoop 100Gi RWX 58m 6\u3001\u6570\u636e\u9884\u70ed \u00a7 \u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf \u00a7 \u53ef\u4ee5\u8fdb\u884c\u6570\u636e\u9884\u70ed\u63d0\u524d\u5c06\u6587\u4ef6\u7f13\u5b58\u5230\u672c\u5730\uff0c\u5047\u8bbe\u9700\u8981\u52a0\u901f\u8be5 pvc \u4e0b\u9762\u7684\u4e24\u4e2a\u5b50\u76ee\u5f55 /data1 \u548c /data1 \u540c\u65f6\u5206\u522b\u6307\u5b9a\u7f13\u5b58\u526f\u672c\u6570\u91cf\uff0c\u53ef\u4ee5\u53c2\u8003\u5982\u4e0bdataload.yaml\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: test spec: dataset: name: pvc namespace: default loadMetadata: true target: - path: /data1/ replicas: 1 - path: /data2/ replicas: 2 kubectl create dataload.yaml \u89c2\u5bdf dataload \u72b6\u6001 kubectl get dataload test \u5177\u4f53 dataload \u7684\u529f\u80fd\u53ef\u53c2\u8003 \u6570\u636e\u9884\u52a0\u8f7d 7\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime \u548c dataset kubectl delete dataset pvc","title":"\u52a0\u901fPVC\u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/#pvc","text":"\u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c PVC \u6570\u636e\u5377\u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002","title":"\u52a0\u901fPVC\u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/#2-fluid","text":"","title":"2\u3001\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/#5","text":"","title":"5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/#51-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u4e3a\u5df2\u5b58\u5728\u7684PVC\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 testPVC\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: pvc spec: mounts: - mountPoint: pvc://testPVC name: data path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: pvc spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d pvc \u7684\u8def\u5f84\uff0c pvc:// \u4e3a\u6302\u8f7d\u672c\u5730volume\u524d\u7f00\uff0c testPVC \u8868\u793a\u9700\u8981\u52a0\u901f\u7684PVC\u7684\u540d\u5b57\uff0c\u9700\u8981\u6709\u8bbf\u95ee\u6743\u9650\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JinRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime pvc NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE pvc Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset pvc NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE pvc 511MiB 0.00B 180.00GiB 0.0% Bound 1h \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/pvc 100Gi ROX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/pvc Bound hadoop 100Gi RWX 58m","title":"5.1 \u521b\u5efa dataset \u548c JindoRuntime"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/#6","text":"","title":"6\u3001\u6570\u636e\u9884\u70ed"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/#_1","text":"\u53ef\u4ee5\u8fdb\u884c\u6570\u636e\u9884\u70ed\u63d0\u524d\u5c06\u6587\u4ef6\u7f13\u5b58\u5230\u672c\u5730\uff0c\u5047\u8bbe\u9700\u8981\u52a0\u901f\u8be5 pvc \u4e0b\u9762\u7684\u4e24\u4e2a\u5b50\u76ee\u5f55 /data1 \u548c /data1 \u540c\u65f6\u5206\u522b\u6307\u5b9a\u7f13\u5b58\u526f\u672c\u6570\u91cf\uff0c\u53ef\u4ee5\u53c2\u8003\u5982\u4e0bdataload.yaml\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: test spec: dataset: name: pvc namespace: default loadMetadata: true target: - path: /data1/ replicas: 1 - path: /data2/ replicas: 2 kubectl create dataload.yaml \u89c2\u5bdf dataload \u72b6\u6001 kubectl get dataload test \u5177\u4f53 dataload \u7684\u529f\u80fd\u53ef\u53c2\u8003 \u6570\u636e\u9884\u52a0\u8f7d","title":"\u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/#7","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime \u548c dataset kubectl delete dataset pvc","title":"7\u3001\u73af\u5883\u6e05\u7406"},{"location":"fluid-jindoruntime/jindo_fluid_question/","text":"\u95ee\u9898\u8bca\u65ad\u53ca\u5904\u7406 \u00a7 \u5982\u60a8\u7684\u73af\u5883\u51fa\u73b0\u95ee\u9898\uff0c\u4e5f\u53ef\u5f00 ISSUE \u7ed9\u6211\u4eec\uff0c\u4f1a\u7b2c\u4e00\u65f6\u95f4\u5904\u7406 Fluid \u5b89\u88c5\u4f7f\u7528\u76f8\u5173\u95ee\u9898 \u00a7 1. \u4e3a\u4ec0\u4e48\u6211\u4f7f\u7528Helm\u5b89\u88c5fluid\u5931\u8d25\u4e86\uff1f \u00a7 \u56de\u7b54 :\u63a8\u8350\u6309\u7167\u5b89\u88c5\u6587\u6863\u4f9d\u6b21\u786e\u8ba4Fluid\u7ec4\u4ef6\u662f\u5426\u6b63\u5e38\u8fd0\u884c\u3002 Fluid\u5b89\u88c5\u6587\u6863\u662f\u4ee5 Helm 3 \u4e3a\u4f8b\u8fdb\u884c\u90e8\u7f72\u7684\u3002\u5982\u679c\u60a8\u4f7f\u7528 Helm 3 \u4ee5\u4e0b\u7684\u7248\u672c\u90e8\u7f72Fluid\uff0c \u5e76\u4e14\u9047\u5230\u4e86 CRD\u6ca1\u6709\u6b63\u5e38\u542f\u52a8 \u7684\u60c5\u51b5\uff0c\u8fd9\u53ef\u80fd\u662f\u56e0\u4e3a Helm 3 \u53ca\u5176\u4ee5\u4e0a\u7248\u672c\u4f1a\u5728 helm install \u7684\u65f6\u5019\u81ea\u52a8\u5b89\u88c5CRD\uff0c \u800c\u4f4e\u7248\u672c\u7684Helm\u5219\u4e0d\u4f1a\u3002 \u53c2\u89c1 Helm\u5b98\u65b9\u6587\u6863\u8bf4\u660e \u3002 \u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u60a8\u9700\u8981\u624b\u52a8\u5b89\u88c5CRD\uff1a $ kubectl create -f fluid/crds 2. \u4e3a\u4ec0\u4e48\u6211\u65e0\u6cd5\u5220\u9664Runtime\uff1f \u00a7 \u56de\u7b54 :\u8bf7\u68c0\u67e5\u76f8\u5173Pod\u8fd0\u884c\u72b6\u6001\u548cRuntime\u7684Events\u3002 \u53ea\u8981\u6709\u4efb\u4f55\u6d3b\u8dc3Pod\u8fd8\u5728\u4f7f\u7528Fluid\u521b\u5efa\u7684Volume\uff0cFluid\u5c31\u4e0d\u4f1a\u5b8c\u6210\u5220\u9664\u64cd\u4f5c\u3002 \u5982\u4e0b\u7684\u547d\u4ee4\u53ef\u4ee5\u5feb\u901f\u5730\u627e\u51fa\u8fd9\u4e9b\u6d3b\u8dc3Pod\uff0c\u4f7f\u7528\u65f6\u628a <dataset_name> \u548c <dataset_namespace> \u6362\u6210\u81ea\u5df1\u7684\u5373\u53ef\uff1a kubectl describe pvc <dataset_name> -n <dataset_namespace> | \\ awk '/^Mounted/ {flag=1}; /^Events/ {flag=0}; flag' | \\ awk 'NR==1 {print $3}; NR!=1 {print $1}' | \\ xargs -I {} kubectl get po {} | \\ grep -E \"Running|Terminating|Pending\" | \\ cut -d \" \" -f 1 3. \u4e3a\u4ec0\u4e48\u6211\u5728\u521b\u5efa\u4efb\u52a1\u6302\u8f7d Runtime \u521b\u5efa\u7684 PVC \u7684\u65f6\u5019\u51fa\u73b0 driver name fuse.csi.fluid.io not found in the list of registered CSI drivers \u9519\u8bef\uff1f \u00a7 \u56de\u7b54 :\u8bf7\u67e5\u770b\u4efb\u52a1\u88ab\u8c03\u5ea6\u8282\u70b9\u6240\u5728\u7684 kubelet \u914d\u7f6e\u662f\u5426\u4e3a\u9ed8\u8ba4 /var/lib/kubelet \u3002 \u9996\u5148\u901a\u8fc7\u547d\u4ee4\u67e5\u770bFluid\u7684CSI\u7ec4\u4ef6\u662f\u5426\u6b63\u5e38 \u5982\u4e0b\u7684\u547d\u4ee4\u53ef\u4ee5\u5feb\u901f\u5730\u627e\u51faPod\uff0c\u4f7f\u7528\u65f6\u628a <node_name> \u548c <fluid_namespace> \u6362\u6210\u81ea\u5df1\u7684\u5373\u53ef\uff1a kubectl get pod -n <fluid_namespace> | grep <node_name> # <pod_name> \u4e3a\u4e0a\u4e00\u6b65pod\u540d kubectl logs <pod_name> node-driver-registrar -n <fluid_namespace> kubectl logs <pod_name> plugins -n <fluid_namespace> \u5982\u679c\u4e0a\u8ff0\u6b65\u9aa4\u7684Log\u65e0\u9519\u8bef\uff0c\u8bf7\u67e5\u770bcsidriver\u5bf9\u8c61\u662f\u5426\u5b58\u5728\uff1a kubectl get csidriver \u5982\u679ccsidriver\u5bf9\u8c61\u5b58\u5728\uff0c\u8bf7\u67e5\u770b\u67e5\u770bcsi\u6ce8\u518c\u8282\u70b9\u662f\u5426\u5305\u542b <node_name> \uff1a kubectl get csinode | grep <node_name> \u5982\u679c\u4e0a\u8ff0\u547d\u4ee4\u65e0\u8f93\u51fa\uff0c\u67e5\u770b\u4efb\u52a1\u88ab\u8c03\u5ea6\u8282\u70b9\u6240\u5728\u7684 kubelet \u914d\u7f6e\u662f\u5426\u4e3a\u9ed8\u8ba4 /var/lib/kubelet \u3002\u56e0\u4e3aFluid\u7684CSI\u7ec4\u4ef6\u901a\u8fc7\u56fa\u5b9a\u5730\u5740\u7684socket\u6ce8\u518c\u5230kubelet\uff0c\u9ed8\u8ba4\u4e3a --csi-address=/var/lib/kubelet/csi-plugins/fuse.csi.fluid.io/csi.sock --kubelet-registration-path=/var/lib/kubelet/csi-plugins/fuse.csi.fluid.io/csi.sock \u3002 4. \u4e3a\u4ec0\u4e48\u66f4\u65b0\u4e86fluid\u540e\uff0c\u4f7f\u7528 kubectl get \u67e5\u8be2\u66f4\u65b0\u524d\u521b\u5efa\u7684dataset\uff0c\u53d1\u73b0\u76f8\u6bd4\u65b0\u5efa\u7684dataset\u7f3a\u5c11\u4e86\u67d0\u4e9b\u5b57\u6bb5\uff1f \u00a7 \u56de\u7b54 :\u7531\u4e8e\u6211\u4eec\u5728fluid\u7684\u5347\u7ea7\u8fc7\u7a0b\u4e2d\u53ef\u80fd\u66f4\u65b0\u4e86CRD\uff0c\u4f60\u5728\u65e7\u7248\u672c\u521b\u5efa\u7684dataset\uff0c\u4f1a\u5c06CRD\u4e2d\u65b0\u589e\u7684\u5b57\u6bb5\u8bbe\u7f6e\u4e3a\u7a7a \u4f8b\u5982\uff0c\u5982\u679c\u4f60\u4ecev0.4\u6216\u66f4\u65e9\u7248\u672c\u5347\u7ea7\uff0c\u90a3\u65f6\u5019\u7684dataset\u6ca1\u6709FileNum\u5b57\u6bb5 \u66f4\u65b0fluid\u540e\uff0c\u5982\u679c\u4f60\u4f7f\u7528 kubectl get \u547d\u4ee4\uff0c\u65e0\u6cd5\u67e5\u8be2\u5230\u8be5dataset\u7684FileNum \u4f60\u53ef\u4ee5\u91cd\u5efadataset\uff0c\u65b0\u5efa\u7684dataset\u4f1a\u6b63\u5e38\u663e\u793a\u8fd9\u4e9b\u5b57\u6bb5 5. \u4e3a\u4ec0\u4e48\u5728\u5e94\u7528\u7a0b\u5e8f\u4e2d\u4f7f\u7528 PVC \u65f6\u4f1a\u4ea7\u751f\u4e86 Volume Attachment \u8d85\u65f6\u95ee\u9898\uff1f \u00a7 \u56de\u7b54 :Volume Attachment \u8d85\u65f6\u95ee\u9898\u662f Kubelet \u8fdb\u884c\u8bf7\u6c42 CSI Driver \u65f6\u672a\u6536\u5230 CSI Driver \u7684\u54cd\u5e94\u800c\u9020\u6210\u7684\u8d85\u65f6\u3002 \u8be5\u95ee\u9898\u662f\u7531\u4e8e Fluid \u7684 CSI Driver \u6ca1\u6709\u5b89\u88c5\uff0c\u6216\u8005kubelet\u6ca1\u6709\u8bbf\u95ee CSI Driver \u7684\u6743\u9650\u5bfc\u81f4\u7684\u3002 \u7531\u4e8e CSI Driver \u662f\u7531 Kubelet \u8fdb\u884c\u56de\u8c03\uff0c\u4f46\u662f\u5982\u679c Fluid \u6ca1\u6709\u5b89\u88c5 CSI Driver \u6216\u8005 Kubelet \u6ca1\u6709\u6743\u9650\u67e5\u770b CSI Driver\uff0c\u5c31\u4f1a\u5bfc\u81f4 CSI Plugin \u6ca1\u6709\u88ab\u6b63\u786e\u89e6\u53d1\u3002 \u9996\u5148\u9700\u8981\u4f7f\u7528\u547d\u4ee4 kubectl get csidriver \u67e5\u770b\u662f\u5426\u5b89\u88c5\u4e86 CSI Driver\u3002 \u5982\u679c\u6ca1\u6709\u5b89\u88c5\uff0c\u4f7f\u7528\u547d\u4ee4 kubectl apply -f charts/fluid/fluid/templates/csi/driver.yaml \u8fdb\u884c\u5b89\u88c5\uff0c\u7136\u540e\u89c2\u5bdf PVC \u662f\u5426\u6210\u529f\u6302\u8f7d\u5230\u5e94\u7528\u7a0b\u5e8f\u4e2d\u3002 \u5982\u679c\u4ecd\u672a\u80fd\u89e3\u51b3\uff0c\u4f7f\u7528 export KUBECONFIG=/etc/kubernetes/kubelet.conf && kubectl get csidriver \u6765\u67e5\u770b Kubelet \u80fd\u591f\u5177\u6709\u6743\u9650\u770b\u5230 CSI Driver","title":"\u95ee\u9898\u8bca\u65ad\u53ca\u5904\u7406"},{"location":"fluid-jindoruntime/jindo_fluid_question/#_1","text":"\u5982\u60a8\u7684\u73af\u5883\u51fa\u73b0\u95ee\u9898\uff0c\u4e5f\u53ef\u5f00 ISSUE \u7ed9\u6211\u4eec\uff0c\u4f1a\u7b2c\u4e00\u65f6\u95f4\u5904\u7406","title":"\u95ee\u9898\u8bca\u65ad\u53ca\u5904\u7406"},{"location":"fluid-jindoruntime/jindo_fluid_question/#fluid","text":"","title":"Fluid \u5b89\u88c5\u4f7f\u7528\u76f8\u5173\u95ee\u9898"},{"location":"fluid-jindoruntime/jindo_fluid_question/#1-helmfluid","text":"\u56de\u7b54 :\u63a8\u8350\u6309\u7167\u5b89\u88c5\u6587\u6863\u4f9d\u6b21\u786e\u8ba4Fluid\u7ec4\u4ef6\u662f\u5426\u6b63\u5e38\u8fd0\u884c\u3002 Fluid\u5b89\u88c5\u6587\u6863\u662f\u4ee5 Helm 3 \u4e3a\u4f8b\u8fdb\u884c\u90e8\u7f72\u7684\u3002\u5982\u679c\u60a8\u4f7f\u7528 Helm 3 \u4ee5\u4e0b\u7684\u7248\u672c\u90e8\u7f72Fluid\uff0c \u5e76\u4e14\u9047\u5230\u4e86 CRD\u6ca1\u6709\u6b63\u5e38\u542f\u52a8 \u7684\u60c5\u51b5\uff0c\u8fd9\u53ef\u80fd\u662f\u56e0\u4e3a Helm 3 \u53ca\u5176\u4ee5\u4e0a\u7248\u672c\u4f1a\u5728 helm install \u7684\u65f6\u5019\u81ea\u52a8\u5b89\u88c5CRD\uff0c \u800c\u4f4e\u7248\u672c\u7684Helm\u5219\u4e0d\u4f1a\u3002 \u53c2\u89c1 Helm\u5b98\u65b9\u6587\u6863\u8bf4\u660e \u3002 \u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u60a8\u9700\u8981\u624b\u52a8\u5b89\u88c5CRD\uff1a $ kubectl create -f fluid/crds","title":"1. \u4e3a\u4ec0\u4e48\u6211\u4f7f\u7528Helm\u5b89\u88c5fluid\u5931\u8d25\u4e86\uff1f"},{"location":"fluid-jindoruntime/jindo_fluid_question/#2-runtime","text":"\u56de\u7b54 :\u8bf7\u68c0\u67e5\u76f8\u5173Pod\u8fd0\u884c\u72b6\u6001\u548cRuntime\u7684Events\u3002 \u53ea\u8981\u6709\u4efb\u4f55\u6d3b\u8dc3Pod\u8fd8\u5728\u4f7f\u7528Fluid\u521b\u5efa\u7684Volume\uff0cFluid\u5c31\u4e0d\u4f1a\u5b8c\u6210\u5220\u9664\u64cd\u4f5c\u3002 \u5982\u4e0b\u7684\u547d\u4ee4\u53ef\u4ee5\u5feb\u901f\u5730\u627e\u51fa\u8fd9\u4e9b\u6d3b\u8dc3Pod\uff0c\u4f7f\u7528\u65f6\u628a <dataset_name> \u548c <dataset_namespace> \u6362\u6210\u81ea\u5df1\u7684\u5373\u53ef\uff1a kubectl describe pvc <dataset_name> -n <dataset_namespace> | \\ awk '/^Mounted/ {flag=1}; /^Events/ {flag=0}; flag' | \\ awk 'NR==1 {print $3}; NR!=1 {print $1}' | \\ xargs -I {} kubectl get po {} | \\ grep -E \"Running|Terminating|Pending\" | \\ cut -d \" \" -f 1","title":"2. \u4e3a\u4ec0\u4e48\u6211\u65e0\u6cd5\u5220\u9664Runtime\uff1f"},{"location":"fluid-jindoruntime/jindo_fluid_question/#3-runtime-pvc-driver-name-fusecsifluidio-not-found-in-the-list-of-registered-csi-drivers","text":"\u56de\u7b54 :\u8bf7\u67e5\u770b\u4efb\u52a1\u88ab\u8c03\u5ea6\u8282\u70b9\u6240\u5728\u7684 kubelet \u914d\u7f6e\u662f\u5426\u4e3a\u9ed8\u8ba4 /var/lib/kubelet \u3002 \u9996\u5148\u901a\u8fc7\u547d\u4ee4\u67e5\u770bFluid\u7684CSI\u7ec4\u4ef6\u662f\u5426\u6b63\u5e38 \u5982\u4e0b\u7684\u547d\u4ee4\u53ef\u4ee5\u5feb\u901f\u5730\u627e\u51faPod\uff0c\u4f7f\u7528\u65f6\u628a <node_name> \u548c <fluid_namespace> \u6362\u6210\u81ea\u5df1\u7684\u5373\u53ef\uff1a kubectl get pod -n <fluid_namespace> | grep <node_name> # <pod_name> \u4e3a\u4e0a\u4e00\u6b65pod\u540d kubectl logs <pod_name> node-driver-registrar -n <fluid_namespace> kubectl logs <pod_name> plugins -n <fluid_namespace> \u5982\u679c\u4e0a\u8ff0\u6b65\u9aa4\u7684Log\u65e0\u9519\u8bef\uff0c\u8bf7\u67e5\u770bcsidriver\u5bf9\u8c61\u662f\u5426\u5b58\u5728\uff1a kubectl get csidriver \u5982\u679ccsidriver\u5bf9\u8c61\u5b58\u5728\uff0c\u8bf7\u67e5\u770b\u67e5\u770bcsi\u6ce8\u518c\u8282\u70b9\u662f\u5426\u5305\u542b <node_name> \uff1a kubectl get csinode | grep <node_name> \u5982\u679c\u4e0a\u8ff0\u547d\u4ee4\u65e0\u8f93\u51fa\uff0c\u67e5\u770b\u4efb\u52a1\u88ab\u8c03\u5ea6\u8282\u70b9\u6240\u5728\u7684 kubelet \u914d\u7f6e\u662f\u5426\u4e3a\u9ed8\u8ba4 /var/lib/kubelet \u3002\u56e0\u4e3aFluid\u7684CSI\u7ec4\u4ef6\u901a\u8fc7\u56fa\u5b9a\u5730\u5740\u7684socket\u6ce8\u518c\u5230kubelet\uff0c\u9ed8\u8ba4\u4e3a --csi-address=/var/lib/kubelet/csi-plugins/fuse.csi.fluid.io/csi.sock --kubelet-registration-path=/var/lib/kubelet/csi-plugins/fuse.csi.fluid.io/csi.sock \u3002","title":"3. \u4e3a\u4ec0\u4e48\u6211\u5728\u521b\u5efa\u4efb\u52a1\u6302\u8f7d Runtime \u521b\u5efa\u7684 PVC \u7684\u65f6\u5019\u51fa\u73b0 driver name fuse.csi.fluid.io not found in the list of registered CSI drivers \u9519\u8bef\uff1f"},{"location":"fluid-jindoruntime/jindo_fluid_question/#4-fluid-kubectl-get-datasetdataset","text":"\u56de\u7b54 :\u7531\u4e8e\u6211\u4eec\u5728fluid\u7684\u5347\u7ea7\u8fc7\u7a0b\u4e2d\u53ef\u80fd\u66f4\u65b0\u4e86CRD\uff0c\u4f60\u5728\u65e7\u7248\u672c\u521b\u5efa\u7684dataset\uff0c\u4f1a\u5c06CRD\u4e2d\u65b0\u589e\u7684\u5b57\u6bb5\u8bbe\u7f6e\u4e3a\u7a7a \u4f8b\u5982\uff0c\u5982\u679c\u4f60\u4ecev0.4\u6216\u66f4\u65e9\u7248\u672c\u5347\u7ea7\uff0c\u90a3\u65f6\u5019\u7684dataset\u6ca1\u6709FileNum\u5b57\u6bb5 \u66f4\u65b0fluid\u540e\uff0c\u5982\u679c\u4f60\u4f7f\u7528 kubectl get \u547d\u4ee4\uff0c\u65e0\u6cd5\u67e5\u8be2\u5230\u8be5dataset\u7684FileNum \u4f60\u53ef\u4ee5\u91cd\u5efadataset\uff0c\u65b0\u5efa\u7684dataset\u4f1a\u6b63\u5e38\u663e\u793a\u8fd9\u4e9b\u5b57\u6bb5","title":"4. \u4e3a\u4ec0\u4e48\u66f4\u65b0\u4e86fluid\u540e\uff0c\u4f7f\u7528 kubectl get \u67e5\u8be2\u66f4\u65b0\u524d\u521b\u5efa\u7684dataset\uff0c\u53d1\u73b0\u76f8\u6bd4\u65b0\u5efa\u7684dataset\u7f3a\u5c11\u4e86\u67d0\u4e9b\u5b57\u6bb5\uff1f"},{"location":"fluid-jindoruntime/jindo_fluid_question/#5-pvc-volume-attachment","text":"\u56de\u7b54 :Volume Attachment \u8d85\u65f6\u95ee\u9898\u662f Kubelet \u8fdb\u884c\u8bf7\u6c42 CSI Driver \u65f6\u672a\u6536\u5230 CSI Driver \u7684\u54cd\u5e94\u800c\u9020\u6210\u7684\u8d85\u65f6\u3002 \u8be5\u95ee\u9898\u662f\u7531\u4e8e Fluid \u7684 CSI Driver \u6ca1\u6709\u5b89\u88c5\uff0c\u6216\u8005kubelet\u6ca1\u6709\u8bbf\u95ee CSI Driver \u7684\u6743\u9650\u5bfc\u81f4\u7684\u3002 \u7531\u4e8e CSI Driver \u662f\u7531 Kubelet \u8fdb\u884c\u56de\u8c03\uff0c\u4f46\u662f\u5982\u679c Fluid \u6ca1\u6709\u5b89\u88c5 CSI Driver \u6216\u8005 Kubelet \u6ca1\u6709\u6743\u9650\u67e5\u770b CSI Driver\uff0c\u5c31\u4f1a\u5bfc\u81f4 CSI Plugin \u6ca1\u6709\u88ab\u6b63\u786e\u89e6\u53d1\u3002 \u9996\u5148\u9700\u8981\u4f7f\u7528\u547d\u4ee4 kubectl get csidriver \u67e5\u770b\u662f\u5426\u5b89\u88c5\u4e86 CSI Driver\u3002 \u5982\u679c\u6ca1\u6709\u5b89\u88c5\uff0c\u4f7f\u7528\u547d\u4ee4 kubectl apply -f charts/fluid/fluid/templates/csi/driver.yaml \u8fdb\u884c\u5b89\u88c5\uff0c\u7136\u540e\u89c2\u5bdf PVC \u662f\u5426\u6210\u529f\u6302\u8f7d\u5230\u5e94\u7528\u7a0b\u5e8f\u4e2d\u3002 \u5982\u679c\u4ecd\u672a\u80fd\u89e3\u51b3\uff0c\u4f7f\u7528 export KUBECONFIG=/etc/kubernetes/kubelet.conf && kubectl get csidriver \u6765\u67e5\u770b Kubelet \u80fd\u591f\u5177\u6709\u6743\u9650\u770b\u5230 CSI Driver","title":"5. \u4e3a\u4ec0\u4e48\u5728\u5e94\u7528\u7a0b\u5e8f\u4e2d\u4f7f\u7528 PVC \u65f6\u4f1a\u4ea7\u751f\u4e86 Volume Attachment \u8d85\u65f6\u95ee\u9898\uff1f"},{"location":"fluid-jindoruntime/jindo_fluid_s3_ufs_example/","text":"\u62e5\u62b1\u4e91\u539f\u751f\uff0cFluid \u7ed3\u5408 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff1a\u52a0\u901f\u5bf9\u8c61\u5b58\u50a8\u4f7f\u7528 S3 \u534f\u8bae \u00a7 JindoFSx \u63d0\u4f9b\u4e86\u8bbf\u95ee S3 \u534f\u8bae\u7684\u80fd\u529b\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e S3 \u4f5c\u4e3a JindoFSx \u7684\u540e\u7aef\u5b58\u50a8\uff0c\u4f7f Fluid \u901a\u8fc7 JindoFSx \u6765\u8bbf\u95ee S3 \u4e0a\u7684\u6570\u636e\uff0c\u540c\u65f6 JindoFSx \u4e5f\u63d0\u4f9b\u4e86\u5bf9 S3 \u4e0a\u7684\u6570\u636e\u4ee5\u53ca\u5143\u6570\u636e\u7684\u7f13\u5b58\u52a0\u901f\u529f\u80fd\u3002 \u672c\u6587\u6863\u5c55\u793a\u4e86\u5982\u4f55\u5728 Fluid \u4e2d\u4ee5\u58f0\u660e\u5f0f\u7684\u65b9\u5f0f\u5b8c\u6210 JindoFSx \u90e8\u7f72\uff0c\u5bf9\u63a5 S3 \u6570\u636e\u6e90\uff08\u5305\u62ec\u63d0\u4f9b S3 \u517c\u5bb9\u7684\u6240\u6709\u5bf9\u8c61\u5b58\u50a8\uff09\u3002 \u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 \u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz \u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: s3 spec: mounts: - mountPoint: s3://<bucket>/<dir> options: fs.s3.accessKeyId: xxx fs.s3.accessKeySecret: xxx fs.s3.endpoint: s3.ap-east-1.amazonaws.com fs.s3.region: ap-east-1 name: s3 path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: s3 spec: replicas: 2 tieredstore: levels: - mediumtype: MEM path: /dev/shm quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d s3 \u7684\u8def\u5f84\uff0c\u652f\u6301\u6807\u51c6 s3 \u534f\u8bae\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u7684 worker \u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c\u5355\u4f4dGi\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b dataset \u7684\u60c5\u51b5 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE s3 210MiB 0.00B 180.00GiB 0.0% Bound 1h \u6267\u884c create \u547d\u4ee4 $ kubectl create -f runtime.yaml \u67e5\u770b dataset \u72b6\u6001 \u00a7 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE s3 180.01GiB 0.00B 261.00GiB 0.0% Bound 62m \u68c0\u67e5 PV\uff0cPVC \u521b\u5efa\u60c5\u51b5 \u00a7 $ kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/s3 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/s3 Bound hadoop 100Gi RWX 58m","title":"\u62e5\u62b1\u4e91\u539f\u751f\uff0cFluid \u7ed3\u5408 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff1a\u52a0\u901f\u5bf9\u8c61\u5b58\u50a8\u4f7f\u7528 S3 \u534f\u8bae"},{"location":"fluid-jindoruntime/jindo_fluid_s3_ufs_example/#fluid-jindofsx-s3","text":"JindoFSx \u63d0\u4f9b\u4e86\u8bbf\u95ee S3 \u534f\u8bae\u7684\u80fd\u529b\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e S3 \u4f5c\u4e3a JindoFSx \u7684\u540e\u7aef\u5b58\u50a8\uff0c\u4f7f Fluid \u901a\u8fc7 JindoFSx \u6765\u8bbf\u95ee S3 \u4e0a\u7684\u6570\u636e\uff0c\u540c\u65f6 JindoFSx \u4e5f\u63d0\u4f9b\u4e86\u5bf9 S3 \u4e0a\u7684\u6570\u636e\u4ee5\u53ca\u5143\u6570\u636e\u7684\u7f13\u5b58\u52a0\u901f\u529f\u80fd\u3002 \u672c\u6587\u6863\u5c55\u793a\u4e86\u5982\u4f55\u5728 Fluid \u4e2d\u4ee5\u58f0\u660e\u5f0f\u7684\u65b9\u5f0f\u5b8c\u6210 JindoFSx \u90e8\u7f72\uff0c\u5bf9\u63a5 S3 \u6570\u636e\u6e90\uff08\u5305\u62ec\u63d0\u4f9b S3 \u517c\u5bb9\u7684\u6240\u6709\u5bf9\u8c61\u5b58\u50a8\uff09\u3002","title":"\u62e5\u62b1\u4e91\u539f\u751f\uff0cFluid \u7ed3\u5408 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff1a\u52a0\u901f\u5bf9\u8c61\u5b58\u50a8\u4f7f\u7528 S3 \u534f\u8bae"},{"location":"fluid-jindoruntime/jindo_fluid_s3_ufs_example/#_1","text":"kubectl create ns fluid-system","title":"\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"fluid-jindoruntime/jindo_fluid_s3_ufs_example/#fluid","text":"","title":"\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"fluid-jindoruntime/jindo_fluid_s3_ufs_example/#helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"fluid-jindoruntime/jindo_fluid_s3_ufs_example/#fluid_1","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"fluid-jindoruntime/jindo_fluid_s3_ufs_example/#dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: s3 spec: mounts: - mountPoint: s3://<bucket>/<dir> options: fs.s3.accessKeyId: xxx fs.s3.accessKeySecret: xxx fs.s3.endpoint: s3.ap-east-1.amazonaws.com fs.s3.region: ap-east-1 name: s3 path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: s3 spec: replicas: 2 tieredstore: levels: - mediumtype: MEM path: /dev/shm quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d s3 \u7684\u8def\u5f84\uff0c\u652f\u6301\u6807\u51c6 s3 \u534f\u8bae\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u7684 worker \u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c\u5355\u4f4dGi\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b dataset \u7684\u60c5\u51b5 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE s3 210MiB 0.00B 180.00GiB 0.0% Bound 1h \u6267\u884c create \u547d\u4ee4 $ kubectl create -f runtime.yaml","title":"\u521b\u5efa dataset \u548c JindoRuntime"},{"location":"fluid-jindoruntime/jindo_fluid_s3_ufs_example/#dataset","text":"$ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE s3 180.01GiB 0.00B 261.00GiB 0.0% Bound 62m","title":"\u67e5\u770b dataset \u72b6\u6001"},{"location":"fluid-jindoruntime/jindo_fluid_s3_ufs_example/#pvpvc","text":"$ kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/s3 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/s3 Bound hadoop 100Gi RWX 58m","title":"\u68c0\u67e5 PV\uff0cPVC \u521b\u5efa\u60c5\u51b5"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/","text":"Fluid + JindoCache \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u9ad8\u9636\u529f\u80fd\u4f7f\u7528 \u00a7 \u4f7f\u7528\u5217\u8868 \u6302\u8f7d\u70b9\u5728\u6839\u76ee\u5f55\u4e0b Secret \u52a0\u5bc6 AK \u53c2\u6570 Raft 3 master \u6a21\u5f0f \u4f7f\u7528 Placement \u90e8\u7f72\u591a\u4e2a runtime \u4f7f\u7528 NoseSelector \u90e8\u7f72\u8282\u70b9 \u4f7f\u7528 dataset nodeAffinity \u529f\u80fd Worker \u4e2a\u6570\u6269\u7f29\u5bb9 \u4f7f\u7528 tolerations \u529f\u80fd Resource \u8d44\u6e90 Fuse \u56de\u6536\u7b56\u7565 JindoCache Fuse \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528 FuseOnly \u4f7f\u7528\u65b9\u5f0f \u591a\u6302\u8f7d\u70b9 master\u5143\u6570\u636e\u6301\u4e45\u5316 \u66f4\u591a\u7f13\u5b58\u7b56\u7565 \u6302\u8f7d\u70b9\u5728\u6839\u76ee\u5f55\u4e0b \u00a7 \u9ed8\u8ba4\u4f7f\u7528 JindoRuntime \u4f1a\u5728\u6302\u8f7d\u70b9\u591a\u4e00\u5c42 / \u7684\u76ee\u5f55\uff0c\u5982\u679c\u60f3\u6302\u8f7d\u5728\u6839\u76ee\u5f55\u4e0b\u53ef\u4ee5\u5728 dataset \u91cc\u8fdb\u884c\u53c2\u6570\u6307\u5b9a apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase path: / \u6307\u5b9a spec.mounts.path = /\uff0c\u5219\u4f1a\u5c06\u6587\u4ef6\u6302\u8f7d\u5728\u6839\u76ee\u5f55\u4e0b Secret \u52a0\u5bc6 AK \u53c2\u6570 \u00a7 apiVersion: v1 kind: Secret metadata: name: mysecret stringData: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> \u5728 dataset \u91cc\u4f7f\u7528 secret apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://<OSS_BUCKET>/<OSS_DIRECTORY>/ name: hadoop options: fs.oss.endpoint: <OSS_ENDPOINT> encryptOptions: - name: fs.oss.accessKeyId valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeyId - name: fs.oss.accessKeySecret valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeySecret Raft 3 master \u6a21\u5f0f \u00a7 JindoRuntime \u6a21\u5f0f\u542f\u52a8\u4e00\u4e2a master\uff0c\u53ef\u4ee5\u901a\u8fc7\u4f7f\u7528 master.replicas \u6765\u542f\u52a8 3 \u4e2a master \u8fdb\u884c HA \u8f6c\u6362\u3002 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 3 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 2G high: \"0.8\" low: \"0.7\" master: replicas: 3 master.replicas = 3 : \u542f\u52a8 3 \u4e2a master pod \u4f7f\u7528 Placement \u90e8\u7f72\u591a\u4e2a runtime \u00a7 JindoRuntime \u542f\u52a8\u540e\uff0c\u9ed8\u8ba4 worker \u8282\u70b9\u4e0a\u53ea\u80fd\u542f\u52a8\u4e00\u4e2a runtime \u7684worker\uff0c\u5c5e\u4e8e\u72ec\u5360\u6a21\u5f0f\uff0c\u5982\u679c\u60f3\u8981\u5728\u540c\u4e00\u4e2a\u8282\u70b9\u4e0a\u652f\u6301\u90e8\u7f72\u591a\u4e2a worker\uff0c\u53ef\u4ee5\u4f7f\u7528 shared \u6a21\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase placement: \"Shared\" \u4f7f\u7528 NoseSelector \u90e8\u7f72\u8282\u70b9 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /mnt/disk1 quota: 10Gi high: \"0.9\" low: \"0.8\" master: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1 worker: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1 fuse: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1 \u4f7f\u7528 dataset nodeAffinity \u529f\u80fd \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: hbase-cache operator: In values: - \"true\" Worker \u4e2a\u6570\u6269\u7f29\u5bb9 \u00a7 \u4f7f\u7528 kubectl scale \u5bf9 Runtime \u7684 Worker \u6570\u91cf\u8fdb\u884c\u8c03\u6574 $ kubectl scale jindoruntime <runtime_name> --replicas=<replica_num> \u5176\u4e2d * runtime_name\uff1aruntime \u7684\u540d\u5b57 * replica_num\uff1a\u8868\u793a\u6269\u6240\u5bb9\u7684 Worker \u6570\u91cf \u4f7f\u7528 tolerations \u529f\u80fd \u00a7 \u60a8\u53ef\u4ee5\u5728 dataset \u91cc\u5b9a\u4e49 tolerations \u6307\u5b9a worker \u8282\u70b9\u7684\u8c03\u5ea6 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase tolerations: - key: hbase operator: Equal value: \"true\" \u4e5f\u53ef\u4ee5\u4e3a master/worker/fuse \u5355\u72ec\u6307\u5b9a tolerations \u8c03\u5ea6\u7b56\u7565 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 3 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 2Gi high: \"0.8\" low: \"0.7\" master: tolerations: - key: hbase operator: Equal value: \"true\" worker: tolerations: - key: hbase operator: Equal value: \"true\" fuse: tolerations: - key: hbase operator: Equal value: \"true\" Resource \u8d44\u6e90 \u00a7 \u53ef\u4ee5\u6307\u5b9a master/worker \u7b49\u7684 resource \u8d44\u6e90 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" master: resources: limits: cpu: \"4\" memory: \"8Gi\" requests: cpu: \"2\" memory: \"3Gi\" worker: resources: limits: cpu: \"4\" memory: \"8Gi\" requests: cpu: \"2\" memory: \"3Gi\" Fuse \u56de\u6536\u7b56\u7565 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: cleanPolicy: OnDemand fuse.cleanPolicy: onDemand / OnRuntimeDeleted \u5206\u522b\u8868\u793a\u6309\u9700\u542f\u52a8\uff0c\u4efb\u52a1\u7ed3\u675f fuse \u4e5f\u7ed3\u675f\u548c\u5728 runtime \u9500\u6bc1\u7684\u65f6\u5019\u548c\u5176\u4ed6\u7ec4\u4ef6\u4e00\u8d77\u9500\u6bc1\u6389 JindoCache Fuse \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528 \u00a7 \u53c2\u6570\u540d\u79f0 \u53c2\u6570\u8bf4\u660e \u4f7f\u7528\u8303\u4f8b auto_unmount fuse\u8fdb\u7a0b\u9000\u51fa\u540e\u81ea\u52a8umount\u6302\u8f7d\u8282\u70b9\u3002 -oauto_unmount ro \u53ea\u8bfb\u6302\u8f7d\uff0c\u542f\u7528\u53c2\u6570\u540e\u4e0d\u5141\u8bb8\u5199\u64cd\u4f5c\u3002 -oro direct_io \u5f00\u542f\u540e\uff0c\u8bfb\u5199\u6587\u4ef6\u53ef\u4ee5\u7ed5\u8fc7page cache\u3002 -odirect_io kernel_cache \u5f00\u542f\u540e\uff0c\u5229\u7528\u5185\u6838\u7f13\u5b58\u4f18\u5316\u8bfb\u6027\u80fd\u3002 -okernel_cache auto_cache \u9ed8\u8ba4\u5f00\u542f\uff0c\u4e0ekernel_cache \u4e8c\u9009\u4e00\uff0c\u4e0ekernel_cache\u4e0d\u540c\u7684\u662f\uff0c\u5982\u679c\u6587\u4ef6\u5927\u5c0f\u6216\u4fee\u6539\u65f6\u95f4\u53d1\u751f\u53d8\u5316\uff0c\u7f13\u5b58\u5c31\u4f1a\u5931\u6548\u3002 -oauto_cache entry_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-oentry_timeout=60 attr_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-oattr_timeout=60 negative_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u5931\u8d25\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-onegative_timeout=30 \u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: args: - -oro - -oattr_timeout=60 - -oentry_timeout=60 - -onegative_timeout=60 \u4ee5\u6570\u7ec4\u65b9\u5f0f\u5199\u5728 spec.fuse.args \u91cc\uff0c\u6309\u7167\u9700\u8981\u586b\u5199\u5373\u53ef \u5982\u5355\u72ec\u914d\u7f6e\u5ba2\u6237\u7aefpread\u53c2\u6570\uff1a apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: args: - -oro - -ometrics_port=0 - -okernel_cache - -oattr_timeout=7200 - -oentry_timeout=7200 - -onegative_timeout=7200 - -opread FuseOnly \u4f7f\u7528\u65b9\u5f0f \u00a7 \u5982\u60a8\u60f3\u4f7f\u7528 FuseOnly \u65b9\u5f0f\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: master: disabled: true worker: disabled: true \u591a\u6302\u8f7d\u70b9 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: spark - mountPoint: oss://test-bucket-2/dir2/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop \u5728\u6302\u8f7d pvc \u548c fuse \u4ee5\u53ca\u505a dataload \u7684\u65f6\u5019\uff0c\u8fd9\u4e24\u4e2a\u6570\u636e\u6e90\u7684\u65b9\u5f0f\u5c06\u4ee5 mounts.name \u4f5c\u4e3a\u533a\u5206\uff0c\u6bd4\u5982 /spark \u8def\u5f84\u4e0b\u8bbf\u95ee oss://test-bucket-1/dir1/ \u4e0b\u7684\u6587\u4ef6\uff0c/hadoop \u8def\u5f84\u4e0b\u8bbf\u95ee oss://test-bucket-2/dir2/ \u4e0b\u7684\u6587\u4ef6\u5185\u5bb9 master\u5143\u6570\u636e\u6301\u4e45\u5316 \u00a7 \u5982\u679c\u60f3\u901a\u8fc7\u7ed9master\u6302\u8f7dvolume\u7684\u65b9\u5f0f\uff0c\u8ba9master\u7684\u5143\u6570\u636e\u6301\u4e45\u5316\u5230\u6307\u5b9a\u7684\u5b58\u50a8\u4e0a\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" volumes: - name: nas persistentVolumeClaim: claimName: nas-test master: volumeMounts: - name: nas subPath: test mountPath: /var/cache/ properties: namespace.meta-dir: \"/var/cache/\" \u8be5\u4f8b\u5b50\u5c06\u5143\u6570\u636e\u76ee\u5f55 namespace.meta-dir \u6307\u5b9a\u5230\u6302\u8f7d\u7684volume\u4e0a\uff0c\u5e76\u8fdb\u884c\u6301\u4e45\u5316\uff0c\u6ce8\u610f\u6302\u8f7d\u7684vomlume\u5fc5\u987b\u5c31\u6709\u8bfb/\u5199/\u5220\u9664\u6743\u9650\uff0c\u4e0d\u53ef\u6307\u5b9a\u4e3a\u53ea\u8bfb \u66f4\u591a\u7f13\u5b58\u7b56\u7565 \u00a7 \u5143\u6570\u636e\u7f13\u5b58 \u00a7 \u5143\u6570\u636e\u7f13\u5b58\u6709\u4e24\u79cd\u7b56\u7565\uff0c\u53ef\u4ee5 dataset \u7684\u5b9a\u4e49\u4e2d\u8fdb\u884c\u8bbe\u7f6e\uff0c\u6253\u5f00\u548c\u5173\u95ed\u5206\u522b\u5bf9\u5e94 ONCE \u548c ALWAYS\uff0c\u9ed8\u8ba4\u7b56\u7565\u662f\u5173\u95ed\u3002 \u5143\u6570\u636e\u7f13\u5b58\u5f00\u5173 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: metaPolicy: ONCE fs.oss.endpoint: <OSS_ENDPOINT> \u5173\u95ed\u5143\u6570\u636e\u7f13\u5b58\uff08\u9ed8\u8ba4\uff09 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: metaPolicy: ALWAYS fs.oss.endpoint: <OSS_ENDPOINT> \u5199\u65f6\u843d\u7f13\u5b58\uff08WRITE_THROUGH\uff09 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: writePolicy: WRITE_THROUGH metaPolicy: ONCE # \u53ef\u4ee5\u6253\u5f00\u6216\u5173\u95ed fs.oss.endpoint: <OSS_ENDPOINT> \u5199\u4e34\u65f6\u7f13\u5b58\uff08CACHE_ONLY\uff09 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: writePolicy: CACHE_ONLY metaPolicy: ONCE #\u5fc5\u987b\u6253\u5f00\u5143\u6570\u636e\u7f13\u5b58 fs.oss.endpoint: <OSS_ENDPOINT> DHT \u7b56\u7565\uff08\u6d77\u91cf\u5c0f\u6587\u4ef6\u53ea\u8bfb\u573a\u666f\uff09 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: cacheStrategy: DHT metaPolicy: ONCE #\u5fc5\u987b\u6253\u5f00\u5143\u6570\u636e\u7f13\u5b58 fs.oss.endpoint: <OSS_ENDPOINT>","title":"Fluid + JindoCache \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u9ad8\u9636\u529f\u80fd\u4f7f\u7528"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#fluid-jindocache","text":"\u4f7f\u7528\u5217\u8868 \u6302\u8f7d\u70b9\u5728\u6839\u76ee\u5f55\u4e0b Secret \u52a0\u5bc6 AK \u53c2\u6570 Raft 3 master \u6a21\u5f0f \u4f7f\u7528 Placement \u90e8\u7f72\u591a\u4e2a runtime \u4f7f\u7528 NoseSelector \u90e8\u7f72\u8282\u70b9 \u4f7f\u7528 dataset nodeAffinity \u529f\u80fd Worker \u4e2a\u6570\u6269\u7f29\u5bb9 \u4f7f\u7528 tolerations \u529f\u80fd Resource \u8d44\u6e90 Fuse \u56de\u6536\u7b56\u7565 JindoCache Fuse \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528 FuseOnly \u4f7f\u7528\u65b9\u5f0f \u591a\u6302\u8f7d\u70b9 master\u5143\u6570\u636e\u6301\u4e45\u5316 \u66f4\u591a\u7f13\u5b58\u7b56\u7565","title":"Fluid + JindoCache \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u9ad8\u9636\u529f\u80fd\u4f7f\u7528"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#_1","text":"\u9ed8\u8ba4\u4f7f\u7528 JindoRuntime \u4f1a\u5728\u6302\u8f7d\u70b9\u591a\u4e00\u5c42 / \u7684\u76ee\u5f55\uff0c\u5982\u679c\u60f3\u6302\u8f7d\u5728\u6839\u76ee\u5f55\u4e0b\u53ef\u4ee5\u5728 dataset \u91cc\u8fdb\u884c\u53c2\u6570\u6307\u5b9a apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase path: / \u6307\u5b9a spec.mounts.path = /\uff0c\u5219\u4f1a\u5c06\u6587\u4ef6\u6302\u8f7d\u5728\u6839\u76ee\u5f55\u4e0b","title":"\u6302\u8f7d\u70b9\u5728\u6839\u76ee\u5f55\u4e0b"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#secret-ak","text":"apiVersion: v1 kind: Secret metadata: name: mysecret stringData: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> \u5728 dataset \u91cc\u4f7f\u7528 secret apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://<OSS_BUCKET>/<OSS_DIRECTORY>/ name: hadoop options: fs.oss.endpoint: <OSS_ENDPOINT> encryptOptions: - name: fs.oss.accessKeyId valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeyId - name: fs.oss.accessKeySecret valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeySecret","title":"Secret \u52a0\u5bc6 AK \u53c2\u6570"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#raft-3-master","text":"JindoRuntime \u6a21\u5f0f\u542f\u52a8\u4e00\u4e2a master\uff0c\u53ef\u4ee5\u901a\u8fc7\u4f7f\u7528 master.replicas \u6765\u542f\u52a8 3 \u4e2a master \u8fdb\u884c HA \u8f6c\u6362\u3002 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 3 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 2G high: \"0.8\" low: \"0.7\" master: replicas: 3 master.replicas = 3 : \u542f\u52a8 3 \u4e2a master pod","title":"Raft 3 master \u6a21\u5f0f"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#placement-runtime","text":"JindoRuntime \u542f\u52a8\u540e\uff0c\u9ed8\u8ba4 worker \u8282\u70b9\u4e0a\u53ea\u80fd\u542f\u52a8\u4e00\u4e2a runtime \u7684worker\uff0c\u5c5e\u4e8e\u72ec\u5360\u6a21\u5f0f\uff0c\u5982\u679c\u60f3\u8981\u5728\u540c\u4e00\u4e2a\u8282\u70b9\u4e0a\u652f\u6301\u90e8\u7f72\u591a\u4e2a worker\uff0c\u53ef\u4ee5\u4f7f\u7528 shared \u6a21\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase placement: \"Shared\"","title":"\u4f7f\u7528 Placement \u90e8\u7f72\u591a\u4e2a runtime"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#noseselector","text":"apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /mnt/disk1 quota: 10Gi high: \"0.9\" low: \"0.8\" master: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1 worker: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1 fuse: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1","title":"\u4f7f\u7528 NoseSelector \u90e8\u7f72\u8282\u70b9"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#dataset-nodeaffinity","text":"apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: hbase-cache operator: In values: - \"true\"","title":"\u4f7f\u7528 dataset nodeAffinity \u529f\u80fd"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#worker","text":"\u4f7f\u7528 kubectl scale \u5bf9 Runtime \u7684 Worker \u6570\u91cf\u8fdb\u884c\u8c03\u6574 $ kubectl scale jindoruntime <runtime_name> --replicas=<replica_num> \u5176\u4e2d * runtime_name\uff1aruntime \u7684\u540d\u5b57 * replica_num\uff1a\u8868\u793a\u6269\u6240\u5bb9\u7684 Worker \u6570\u91cf","title":"Worker \u4e2a\u6570\u6269\u7f29\u5bb9"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#tolerations","text":"\u60a8\u53ef\u4ee5\u5728 dataset \u91cc\u5b9a\u4e49 tolerations \u6307\u5b9a worker \u8282\u70b9\u7684\u8c03\u5ea6 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase tolerations: - key: hbase operator: Equal value: \"true\" \u4e5f\u53ef\u4ee5\u4e3a master/worker/fuse \u5355\u72ec\u6307\u5b9a tolerations \u8c03\u5ea6\u7b56\u7565 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 3 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 2Gi high: \"0.8\" low: \"0.7\" master: tolerations: - key: hbase operator: Equal value: \"true\" worker: tolerations: - key: hbase operator: Equal value: \"true\" fuse: tolerations: - key: hbase operator: Equal value: \"true\"","title":"\u4f7f\u7528 tolerations \u529f\u80fd"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#resource","text":"\u53ef\u4ee5\u6307\u5b9a master/worker \u7b49\u7684 resource \u8d44\u6e90 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" master: resources: limits: cpu: \"4\" memory: \"8Gi\" requests: cpu: \"2\" memory: \"3Gi\" worker: resources: limits: cpu: \"4\" memory: \"8Gi\" requests: cpu: \"2\" memory: \"3Gi\"","title":"Resource \u8d44\u6e90"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#fuse","text":"apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: cleanPolicy: OnDemand fuse.cleanPolicy: onDemand / OnRuntimeDeleted \u5206\u522b\u8868\u793a\u6309\u9700\u542f\u52a8\uff0c\u4efb\u52a1\u7ed3\u675f fuse \u4e5f\u7ed3\u675f\u548c\u5728 runtime \u9500\u6bc1\u7684\u65f6\u5019\u548c\u5176\u4ed6\u7ec4\u4ef6\u4e00\u8d77\u9500\u6bc1\u6389","title":"Fuse \u56de\u6536\u7b56\u7565"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#jindocache-fuse","text":"\u53c2\u6570\u540d\u79f0 \u53c2\u6570\u8bf4\u660e \u4f7f\u7528\u8303\u4f8b auto_unmount fuse\u8fdb\u7a0b\u9000\u51fa\u540e\u81ea\u52a8umount\u6302\u8f7d\u8282\u70b9\u3002 -oauto_unmount ro \u53ea\u8bfb\u6302\u8f7d\uff0c\u542f\u7528\u53c2\u6570\u540e\u4e0d\u5141\u8bb8\u5199\u64cd\u4f5c\u3002 -oro direct_io \u5f00\u542f\u540e\uff0c\u8bfb\u5199\u6587\u4ef6\u53ef\u4ee5\u7ed5\u8fc7page cache\u3002 -odirect_io kernel_cache \u5f00\u542f\u540e\uff0c\u5229\u7528\u5185\u6838\u7f13\u5b58\u4f18\u5316\u8bfb\u6027\u80fd\u3002 -okernel_cache auto_cache \u9ed8\u8ba4\u5f00\u542f\uff0c\u4e0ekernel_cache \u4e8c\u9009\u4e00\uff0c\u4e0ekernel_cache\u4e0d\u540c\u7684\u662f\uff0c\u5982\u679c\u6587\u4ef6\u5927\u5c0f\u6216\u4fee\u6539\u65f6\u95f4\u53d1\u751f\u53d8\u5316\uff0c\u7f13\u5b58\u5c31\u4f1a\u5931\u6548\u3002 -oauto_cache entry_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-oentry_timeout=60 attr_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-oattr_timeout=60 negative_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u5931\u8d25\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-onegative_timeout=30 \u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: args: - -oro - -oattr_timeout=60 - -oentry_timeout=60 - -onegative_timeout=60 \u4ee5\u6570\u7ec4\u65b9\u5f0f\u5199\u5728 spec.fuse.args \u91cc\uff0c\u6309\u7167\u9700\u8981\u586b\u5199\u5373\u53ef \u5982\u5355\u72ec\u914d\u7f6e\u5ba2\u6237\u7aefpread\u53c2\u6570\uff1a apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: args: - -oro - -ometrics_port=0 - -okernel_cache - -oattr_timeout=7200 - -oentry_timeout=7200 - -onegative_timeout=7200 - -opread","title":"JindoCache Fuse \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#fuseonly","text":"\u5982\u60a8\u60f3\u4f7f\u7528 FuseOnly \u65b9\u5f0f\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: master: disabled: true worker: disabled: true","title":"FuseOnly \u4f7f\u7528\u65b9\u5f0f"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#_2","text":"apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: spark - mountPoint: oss://test-bucket-2/dir2/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop \u5728\u6302\u8f7d pvc \u548c fuse \u4ee5\u53ca\u505a dataload \u7684\u65f6\u5019\uff0c\u8fd9\u4e24\u4e2a\u6570\u636e\u6e90\u7684\u65b9\u5f0f\u5c06\u4ee5 mounts.name \u4f5c\u4e3a\u533a\u5206\uff0c\u6bd4\u5982 /spark \u8def\u5f84\u4e0b\u8bbf\u95ee oss://test-bucket-1/dir1/ \u4e0b\u7684\u6587\u4ef6\uff0c/hadoop \u8def\u5f84\u4e0b\u8bbf\u95ee oss://test-bucket-2/dir2/ \u4e0b\u7684\u6587\u4ef6\u5185\u5bb9","title":"\u591a\u6302\u8f7d\u70b9"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#master","text":"\u5982\u679c\u60f3\u901a\u8fc7\u7ed9master\u6302\u8f7dvolume\u7684\u65b9\u5f0f\uff0c\u8ba9master\u7684\u5143\u6570\u636e\u6301\u4e45\u5316\u5230\u6307\u5b9a\u7684\u5b58\u50a8\u4e0a\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" volumes: - name: nas persistentVolumeClaim: claimName: nas-test master: volumeMounts: - name: nas subPath: test mountPath: /var/cache/ properties: namespace.meta-dir: \"/var/cache/\" \u8be5\u4f8b\u5b50\u5c06\u5143\u6570\u636e\u76ee\u5f55 namespace.meta-dir \u6307\u5b9a\u5230\u6302\u8f7d\u7684volume\u4e0a\uff0c\u5e76\u8fdb\u884c\u6301\u4e45\u5316\uff0c\u6ce8\u610f\u6302\u8f7d\u7684vomlume\u5fc5\u987b\u5c31\u6709\u8bfb/\u5199/\u5220\u9664\u6743\u9650\uff0c\u4e0d\u53ef\u6307\u5b9a\u4e3a\u53ea\u8bfb","title":"master\u5143\u6570\u636e\u6301\u4e45\u5316"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#_3","text":"","title":"\u66f4\u591a\u7f13\u5b58\u7b56\u7565"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#_4","text":"\u5143\u6570\u636e\u7f13\u5b58\u6709\u4e24\u79cd\u7b56\u7565\uff0c\u53ef\u4ee5 dataset \u7684\u5b9a\u4e49\u4e2d\u8fdb\u884c\u8bbe\u7f6e\uff0c\u6253\u5f00\u548c\u5173\u95ed\u5206\u522b\u5bf9\u5e94 ONCE \u548c ALWAYS\uff0c\u9ed8\u8ba4\u7b56\u7565\u662f\u5173\u95ed\u3002 \u5143\u6570\u636e\u7f13\u5b58\u5f00\u5173 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: metaPolicy: ONCE fs.oss.endpoint: <OSS_ENDPOINT> \u5173\u95ed\u5143\u6570\u636e\u7f13\u5b58\uff08\u9ed8\u8ba4\uff09 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: metaPolicy: ALWAYS fs.oss.endpoint: <OSS_ENDPOINT>","title":"\u5143\u6570\u636e\u7f13\u5b58"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#write_through","text":"apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: writePolicy: WRITE_THROUGH metaPolicy: ONCE # \u53ef\u4ee5\u6253\u5f00\u6216\u5173\u95ed fs.oss.endpoint: <OSS_ENDPOINT>","title":"\u5199\u65f6\u843d\u7f13\u5b58\uff08WRITE_THROUGH\uff09"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#cache_only","text":"apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: writePolicy: CACHE_ONLY metaPolicy: ONCE #\u5fc5\u987b\u6253\u5f00\u5143\u6570\u636e\u7f13\u5b58 fs.oss.endpoint: <OSS_ENDPOINT>","title":"\u5199\u4e34\u65f6\u7f13\u5b58\uff08CACHE_ONLY\uff09"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#dht","text":"apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: cacheStrategy: DHT metaPolicy: ONCE #\u5fc5\u987b\u6253\u5f00\u5143\u6570\u636e\u7f13\u5b58 fs.oss.endpoint: <OSS_ENDPOINT>","title":"DHT \u7b56\u7565\uff08\u6d77\u91cf\u5c0f\u6587\u4ef6\u53ea\u8bfb\u573a\u666f\uff09"},{"location":"fluid-jindoruntime/case/jindo_fluid_nas_ufs_example/","text":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001\u4e0b\u8f7d\u5b89\u88c5\u5305 \u00a7 \u70b9\u51fb\u4e0b\u8f7d 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-0.7.0.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907 \u00a7 5.1 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: nas spec: mounts: - mountPoint: local:///mnt/nas name: nasData --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: nas spec: replicas: 2 tieredstore: levels: - mediumtype: MEM path: /dev/shm quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d NAS \u7684\u8def\u5f84\uff0c local:// \u4e3a\u672c\u5730\u8def\u5f84\u524d\u7f00\uff0c /mnt/nas \u8868\u793a\u672c\u5730\u771f\u5b9e\u8def\u5f84\uff0c\u8be5\u8def\u5f84\u9700\u8981\u5728\u5bbf\u4e3b\u673a\u4e0a\u53ef\u89c1\u3002 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JindoRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime nas NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE nas Ready Ready 62m \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/nas 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/nas Bound hadoop 100Gi RWX 58m \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 NAS \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: test spec: dataset: name: nas namespace: default target: - path: /nasData replicas: 1 \u5176\u4e2d /nasData \u4e2d nasData \u4e3a dataset \u4e2d\u6302\u8f7d\u70b9\u7684 name\uff0c\u53ef\u4ee5\u4f7f\u7528 /nasData/xxx \u6765\u9884\u70ed\u5176\u4e2d\u7684\u5b50\u76ee\u5f55\u3002 replicas \u8868\u793a\u7f13\u5b58\u7684\u5907\u4efd\u6570\uff0c\u9ed8\u8ba4\u662f1\u3002 \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload test-dataload NAME DATASET PHASE AGE dataload nas Completed 2m13s 6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: nas \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u8bbf\u95ee\u6302\u8f7d\u70b9\u5bf9\u5e94\u7684\u5e95\u5c42\u6587\u4ef6 $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 7\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime nas \u5220\u9664 dataset kubectl delete dataset nas","title":"Jindo fluid nas ufs example"},{"location":"fluid-jindoruntime/case/jindo_fluid_nas_ufs_example/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"fluid-jindoruntime/case/jindo_fluid_nas_ufs_example/#2","text":"\u70b9\u51fb\u4e0b\u8f7d","title":"2\u3001\u4e0b\u8f7d\u5b89\u88c5\u5305"},{"location":"fluid-jindoruntime/case/jindo_fluid_nas_ufs_example/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-0.7.0.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"fluid-jindoruntime/case/jindo_fluid_nas_ufs_example/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"fluid-jindoruntime/case/jindo_fluid_nas_ufs_example/#5","text":"","title":"5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907"},{"location":"fluid-jindoruntime/case/jindo_fluid_nas_ufs_example/#51-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: nas spec: mounts: - mountPoint: local:///mnt/nas name: nasData --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: nas spec: replicas: 2 tieredstore: levels: - mediumtype: MEM path: /dev/shm quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d NAS \u7684\u8def\u5f84\uff0c local:// \u4e3a\u672c\u5730\u8def\u5f84\u524d\u7f00\uff0c /mnt/nas \u8868\u793a\u672c\u5730\u771f\u5b9e\u8def\u5f84\uff0c\u8be5\u8def\u5f84\u9700\u8981\u5728\u5bbf\u4e3b\u673a\u4e0a\u53ef\u89c1\u3002 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JindoRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime nas NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE nas Ready Ready 62m \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/nas 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/nas Bound hadoop 100Gi RWX 58m \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 NAS \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: test spec: dataset: name: nas namespace: default target: - path: /nasData replicas: 1 \u5176\u4e2d /nasData \u4e2d nasData \u4e3a dataset \u4e2d\u6302\u8f7d\u70b9\u7684 name\uff0c\u53ef\u4ee5\u4f7f\u7528 /nasData/xxx \u6765\u9884\u70ed\u5176\u4e2d\u7684\u5b50\u76ee\u5f55\u3002 replicas \u8868\u793a\u7f13\u5b58\u7684\u5907\u4efd\u6570\uff0c\u9ed8\u8ba4\u662f1\u3002 \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload test-dataload NAME DATASET PHASE AGE dataload nas Completed 2m13s","title":"5.1 \u521b\u5efa dataset \u548c JindoRuntime"},{"location":"fluid-jindoruntime/case/jindo_fluid_nas_ufs_example/#6","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: nas \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u8bbf\u95ee\u6302\u8f7d\u70b9\u5bf9\u5e94\u7684\u5e95\u5c42\u6587\u4ef6 $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","title":"6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c"},{"location":"fluid-jindoruntime/case/jindo_fluid_nas_ufs_example/#7","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime nas \u5220\u9664 dataset kubectl delete dataset nas","title":"7\u3001\u73af\u5883\u6e05\u7406"},{"location":"fluid-jindoruntime/case/jindo_fluid_oss_append_fuseonly/","text":"\u52a0\u901fOSS\u4e0a\u6570\u636e \u00a7 \u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002 1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001 \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s 5\u3001\u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop path: / accessModes: - ReadWriteMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: master: disabled: true worker: disabled: true mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aoss bucket\u7684endpoint\u4fe1\u606f\uff0c\u516c\u7f51\u6216\u5185\u7f51\u5730\u5740\u7686\u53ef\uff0c\u5982\u60a8\u7684 bucket \u5728\u676d\u5dde Region\uff0c\u90a3\u4e48\u516c\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou.aliyuncs.com\uff0c\u5185\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou-internal.aliyuncs.com(\u5185\u7f51\u5730\u5740\u4f7f\u7528\u6761\u4ef6\u4e3a\u60a8\u7684 K8S \u96c6\u7fa4\u6240\u5728\u533a\u57df\u548coss\u533a\u57df\u76f8\u540c)\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m 6\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/case/jindo_fluid_oss_append_fuseonly/#oss","text":"\u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/case/jindo_fluid_oss_append_fuseonly/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"fluid-jindoruntime/case/jindo_fluid_oss_append_fuseonly/#2-fluid","text":"","title":"2\u3001\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"fluid-jindoruntime/case/jindo_fluid_oss_append_fuseonly/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"fluid-jindoruntime/case/jindo_fluid_oss_append_fuseonly/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"fluid-jindoruntime/case/jindo_fluid_oss_append_fuseonly/#5-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop path: / accessModes: - ReadWriteMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: master: disabled: true worker: disabled: true mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aoss bucket\u7684endpoint\u4fe1\u606f\uff0c\u516c\u7f51\u6216\u5185\u7f51\u5730\u5740\u7686\u53ef\uff0c\u5982\u60a8\u7684 bucket \u5728\u676d\u5dde Region\uff0c\u90a3\u4e48\u516c\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou.aliyuncs.com\uff0c\u5185\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou-internal.aliyuncs.com(\u5185\u7f51\u5730\u5740\u4f7f\u7528\u6761\u4ef6\u4e3a\u60a8\u7684 K8S \u96c6\u7fa4\u6240\u5728\u533a\u57df\u548coss\u533a\u57df\u76f8\u540c)\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m","title":"5\u3001\u521b\u5efa dataset \u548c JindoRuntime"},{"location":"fluid-jindoruntime/case/jindo_fluid_oss_append_fuseonly/#6","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"6\u3001\u73af\u5883\u6e05\u7406"},{"location":"fluid-jindoruntime/case/jindo_fluid_sdk_connect_example/","text":"\u4f7f\u7528 JindoSDK \u8bbf\u95ee Fluid JindoRuntime \u96c6\u7fa4 \u00a7 \u6b65\u9aa4 \u00a7 \u524d\u63d0\uff1a\u542f\u52a8 Fluid JindoRuntime \u96c6\u7fa4 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e core-site \u00a7 \u767b\u9646\u5230 xxx-jindofs-master-0 \u5bb9\u5668\u91cc kubectl exec -ti xxx-jindofs-master-0 bash \u627e\u5230 /hdfs-3.2.1/etc/hadoop/core-site.xml \u6587\u4ef6\uff0c\u5c06\u6587\u4ef6\u5185\u5bb9\u5168\u90e8\u52a0\u5165\u5230 hadoop \u7684 classpath \u4e0b\u7684 core-site \u6587\u4ef6\u4e2d 4. shell\u547d\u4ee4 \u00a7 \u53ef\u4ee5\u6267\u884c hadoop shell \u547d\u4ee4\u6765\u5224\u65ad\u670d\u52a1\u662f\u5426\u8fde\u63a5\u6210\u529f hadoop fs -ls oss://<Bucket>/<Dir> 6. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoFSx \u53c2\u6570\u8bf4\u660e JindoFSx \u5305\u542b\u4e00\u4e9b\u7f13\u5b58\u76f8\u5173\u7684\u53c2\u6570\uff0c\u8bf7\u53c2\u8003\u6587\u6863 JindoFSx \u5ba2\u6237\u7aef\u914d\u7f6e\u5217\u8868 * JindoSDK \u53c2\u6570\u8bf4\u660e JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 7. \u6743\u9650\u7ba1\u7406 \u00a7 \u81ea\u5efa\u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u963f\u91cc\u4e91 E-MapReduce \u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848","title":"\u4f7f\u7528 JindoSDK \u8bbf\u95ee Fluid JindoRuntime \u96c6\u7fa4"},{"location":"fluid-jindoruntime/case/jindo_fluid_sdk_connect_example/#jindosdk-fluid-jindoruntime","text":"","title":"\u4f7f\u7528 JindoSDK \u8bbf\u95ee Fluid JindoRuntime \u96c6\u7fa4"},{"location":"fluid-jindoruntime/case/jindo_fluid_sdk_connect_example/#_1","text":"\u524d\u63d0\uff1a\u542f\u52a8 Fluid JindoRuntime \u96c6\u7fa4","title":"\u6b65\u9aa4"},{"location":"fluid-jindoruntime/case/jindo_fluid_sdk_connect_example/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"fluid-jindoruntime/case/jindo_fluid_sdk_connect_example/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"fluid-jindoruntime/case/jindo_fluid_sdk_connect_example/#3-core-site","text":"\u767b\u9646\u5230 xxx-jindofs-master-0 \u5bb9\u5668\u91cc kubectl exec -ti xxx-jindofs-master-0 bash \u627e\u5230 /hdfs-3.2.1/etc/hadoop/core-site.xml \u6587\u4ef6\uff0c\u5c06\u6587\u4ef6\u5185\u5bb9\u5168\u90e8\u52a0\u5165\u5230 hadoop \u7684 classpath \u4e0b\u7684 core-site \u6587\u4ef6\u4e2d","title":"3. \u914d\u7f6e core-site"},{"location":"fluid-jindoruntime/case/jindo_fluid_sdk_connect_example/#4-shell","text":"\u53ef\u4ee5\u6267\u884c hadoop shell \u547d\u4ee4\u6765\u5224\u65ad\u670d\u52a1\u662f\u5426\u8fde\u63a5\u6210\u529f hadoop fs -ls oss://<Bucket>/<Dir>","title":"4. shell\u547d\u4ee4"},{"location":"fluid-jindoruntime/case/jindo_fluid_sdk_connect_example/#6","text":"JindoFSx \u53c2\u6570\u8bf4\u660e JindoFSx \u5305\u542b\u4e00\u4e9b\u7f13\u5b58\u76f8\u5173\u7684\u53c2\u6570\uff0c\u8bf7\u53c2\u8003\u6587\u6863 JindoFSx \u5ba2\u6237\u7aef\u914d\u7f6e\u5217\u8868 * JindoSDK \u53c2\u6570\u8bf4\u660e JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"6. \u53c2\u6570\u8c03\u4f18"},{"location":"fluid-jindoruntime/case/jindo_fluid_sdk_connect_example/#7","text":"\u81ea\u5efa\u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u963f\u91cc\u4e91 E-MapReduce \u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848","title":"7. \u6743\u9650\u7ba1\u7406"},{"location":"fluid-jindoruntime/case/jindo_fluid_update_version_example/","text":"Fluid JindoRuntime \u955c\u50cf\u5347\u7ea7 \u00a7 1\u3001\u4fee\u6539 jindoruntime-controller \u7684\u4fe1\u606f kubectl edit deployment jindoruntime-controller -n fluid-system 2\u3001\u7f16\u8f91 controller\uff0c\u627e\u5230\u5982\u4e0b\u5185\u5bb9 - name: JINDO_SMARTDATA_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:4.5.2 - name: JINDO_FUSE_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:4.5.2 3\u3001\u4f8b\u5982\u4ece 4.5.1 \u5347\u7ea7\u5230 4.5.2\uff0c\u4fee\u6539\u5982\u4e0b\u955c\u50cf\u5373\u53ef - name: JINDO_SMARTDATA_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:4.5.2 - name: JINDO_FUSE_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:4.5.2 \u5982\u679c\u662f fuseOnly \u6a21\u5f0f\uff0c\u90a3\u4e48\u53ea\u9700\u4fee\u6539 JINDO_FUSE_IMAGE_ENV \u5b57\u6bb5\u5373\u53ef 4\u3001\u4fdd\u5b58\u4fee\u6539\uff0c\u91cd\u65b0\u521b\u5efa\u96c6\u7fa4\u5373\u53ef","title":"Fluid JindoRuntime \u955c\u50cf\u5347\u7ea7"},{"location":"fluid-jindoruntime/case/jindo_fluid_update_version_example/#fluid-jindoruntime","text":"1\u3001\u4fee\u6539 jindoruntime-controller \u7684\u4fe1\u606f kubectl edit deployment jindoruntime-controller -n fluid-system 2\u3001\u7f16\u8f91 controller\uff0c\u627e\u5230\u5982\u4e0b\u5185\u5bb9 - name: JINDO_SMARTDATA_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:4.5.2 - name: JINDO_FUSE_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:4.5.2 3\u3001\u4f8b\u5982\u4ece 4.5.1 \u5347\u7ea7\u5230 4.5.2\uff0c\u4fee\u6539\u5982\u4e0b\u955c\u50cf\u5373\u53ef - name: JINDO_SMARTDATA_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:4.5.2 - name: JINDO_FUSE_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:4.5.2 \u5982\u679c\u662f fuseOnly \u6a21\u5f0f\uff0c\u90a3\u4e48\u53ea\u9700\u4fee\u6539 JINDO_FUSE_IMAGE_ENV \u5b57\u6bb5\u5373\u53ef 4\u3001\u4fdd\u5b58\u4fee\u6539\uff0c\u91cd\u65b0\u521b\u5efa\u96c6\u7fa4\u5373\u53ef","title":"Fluid JindoRuntime \u955c\u50cf\u5347\u7ea7"},{"location":"jindoauth/jindoauth_emr-next_kerberos/","text":"\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\uff08\u9ad8\u5b89\u5168 EMR \u96c6\u7fa4\uff09 \u00a7 \u672c\u6587\u4ecb\u7ecd\u5728 E-MapReduce \u65b0\u7248\u63a7\u5236\u53f0( EMR-5.15.0/EMR-3.49.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4)\u914d\u7f6e\u963f\u91cc\u4e91 OSS \u548c OSS-HDFS \u670d\u52a1\u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\u3002 \u80cc\u666f \u00a7 Apache Ranger \u63d0\u4f9b\u96c6\u4e2d\u5f0f\u7684\u6743\u9650\u7ba1\u7406\u6846\u67b6\uff0c\u53ef\u4ee5\u5bf9 Hadoop \u751f\u6001\u4e2d\u7684\u591a\u4e2a\u7ec4\u4ef6\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u8bbf\u95ee\u63a7\u5236\u3002\u5f53\u7528\u6237\u5c06\u6570\u636e\u5b58\u653e\u5728\u963f\u91cc\u4e91 OSS \u65f6\uff0c\u5219\u662f\u901a\u8fc7\u5728\u963f\u91cc\u4e91 RAM \u4ea7\u54c1\u521b\u5efa\u6216\u7ba1\u7406 RAM \u7528\u6237\uff0c\u5b9e\u73b0\u5bf9 OSS \u8d44\u6e90\u7684\u8bbf\u95ee\u63a7\u5236\u3002 \u4e3a\u7ef4\u6301\u5927\u6570\u636e\u5ba2\u6237\u7684\u4f7f\u7528\u4e60\u60ef\uff0c\u901a\u8fc7 JindoAuth \u63a5\u5165 Ranger \u7684\u5ba2\u6237\u7aef\uff0c\u65b9\u4fbf\u7528\u6237\u7edf\u4e00\u7ba1\u7406\u5927\u6570\u636e\u7ec4\u4ef6\u6743\u9650\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u521b\u5efa E-MapReduce EMR-5.15.0/EMR-3.49.0 \u6216\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\uff0c\u5e76\u4e14\u9009\u62e9\u4e86 Ranger \u548c Ranger-plugin \u670d\u52a1\u3002 \u542f\u7528 Kerberos \u8ba4\u8bc1\u6ee1\u8db3\u8ba4\u8bc1\u9700\u6c42\u3002 \u82e5\u4f7f\u7528\u96c6\u7fa4\u90fd\u662f\u53ef\u4fe1\u7528\u6237\uff0c\u5219\u53ef\u4e0d\u542f\u7528 Kerberos \u8ba4\u8bc1\u3002 \u6ce8\u610f\uff1aEMR \u96c6\u7fa4\u5f00\u542f Kerberos \u540e\uff0c\u5c06\u65e0\u6cd5\u5173\u95ed Kerberos \u8ba4\u8bc1\uff0c\u8bf7\u8c28\u614e\u9009\u62e9\u3002\u5f00\u542f Kerberos \u540e\uff0c\u96c6\u7fa4\u6240\u6709\u5927\u6570\u636e\u7ec4\u4ef6\u7684\u670d\u52a1\u5747\u9700\u8981\u7ecf\u8fc7 Kerberos \u8ba4\u8bc1\uff0c\u5927\u6570\u636e\u4f5c\u4e1a\u63d0\u4ea4\u5230\u96c6\u7fa4\u4f1a\u5148\u7ecf\u8fc7\u8eab\u4efd\u8ba4\u8bc1\u3002Kerberos \u8be6\u60c5\u4fe1\u606f\uff0c\u8bf7\u53c2\u89c1 \u9ad8\u5b89\u5168\u7c7b\u578b\u96c6\u7fa4 1. \u8fdb\u5165\u96c6\u7fa4\u670d\u52a1\u9875\u9762\u3002 \u00a7 a. \u767b\u5f55EMR on ECS\u63a7\u5236\u53f0\u3002 b. \u9876\u90e8\u83dc\u5355\u680f\u5904\uff0c\u6839\u636e\u5b9e\u9645\u60c5\u51b5\u9009\u62e9\u5730\u57df\u548c\u8d44\u6e90\u7ec4\u3002 c. \u5728\u96c6\u7fa4\u7ba1\u7406\u9875\u9762\uff0c\u5355\u51fb\u76ee\u6807\u96c6\u7fa4\u64cd\u4f5c\u5217\u7684\u96c6\u7fa4\u670d\u52a1\u3002 2. Ranger\u542f\u7528OSS\u3002 \u00a7 a. \u5728\u96c6\u7fa4\u670d\u52a1\u9875\u9762\uff0c\u5355\u51fbRanger-plugin\u670d\u52a1\u533a\u57df\u7684\u72b6\u6001\u3002 b. \u5728\u670d\u52a1\u6982\u8ff0\u533a\u57df\uff0c\u6253\u5f00enableOSS\u5f00\u5173\u3002 c. \u5728\u5f39\u51fa\u7684\u5bf9\u8bdd\u6846\u4e2d\uff0c\u5355\u51fb\u786e\u5b9a\u3002 3. \u90e8\u7f72\u5ba2\u6237\u7aef\u914d\u7f6e\u3002 \u00a7 a. \u5728\u96c6\u7fa4\u670d\u52a1\u9875\u9762\uff0c\u5355\u51fb HADOOP-COMMON \u670d\u52a1\u9875\u9762\u3002 b. \u5355\u51fb\u914d\u7f6e\u3002 c. \u5355\u51fb\u90e8\u7f72\u5ba2\u6237\u7aef\u914d\u7f6e\u3002 4. \u91cd\u542f HiveServer2 \u7b49\u5e38\u9a7b\u670d\u52a1\u3002 \u00a7 a. \u5728\u96c6\u7fa4\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9more > Hive\u3002 b. \u5728\u7ec4\u4ef6\u5217\u8868\u533a\u57df\uff0c\u5355\u51fbHiveServer\u64cd\u4f5c\u5217\u7684\u91cd\u542f\u3002 c. \u5728\u5f39\u51fa\u7684\u5bf9\u8bdd\u6846\u4e2d\uff0c\u8f93\u5165\u6267\u884c\u539f\u56e0\uff0c\u5355\u51fb\u786e\u5b9a\u3002 d. \u5728\u786e\u8ba4\u5bf9\u8bdd\u6846\u4e2d\uff0c\u5355\u51fb\u786e\u5b9a\u3002 5. \u6743\u9650\u914d\u7f6e\u793a\u4f8b\u3002 \u00a7 Ranger\u542f\u7528OSS\u540e\uff0c\u5df2\u9ed8\u8ba4\u6dfb\u52a0\u597d\u4e86OSS Service\u3002 \u4f8b\uff1a\u914d\u7f6e\u7528\u6237test\u62e5\u6709\u8bbf\u95ee oss://jindoauth-runjob-cn-shanghai/user/test \u76ee\u5f55\u7684\u6240\u4ee5\u6743\u9650\u7684\u6b65\u9aa4\uff1a \u914d\u7f6e test \u7528\u6237\u8bbf\u95ee oss://jindoauth-runjob-cn-shanghai/user/test \u76ee\u5f55\u7684\u8bbf\u95ee\u6743\u9650\u4e3a ALL\u3002 \u00a7 \u8bf4\u660e\uff1a \u00a7 \u89c4\u5219\u914d\u7f6e\u9875\u9762\u4e2d\uff0c\u914d\u7f6e\u7684 path \u6ca1\u6709 oss:// \u7684\u524d\u7f00\u3002 recursive\u6309\u94ae\u4e0d\u53ef\u5173\u95ed\u3002 \u8def\u5f84\u672b\u5c3e\u65e0\u9700\u5e26\u6b63\u659c\u7ebf\uff08/\uff09\u3002 6. \u8bbf\u95eeOSS\u3002 \u00a7 \u82e5\u7528\u6237\u8bbf\u95ee Ranger \u6ca1\u6709\u6388\u6743\u7684\u8def\u5f84\uff0c\u5219\u4f1a\u62a5\u5982\u4e0b\u9519\u8bef\uff1a org.apache.hadoop.security.AccessControlException: Permission denied: user=test, access=READ_EXECUTE, resourcePath=\"bucket-test-hangzhou/\"","title":"JindoAuth \u5feb\u901f\u5165\u95e8"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#oss-ranger-emr","text":"\u672c\u6587\u4ecb\u7ecd\u5728 E-MapReduce \u65b0\u7248\u63a7\u5236\u53f0( EMR-5.15.0/EMR-3.49.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4)\u914d\u7f6e\u963f\u91cc\u4e91 OSS \u548c OSS-HDFS \u670d\u52a1\u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\u3002","title":"\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\uff08\u9ad8\u5b89\u5168 EMR \u96c6\u7fa4\uff09"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#_1","text":"Apache Ranger \u63d0\u4f9b\u96c6\u4e2d\u5f0f\u7684\u6743\u9650\u7ba1\u7406\u6846\u67b6\uff0c\u53ef\u4ee5\u5bf9 Hadoop \u751f\u6001\u4e2d\u7684\u591a\u4e2a\u7ec4\u4ef6\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u8bbf\u95ee\u63a7\u5236\u3002\u5f53\u7528\u6237\u5c06\u6570\u636e\u5b58\u653e\u5728\u963f\u91cc\u4e91 OSS \u65f6\uff0c\u5219\u662f\u901a\u8fc7\u5728\u963f\u91cc\u4e91 RAM \u4ea7\u54c1\u521b\u5efa\u6216\u7ba1\u7406 RAM \u7528\u6237\uff0c\u5b9e\u73b0\u5bf9 OSS \u8d44\u6e90\u7684\u8bbf\u95ee\u63a7\u5236\u3002 \u4e3a\u7ef4\u6301\u5927\u6570\u636e\u5ba2\u6237\u7684\u4f7f\u7528\u4e60\u60ef\uff0c\u901a\u8fc7 JindoAuth \u63a5\u5165 Ranger \u7684\u5ba2\u6237\u7aef\uff0c\u65b9\u4fbf\u7528\u6237\u7edf\u4e00\u7ba1\u7406\u5927\u6570\u636e\u7ec4\u4ef6\u6743\u9650\u3002","title":"\u80cc\u666f"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#_2","text":"\u5df2\u521b\u5efa E-MapReduce EMR-5.15.0/EMR-3.49.0 \u6216\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\uff0c\u5e76\u4e14\u9009\u62e9\u4e86 Ranger \u548c Ranger-plugin \u670d\u52a1\u3002 \u542f\u7528 Kerberos \u8ba4\u8bc1\u6ee1\u8db3\u8ba4\u8bc1\u9700\u6c42\u3002 \u82e5\u4f7f\u7528\u96c6\u7fa4\u90fd\u662f\u53ef\u4fe1\u7528\u6237\uff0c\u5219\u53ef\u4e0d\u542f\u7528 Kerberos \u8ba4\u8bc1\u3002 \u6ce8\u610f\uff1aEMR \u96c6\u7fa4\u5f00\u542f Kerberos \u540e\uff0c\u5c06\u65e0\u6cd5\u5173\u95ed Kerberos \u8ba4\u8bc1\uff0c\u8bf7\u8c28\u614e\u9009\u62e9\u3002\u5f00\u542f Kerberos \u540e\uff0c\u96c6\u7fa4\u6240\u6709\u5927\u6570\u636e\u7ec4\u4ef6\u7684\u670d\u52a1\u5747\u9700\u8981\u7ecf\u8fc7 Kerberos \u8ba4\u8bc1\uff0c\u5927\u6570\u636e\u4f5c\u4e1a\u63d0\u4ea4\u5230\u96c6\u7fa4\u4f1a\u5148\u7ecf\u8fc7\u8eab\u4efd\u8ba4\u8bc1\u3002Kerberos \u8be6\u60c5\u4fe1\u606f\uff0c\u8bf7\u53c2\u89c1 \u9ad8\u5b89\u5168\u7c7b\u578b\u96c6\u7fa4","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#1","text":"a. \u767b\u5f55EMR on ECS\u63a7\u5236\u53f0\u3002 b. \u9876\u90e8\u83dc\u5355\u680f\u5904\uff0c\u6839\u636e\u5b9e\u9645\u60c5\u51b5\u9009\u62e9\u5730\u57df\u548c\u8d44\u6e90\u7ec4\u3002 c. \u5728\u96c6\u7fa4\u7ba1\u7406\u9875\u9762\uff0c\u5355\u51fb\u76ee\u6807\u96c6\u7fa4\u64cd\u4f5c\u5217\u7684\u96c6\u7fa4\u670d\u52a1\u3002","title":"1. \u8fdb\u5165\u96c6\u7fa4\u670d\u52a1\u9875\u9762\u3002"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#2-rangeross","text":"a. \u5728\u96c6\u7fa4\u670d\u52a1\u9875\u9762\uff0c\u5355\u51fbRanger-plugin\u670d\u52a1\u533a\u57df\u7684\u72b6\u6001\u3002 b. \u5728\u670d\u52a1\u6982\u8ff0\u533a\u57df\uff0c\u6253\u5f00enableOSS\u5f00\u5173\u3002 c. \u5728\u5f39\u51fa\u7684\u5bf9\u8bdd\u6846\u4e2d\uff0c\u5355\u51fb\u786e\u5b9a\u3002","title":"2. Ranger\u542f\u7528OSS\u3002"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#3","text":"a. \u5728\u96c6\u7fa4\u670d\u52a1\u9875\u9762\uff0c\u5355\u51fb HADOOP-COMMON \u670d\u52a1\u9875\u9762\u3002 b. \u5355\u51fb\u914d\u7f6e\u3002 c. \u5355\u51fb\u90e8\u7f72\u5ba2\u6237\u7aef\u914d\u7f6e\u3002","title":"3. \u90e8\u7f72\u5ba2\u6237\u7aef\u914d\u7f6e\u3002"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#4-hiveserver2","text":"a. \u5728\u96c6\u7fa4\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9more > Hive\u3002 b. \u5728\u7ec4\u4ef6\u5217\u8868\u533a\u57df\uff0c\u5355\u51fbHiveServer\u64cd\u4f5c\u5217\u7684\u91cd\u542f\u3002 c. \u5728\u5f39\u51fa\u7684\u5bf9\u8bdd\u6846\u4e2d\uff0c\u8f93\u5165\u6267\u884c\u539f\u56e0\uff0c\u5355\u51fb\u786e\u5b9a\u3002 d. \u5728\u786e\u8ba4\u5bf9\u8bdd\u6846\u4e2d\uff0c\u5355\u51fb\u786e\u5b9a\u3002","title":"4. \u91cd\u542f HiveServer2 \u7b49\u5e38\u9a7b\u670d\u52a1\u3002"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#5","text":"Ranger\u542f\u7528OSS\u540e\uff0c\u5df2\u9ed8\u8ba4\u6dfb\u52a0\u597d\u4e86OSS Service\u3002 \u4f8b\uff1a\u914d\u7f6e\u7528\u6237test\u62e5\u6709\u8bbf\u95ee oss://jindoauth-runjob-cn-shanghai/user/test \u76ee\u5f55\u7684\u6240\u4ee5\u6743\u9650\u7684\u6b65\u9aa4\uff1a","title":"5. \u6743\u9650\u914d\u7f6e\u793a\u4f8b\u3002"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#test-ossjindoauth-runjob-cn-shanghaiusertest-all","text":"","title":"\u914d\u7f6e test \u7528\u6237\u8bbf\u95eeoss://jindoauth-runjob-cn-shanghai/user/test\u76ee\u5f55\u7684\u8bbf\u95ee\u6743\u9650\u4e3a ALL\u3002"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#_3","text":"\u89c4\u5219\u914d\u7f6e\u9875\u9762\u4e2d\uff0c\u914d\u7f6e\u7684 path \u6ca1\u6709 oss:// \u7684\u524d\u7f00\u3002 recursive\u6309\u94ae\u4e0d\u53ef\u5173\u95ed\u3002 \u8def\u5f84\u672b\u5c3e\u65e0\u9700\u5e26\u6b63\u659c\u7ebf\uff08/\uff09\u3002","title":"\u8bf4\u660e\uff1a"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#6-oss","text":"\u82e5\u7528\u6237\u8bbf\u95ee Ranger \u6ca1\u6709\u6388\u6743\u7684\u8def\u5f84\uff0c\u5219\u4f1a\u62a5\u5982\u4e0b\u9519\u8bef\uff1a org.apache.hadoop.security.AccessControlException: Permission denied: user=test, access=READ_EXECUTE, resourcePath=\"bucket-test-hangzhou/\"","title":"6. \u8bbf\u95eeOSS\u3002"},{"location":"jindofs/jindofs_acls/","text":"\u80cc\u666f \u00a7 OSS-HDFS \u5b9e\u73b0\u4e86\u4e00\u4e2aPOSIX\u7cfb\u7edf\u7c7b\u4f3c\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u6743\u9650\u6a21\u578b\uff0c\u6bcf\u4e2a\u76ee\u5f55\u548c\u6587\u4ef6\u90fd\u5b58\u5728owner\u3001group\u3001others \u4e09\u7c7b\u6743\u9650\uff0c\u76ee\u5f55\u548c\u6587\u4ef6\u5bf9\u4e8e\u4e0d\u540c\u7684\u7528\u6237\u4ee5\u53ca\u7528\u6237\u7ec4\u6709\u7740\u4e0d\u540c\u7684\u6743\u9650\u3002\u6587\u4ef6\u6216\u76ee\u5f55\u7684\u6743\u9650\u5c31\u662f\u5b83\u7684\u6a21\u5f0f\uff08mode\uff09\u3002OSS-HDFS\u91c7\u7528\u4e86Unix\u8868\u793a\u548c\u663e\u793a\u6a21\u5f0f\u7684\u4e60\u60ef\uff0c\u5305\u62ec\u4f7f\u7528\u516b\u8fdb\u5236\u6570\u6765\u8868\u793a\u6743\u9650\u3002 \u5f53\u7528\u6237\u8bbf\u95eeOSS-HDFS\u7684\u6587\u4ef6\u6216\u8005\u76ee\u5f55\uff0cOSS-HDFS\u4f1a\u5bf9\u5176\u8fdb\u884c\u6743\u9650\u68c0\u67e5\uff0c\u68c0\u67e5\u6d41\u7a0b\u5982\u4e0b\uff1a - \u8bbf\u95ee\u7528\u6237\u4e0e\u6587\u4ef6/\u76ee\u5f55owner\u4e00\u81f4\uff0c\u68c0\u67e5owner\u7684\u6743\u9650 - \u8bbf\u95ee\u7528\u6237\u5173\u8054\u7684\u7ec4\u540d\u5728\u7ec4\u540d\u5217\u8868\u4e2d\u51fa\u73b0\uff0c\u68c0\u67e5group\u5bf9\u5e94\u8bbf\u95ee\u6743\u9650 - \u5426\u5219\u68c0\u67e5others \u5bf9\u5e94\u7684\u8bbf\u95ee\u6743\u9650 \u5982\u679c\u6743\u9650\u68c0\u67e5\u5931\u8d25\uff0c\u5219\u5ba2\u6237\u64cd\u4f5c\u4f1a\u5931\u8d25 \u5f00\u901a\u6743\u9650\u7ba1\u7406 \u00a7 \u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 \u6743\u9650\u7ba1\u7406\u7684\u5f00\u901a\u9700\u8981\u7528\u5230\u7684\u662f -putConfig \u4e0e -getConfig \u547d\u4ee4\uff0c\u5177\u4f53\u7528\u6cd5\u5c06\u5728\u4e0b\u6587\u4ecb\u7ecd\u3002 \u914d\u7f6e\u8bbf\u95ee AK \u00a7 \u6743\u9650\u7ba1\u7406\u9700\u8981\u914d\u7f6e\u8fde\u63a5 OSS-HDFS \u6240\u9700\u7684 AK \u4fe1\u606f\u3002\u6709\u4e24\u79cd\u65b9\u6cd5\u53ef\u4ee5\u8fdb\u884c\u914d\u7f6e\uff1a \u914d\u7f6e\u6587\u4ef6\uff08\u63a8\u8350\uff09 \u00a7 \u8bbe\u7f6e\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6307\u5411\u67d0\u4e2a\u76ee\u5f55\uff0c\u7136\u540e\u5728\u76ee\u5f55\u4e0b\u521b\u5efa\u540d\u4e3a jindofs.cfg \u7684\u6587\u4ef6\uff0c\u7136\u540e\u5728\u6587\u4ef6\u91cc\u5199\u5165\u5982\u4e0b\u5185\u5bb9\uff1a [client] fs.oss.accessKeyId = <key> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK key fs.oss.accessKeySecret = <secret> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK secret fs.oss.bucket.<bucket>.accessKeyId = <key> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK key\uff0c\u4f18\u5148\u7ea7\u9ad8\u4e8e\u9ed8\u8ba4 fs.oss.bucket.<bucket>.accessKeySecret = <secret> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK secret --extraConf \u00a7 \u547d\u4ee4\u884c\u5de5\u5177\u7684\u6240\u6709\u6307\u4ee4\u5747\u53ef\u901a\u8fc7 --extraConf \u9009\u9879\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u914d\u7f6e\u6587\u4ef6\uff0c\u4f8b\u5982\uff1a jindofs admin -getConfig -dlsUri <path> -name <keys> \\ --extraConf fs.oss.accessKeyId=<AK key> \\ --extraConf fs.oss.accessKeySecret=<AK secret> \u5c06\u7ed9\u547d\u4ee4\u63d0\u4f9b\u989d\u5916\u7684 AK key \u4e0e AK secret \u914d\u7f6e\u3002 \u6253\u5f00\u6743\u9650\u7ba1\u7406 \u00a7 \u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf namespace.permissions.enabled=true \u67e5\u770b\u6743\u9650\u7ba1\u7406 \u00a7 \u547d\u4ee4\uff1a jindofs admin -getConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -name namespace.permissions.enabled \u4e0a\u8ff0\u547d\u4ee4\u8fd4\u56de\u4e86\u540d\u4e3a \\<bucket>\uff08endpoint \u4e3a \\<oss-hdfs-endpoint>\uff09\u7684\u6876\u7684\u7ba1\u7406\u6743\u9650\u7684\u6807\u7b7e","title":"Jindofs acls"},{"location":"jindofs/jindofs_acls/#_1","text":"OSS-HDFS \u5b9e\u73b0\u4e86\u4e00\u4e2aPOSIX\u7cfb\u7edf\u7c7b\u4f3c\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u6743\u9650\u6a21\u578b\uff0c\u6bcf\u4e2a\u76ee\u5f55\u548c\u6587\u4ef6\u90fd\u5b58\u5728owner\u3001group\u3001others \u4e09\u7c7b\u6743\u9650\uff0c\u76ee\u5f55\u548c\u6587\u4ef6\u5bf9\u4e8e\u4e0d\u540c\u7684\u7528\u6237\u4ee5\u53ca\u7528\u6237\u7ec4\u6709\u7740\u4e0d\u540c\u7684\u6743\u9650\u3002\u6587\u4ef6\u6216\u76ee\u5f55\u7684\u6743\u9650\u5c31\u662f\u5b83\u7684\u6a21\u5f0f\uff08mode\uff09\u3002OSS-HDFS\u91c7\u7528\u4e86Unix\u8868\u793a\u548c\u663e\u793a\u6a21\u5f0f\u7684\u4e60\u60ef\uff0c\u5305\u62ec\u4f7f\u7528\u516b\u8fdb\u5236\u6570\u6765\u8868\u793a\u6743\u9650\u3002 \u5f53\u7528\u6237\u8bbf\u95eeOSS-HDFS\u7684\u6587\u4ef6\u6216\u8005\u76ee\u5f55\uff0cOSS-HDFS\u4f1a\u5bf9\u5176\u8fdb\u884c\u6743\u9650\u68c0\u67e5\uff0c\u68c0\u67e5\u6d41\u7a0b\u5982\u4e0b\uff1a - \u8bbf\u95ee\u7528\u6237\u4e0e\u6587\u4ef6/\u76ee\u5f55owner\u4e00\u81f4\uff0c\u68c0\u67e5owner\u7684\u6743\u9650 - \u8bbf\u95ee\u7528\u6237\u5173\u8054\u7684\u7ec4\u540d\u5728\u7ec4\u540d\u5217\u8868\u4e2d\u51fa\u73b0\uff0c\u68c0\u67e5group\u5bf9\u5e94\u8bbf\u95ee\u6743\u9650 - \u5426\u5219\u68c0\u67e5others \u5bf9\u5e94\u7684\u8bbf\u95ee\u6743\u9650 \u5982\u679c\u6743\u9650\u68c0\u67e5\u5931\u8d25\uff0c\u5219\u5ba2\u6237\u64cd\u4f5c\u4f1a\u5931\u8d25","title":"\u80cc\u666f"},{"location":"jindofs/jindofs_acls/#_2","text":"\u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 \u6743\u9650\u7ba1\u7406\u7684\u5f00\u901a\u9700\u8981\u7528\u5230\u7684\u662f -putConfig \u4e0e -getConfig \u547d\u4ee4\uff0c\u5177\u4f53\u7528\u6cd5\u5c06\u5728\u4e0b\u6587\u4ecb\u7ecd\u3002","title":"\u5f00\u901a\u6743\u9650\u7ba1\u7406"},{"location":"jindofs/jindofs_acls/#ak","text":"\u6743\u9650\u7ba1\u7406\u9700\u8981\u914d\u7f6e\u8fde\u63a5 OSS-HDFS \u6240\u9700\u7684 AK \u4fe1\u606f\u3002\u6709\u4e24\u79cd\u65b9\u6cd5\u53ef\u4ee5\u8fdb\u884c\u914d\u7f6e\uff1a","title":"\u914d\u7f6e\u8bbf\u95ee AK"},{"location":"jindofs/jindofs_acls/#_3","text":"\u8bbe\u7f6e\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6307\u5411\u67d0\u4e2a\u76ee\u5f55\uff0c\u7136\u540e\u5728\u76ee\u5f55\u4e0b\u521b\u5efa\u540d\u4e3a jindofs.cfg \u7684\u6587\u4ef6\uff0c\u7136\u540e\u5728\u6587\u4ef6\u91cc\u5199\u5165\u5982\u4e0b\u5185\u5bb9\uff1a [client] fs.oss.accessKeyId = <key> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK key fs.oss.accessKeySecret = <secret> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK secret fs.oss.bucket.<bucket>.accessKeyId = <key> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK key\uff0c\u4f18\u5148\u7ea7\u9ad8\u4e8e\u9ed8\u8ba4 fs.oss.bucket.<bucket>.accessKeySecret = <secret> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK secret","title":"\u914d\u7f6e\u6587\u4ef6\uff08\u63a8\u8350\uff09"},{"location":"jindofs/jindofs_acls/#-extraconf","text":"\u547d\u4ee4\u884c\u5de5\u5177\u7684\u6240\u6709\u6307\u4ee4\u5747\u53ef\u901a\u8fc7 --extraConf \u9009\u9879\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u914d\u7f6e\u6587\u4ef6\uff0c\u4f8b\u5982\uff1a jindofs admin -getConfig -dlsUri <path> -name <keys> \\ --extraConf fs.oss.accessKeyId=<AK key> \\ --extraConf fs.oss.accessKeySecret=<AK secret> \u5c06\u7ed9\u547d\u4ee4\u63d0\u4f9b\u989d\u5916\u7684 AK key \u4e0e AK secret \u914d\u7f6e\u3002","title":"--extraConf"},{"location":"jindofs/jindofs_acls/#_4","text":"\u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf namespace.permissions.enabled=true","title":"\u6253\u5f00\u6743\u9650\u7ba1\u7406"},{"location":"jindofs/jindofs_acls/#_5","text":"\u547d\u4ee4\uff1a jindofs admin -getConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -name namespace.permissions.enabled \u4e0a\u8ff0\u547d\u4ee4\u8fd4\u56de\u4e86\u540d\u4e3a \\<bucket>\uff08endpoint \u4e3a \\<oss-hdfs-endpoint>\uff09\u7684\u6876\u7684\u7ba1\u7406\u6743\u9650\u7684\u6807\u7b7e","title":"\u67e5\u770b\u6743\u9650\u7ba1\u7406"},{"location":"jindofs/jindofs_archive_direct_read/","text":"\u80cc\u666f \u00a7 OSS-HDFS \u5f52\u6863\u76f4\u8bfb\u5f53\u524d\u53ef\u4ee5\u901a\u8fc7\u547d\u4ee4\u884c\u547d\u4ee4\u7684\u65b9\u5f0f\u8fdb\u884c\u64cd\u4f5c\u3002\u5c06\u6765\u8fd8\u4f1a\u901a\u8fc7\u4ea7\u54c1\u754c\u9762\u7684\u65b9\u5f0f\uff0c\u63d0\u4f9b\u66f4\u52a0\u4fbf\u6377\u6613\u7528\u7684\u67e5\u770b\u548c\u8bbe\u7f6e\u65b9\u6cd5\u3002 \u5f52\u6863\u76f4\u8bfb\u529f\u80fd\u63cf\u8ff0 \u00a7 \u7528\u6237\u9700\u8981\u8bfb\u53d6\u5f52\u6863\u7c7b\u578b\u7684\u6587\u4ef6\u65f6\u5019\uff0c\u901a\u5e38\u9700\u8981\u901a\u8fc7\u89e3\u51bb\u64cd\u4f5c\u5c06\u5f52\u6863\u6587\u4ef6\u89e3\u51bb\u5230\u53ef\u8bfb\u53d6\u7684\u72b6\u6001\uff0c\u8fd9\u79cd\u89e3\u51bb\u64cd\u4f5c\u53ef\u80fd\u6bd4\u8f83\u8017\u65f6\uff0cOSS-HDFS \u5f52\u6863\u76f4\u8bfb\u529f\u80fd\u53ef\u4ee5\u907f\u514d\u89e3\u51bb\u64cd\u4f5c\uff0c\u76f4\u63a5\u8bfb\u53d6\u5f52\u6863\u6587\u4ef6 JindoFS \u547d\u4ee4\u884c\u7b80\u4ecb \u00a7 \u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 \u5f52\u6863\u76f4\u8bfb\u529f\u80fd\u9700\u8981\u7528\u5230\u7684\u662f -putConfig \u4e0e -getConfig \u547d\u4ee4\uff0c\u5177\u4f53\u7528\u6cd5\u5c06\u5728\u4e0b\u6587\u4ecb\u7ecd\u3002 \u914d\u7f6e\u8bbf\u95ee AK \u00a7 \u5f52\u6863\u76f4\u8bfb\u529f\u80fd\u9700\u8981\u914d\u7f6e\u8fde\u63a5 OSS-HDFS \u6240\u9700\u7684 AK \u4fe1\u606f\u3002\u6709\u4e24\u79cd\u65b9\u6cd5\u53ef\u4ee5\u8fdb\u884c\u914d\u7f6e\uff1a \u914d\u7f6e\u6587\u4ef6\uff08\u63a8\u8350\uff09 \u00a7 \u8bbe\u7f6e\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6307\u5411\u67d0\u4e2a\u76ee\u5f55\uff0c\u7136\u540e\u5728\u76ee\u5f55\u4e0b\u521b\u5efa\u540d\u4e3a jindofs.cfg \u7684\u6587\u4ef6\uff0c\u7136\u540e\u5728\u6587\u4ef6\u91cc\u5199\u5165\u5982\u4e0b\u5185\u5bb9\uff1a [client] fs.oss.accessKeyId = <key> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK key fs.oss.accessKeySecret = <secret> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK secret fs.oss.bucket.<bucket>.accessKeyId = <key> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK key\uff0c\u4f18\u5148\u7ea7\u9ad8\u4e8e\u9ed8\u8ba4 fs.oss.bucket.<bucket>.accessKeySecret = <secret> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK secret --extraConf \u00a7 \u547d\u4ee4\u884c\u5de5\u5177\u7684\u6240\u6709\u6307\u4ee4\u5747\u53ef\u901a\u8fc7 --extraConf \u9009\u9879\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u914d\u7f6e\u6587\u4ef6\uff0c\u4f8b\u5982\uff1a jindofs admin -getConfig -dlsUri <path> -name <keys> \\ --extraConf fs.oss.accessKeyId=<AK key> \\ --extraConf fs.oss.accessKeySecret=<AK secret> \u5c06\u7ed9\u547d\u4ee4\u63d0\u4f9b\u989d\u5916\u7684 AK key \u4e0e AK secret \u914d\u7f6e\u3002 \u8bbe\u7f6e\u5f52\u6863\u76f4\u8bfb \u00a7 \u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf namespace.archive.directread.enable=true \u67e5\u770b\u5f52\u6863\u76f4\u8bfb \u00a7 \u547d\u4ee4\uff1a jindofs admin -getConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -name namespace.archive.directread.enable \u4e0a\u8ff0\u547d\u4ee4\u8fd4\u56de\u4e86\u540d\u4e3a \\<bucket>\uff08endpoint \u4e3a \\<oss-hdfs-endpoint>\uff09\u7684\u6876\u7684\u5f52\u6863\u76f4\u8bfb\u6807\u7b7e","title":"Jindofs archive direct read"},{"location":"jindofs/jindofs_archive_direct_read/#_1","text":"OSS-HDFS \u5f52\u6863\u76f4\u8bfb\u5f53\u524d\u53ef\u4ee5\u901a\u8fc7\u547d\u4ee4\u884c\u547d\u4ee4\u7684\u65b9\u5f0f\u8fdb\u884c\u64cd\u4f5c\u3002\u5c06\u6765\u8fd8\u4f1a\u901a\u8fc7\u4ea7\u54c1\u754c\u9762\u7684\u65b9\u5f0f\uff0c\u63d0\u4f9b\u66f4\u52a0\u4fbf\u6377\u6613\u7528\u7684\u67e5\u770b\u548c\u8bbe\u7f6e\u65b9\u6cd5\u3002","title":"\u80cc\u666f"},{"location":"jindofs/jindofs_archive_direct_read/#_2","text":"\u7528\u6237\u9700\u8981\u8bfb\u53d6\u5f52\u6863\u7c7b\u578b\u7684\u6587\u4ef6\u65f6\u5019\uff0c\u901a\u5e38\u9700\u8981\u901a\u8fc7\u89e3\u51bb\u64cd\u4f5c\u5c06\u5f52\u6863\u6587\u4ef6\u89e3\u51bb\u5230\u53ef\u8bfb\u53d6\u7684\u72b6\u6001\uff0c\u8fd9\u79cd\u89e3\u51bb\u64cd\u4f5c\u53ef\u80fd\u6bd4\u8f83\u8017\u65f6\uff0cOSS-HDFS \u5f52\u6863\u76f4\u8bfb\u529f\u80fd\u53ef\u4ee5\u907f\u514d\u89e3\u51bb\u64cd\u4f5c\uff0c\u76f4\u63a5\u8bfb\u53d6\u5f52\u6863\u6587\u4ef6","title":"\u5f52\u6863\u76f4\u8bfb\u529f\u80fd\u63cf\u8ff0"},{"location":"jindofs/jindofs_archive_direct_read/#jindofs","text":"\u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 \u5f52\u6863\u76f4\u8bfb\u529f\u80fd\u9700\u8981\u7528\u5230\u7684\u662f -putConfig \u4e0e -getConfig \u547d\u4ee4\uff0c\u5177\u4f53\u7528\u6cd5\u5c06\u5728\u4e0b\u6587\u4ecb\u7ecd\u3002","title":"JindoFS \u547d\u4ee4\u884c\u7b80\u4ecb"},{"location":"jindofs/jindofs_archive_direct_read/#ak","text":"\u5f52\u6863\u76f4\u8bfb\u529f\u80fd\u9700\u8981\u914d\u7f6e\u8fde\u63a5 OSS-HDFS \u6240\u9700\u7684 AK \u4fe1\u606f\u3002\u6709\u4e24\u79cd\u65b9\u6cd5\u53ef\u4ee5\u8fdb\u884c\u914d\u7f6e\uff1a","title":"\u914d\u7f6e\u8bbf\u95ee AK"},{"location":"jindofs/jindofs_archive_direct_read/#_3","text":"\u8bbe\u7f6e\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6307\u5411\u67d0\u4e2a\u76ee\u5f55\uff0c\u7136\u540e\u5728\u76ee\u5f55\u4e0b\u521b\u5efa\u540d\u4e3a jindofs.cfg \u7684\u6587\u4ef6\uff0c\u7136\u540e\u5728\u6587\u4ef6\u91cc\u5199\u5165\u5982\u4e0b\u5185\u5bb9\uff1a [client] fs.oss.accessKeyId = <key> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK key fs.oss.accessKeySecret = <secret> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK secret fs.oss.bucket.<bucket>.accessKeyId = <key> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK key\uff0c\u4f18\u5148\u7ea7\u9ad8\u4e8e\u9ed8\u8ba4 fs.oss.bucket.<bucket>.accessKeySecret = <secret> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK secret","title":"\u914d\u7f6e\u6587\u4ef6\uff08\u63a8\u8350\uff09"},{"location":"jindofs/jindofs_archive_direct_read/#-extraconf","text":"\u547d\u4ee4\u884c\u5de5\u5177\u7684\u6240\u6709\u6307\u4ee4\u5747\u53ef\u901a\u8fc7 --extraConf \u9009\u9879\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u914d\u7f6e\u6587\u4ef6\uff0c\u4f8b\u5982\uff1a jindofs admin -getConfig -dlsUri <path> -name <keys> \\ --extraConf fs.oss.accessKeyId=<AK key> \\ --extraConf fs.oss.accessKeySecret=<AK secret> \u5c06\u7ed9\u547d\u4ee4\u63d0\u4f9b\u989d\u5916\u7684 AK key \u4e0e AK secret \u914d\u7f6e\u3002","title":"--extraConf"},{"location":"jindofs/jindofs_archive_direct_read/#_4","text":"\u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf namespace.archive.directread.enable=true","title":"\u8bbe\u7f6e\u5f52\u6863\u76f4\u8bfb"},{"location":"jindofs/jindofs_archive_direct_read/#_5","text":"\u547d\u4ee4\uff1a jindofs admin -getConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -name namespace.archive.directread.enable \u4e0a\u8ff0\u547d\u4ee4\u8fd4\u56de\u4e86\u540d\u4e3a \\<bucket>\uff08endpoint \u4e3a \\<oss-hdfs-endpoint>\uff09\u7684\u6876\u7684\u5f52\u6863\u76f4\u8bfb\u6807\u7b7e","title":"\u67e5\u770b\u5f52\u6863\u76f4\u8bfb"},{"location":"jindofs/jindofs_callerContext/","text":"\u5f00\u901aCallerContext \u00a7 \u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 CallerContext\u7684\u5f00\u901a\u9700\u8981\u7528\u5230\u7684\u662f -putConfig \u4e0e -getConfig \u547d\u4ee4\uff0c\u5177\u4f53\u7528\u6cd5\u5c06\u5728\u4e0b\u6587\u4ecb\u7ecd\u3002 \u914d\u7f6e\u8bbf\u95ee AK \u00a7 CallerContext\u9700\u8981\u914d\u7f6e\u8fde\u63a5 OSS-HDFS \u6240\u9700\u7684 AK \u4fe1\u606f\u3002\u6709\u4e24\u79cd\u65b9\u6cd5\u53ef\u4ee5\u8fdb\u884c\u914d\u7f6e\uff1a \u914d\u7f6e\u6587\u4ef6\uff08\u63a8\u8350\uff09 \u00a7 \u8bbe\u7f6e\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6307\u5411\u67d0\u4e2a\u76ee\u5f55\uff0c\u7136\u540e\u5728\u76ee\u5f55\u4e0b\u521b\u5efa\u540d\u4e3a jindofs.cfg \u7684\u6587\u4ef6\uff0c\u7136\u540e\u5728\u6587\u4ef6\u91cc\u5199\u5165\u5982\u4e0b\u5185\u5bb9\uff1a [client] fs.oss.accessKeyId = <key> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK key fs.oss.accessKeySecret = <secret> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK secret fs.oss.bucket.<bucket>.accessKeyId = <key> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK key\uff0c\u4f18\u5148\u7ea7\u9ad8\u4e8e\u9ed8\u8ba4 fs.oss.bucket.<bucket>.accessKeySecret = <secret> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK secret --extraConf \u00a7 \u547d\u4ee4\u884c\u5de5\u5177\u7684\u6240\u6709\u6307\u4ee4\u5747\u53ef\u901a\u8fc7 --extraConf \u9009\u9879\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u914d\u7f6e\u6587\u4ef6\uff0c\u4f8b\u5982\uff1a jindofs admin -getConfig -dlsUri <path> -name <keys> \\ --extraConf fs.oss.accessKeyId=<AK key> \\ --extraConf fs.oss.accessKeySecret=<AK secret> \u5c06\u7ed9\u547d\u4ee4\u63d0\u4f9b\u989d\u5916\u7684 AK key \u4e0e AK secret \u914d\u7f6e\u3002 \u6253\u5f00CallerContext\u7ba1\u7406 \u00a7 \u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf namespace.caller.context.enabled=true \u67e5\u770bCallerContext\u7ba1\u7406 \u00a7 \u547d\u4ee4\uff1a jindofs admin -getConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -name namespace.caller.context.enabled \u4e0a\u8ff0\u547d\u4ee4\u8fd4\u56de\u4e86\u540d\u4e3a \\<bucket>\uff08endpoint \u4e3a \\<oss-hdfs-endpoint>\uff09\u7684\u6876\u7684CallerContext\u7684\u6807\u7b7e","title":"jindofs callerContext"},{"location":"jindofs/jindofs_callerContext/#callercontext","text":"\u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 CallerContext\u7684\u5f00\u901a\u9700\u8981\u7528\u5230\u7684\u662f -putConfig \u4e0e -getConfig \u547d\u4ee4\uff0c\u5177\u4f53\u7528\u6cd5\u5c06\u5728\u4e0b\u6587\u4ecb\u7ecd\u3002","title":"\u5f00\u901aCallerContext"},{"location":"jindofs/jindofs_callerContext/#ak","text":"CallerContext\u9700\u8981\u914d\u7f6e\u8fde\u63a5 OSS-HDFS \u6240\u9700\u7684 AK \u4fe1\u606f\u3002\u6709\u4e24\u79cd\u65b9\u6cd5\u53ef\u4ee5\u8fdb\u884c\u914d\u7f6e\uff1a","title":"\u914d\u7f6e\u8bbf\u95ee AK"},{"location":"jindofs/jindofs_callerContext/#_1","text":"\u8bbe\u7f6e\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6307\u5411\u67d0\u4e2a\u76ee\u5f55\uff0c\u7136\u540e\u5728\u76ee\u5f55\u4e0b\u521b\u5efa\u540d\u4e3a jindofs.cfg \u7684\u6587\u4ef6\uff0c\u7136\u540e\u5728\u6587\u4ef6\u91cc\u5199\u5165\u5982\u4e0b\u5185\u5bb9\uff1a [client] fs.oss.accessKeyId = <key> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK key fs.oss.accessKeySecret = <secret> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK secret fs.oss.bucket.<bucket>.accessKeyId = <key> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK key\uff0c\u4f18\u5148\u7ea7\u9ad8\u4e8e\u9ed8\u8ba4 fs.oss.bucket.<bucket>.accessKeySecret = <secret> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK secret","title":"\u914d\u7f6e\u6587\u4ef6\uff08\u63a8\u8350\uff09"},{"location":"jindofs/jindofs_callerContext/#-extraconf","text":"\u547d\u4ee4\u884c\u5de5\u5177\u7684\u6240\u6709\u6307\u4ee4\u5747\u53ef\u901a\u8fc7 --extraConf \u9009\u9879\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u914d\u7f6e\u6587\u4ef6\uff0c\u4f8b\u5982\uff1a jindofs admin -getConfig -dlsUri <path> -name <keys> \\ --extraConf fs.oss.accessKeyId=<AK key> \\ --extraConf fs.oss.accessKeySecret=<AK secret> \u5c06\u7ed9\u547d\u4ee4\u63d0\u4f9b\u989d\u5916\u7684 AK key \u4e0e AK secret \u914d\u7f6e\u3002","title":"--extraConf"},{"location":"jindofs/jindofs_callerContext/#callercontext_1","text":"\u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf namespace.caller.context.enabled=true","title":"\u6253\u5f00CallerContext\u7ba1\u7406"},{"location":"jindofs/jindofs_callerContext/#callercontext_2","text":"\u547d\u4ee4\uff1a jindofs admin -getConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -name namespace.caller.context.enabled \u4e0a\u8ff0\u547d\u4ee4\u8fd4\u56de\u4e86\u540d\u4e3a \\<bucket>\uff08endpoint \u4e3a \\<oss-hdfs-endpoint>\uff09\u7684\u6876\u7684CallerContext\u7684\u6807\u7b7e","title":"\u67e5\u770bCallerContext\u7ba1\u7406"},{"location":"jindofs/jindofs_client_tools/","text":"JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 \u00a7 \u4ecb\u7ecd \u00a7 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u662f\u8bbf\u95ee OSS-HDFS \u7684\u53ef\u6267\u884c\u7a0b\u5e8f\uff0c\u529f\u80fd\u7c7b\u4f3c\u4e8e\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u7684 ossutil\u3002\u65e2\u652f\u6301\u5e38\u7528\u7684\u6587\u4ef6\u5143\u6570\u636e\u548c\u8bfb\u5199\u6d41\u64cd\u4f5c\uff0c\u5982\u521b\u5efa\u76ee\u5f55\u3001\u62f7\u8d1d\u6587\u4ef6\uff1b\u4e5f\u652f\u6301 OSS-HDFS \u7279\u6709\u7684\u65b9\u6cd5\uff0c\u5982\u6e05\u5355\u5bfc\u51fa\u7b49\u3002\u91c7\u7528\u4e86 Native \u5b9e\u73b0\uff0c\u76ee\u6807\u662f\u5bf9 OSS-HDFS \u63d0\u4f9b\u9ad8\u6548\u6613\u7528\u7684\u547d\u4ee4\u884c\u652f\u6301\u3002 \u4e0e JindoSDK \u63d0\u4f9b\u7684\u547d\u4ee4\u884c\u5de5\u5177\u76f8\u6bd4\uff0cJindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4e0d\u4ec5\u63d0\u4f9b\u5bf9\u6807 Hdfs shell \u7684\u5e38\u7528\u547d\u4ee4\uff0c\u8fd8\u63d0\u4f9b OSS-HDFS \u7684\u7279\u6709\u65b9\u6cd5\u3002\u7136\u800c\uff0cJindoFS \u547d\u4ee4\u884c\u5de5\u5177\u53ea\u652f\u6301 OSS-HDFS\uff0c\u4e3b\u8981\u4f5c\u4e3a OSS-HDFS \u7684\u7279\u6709\u5de5\u5177\u4f7f\u7528\u3002JindoSDK \u7684\u547d\u4ee4\u884c\u5de5\u5177\u53c2\u89c1 \u300aJindo CLI \u4f7f\u7528\u6307\u5357\u300b \u83b7\u53d6 \u00a7 \u76ee\u524d\u53ea\u63d0\u4f9b\u4e86 Linux \u7248\u672c\u7684\u547d\u4ee4\u884c\u5de5\u5177\uff0c \u4e0b\u8f7d\u94fe\u63a5 \u4e0b\u8f7d\u540e\u89e3\u538b\u7f29\uff0c\u5728 bin/ \u76ee\u5f55\u4e0b\u5373\u53ef\u627e\u5230\u547d\u4ee4\u884c\u5de5\u5177\u7684\u4e8c\u8fdb\u5236\u6587\u4ef6\uff0c\u6587\u4ef6\u540d\u4e3a jindofs \u3002 \u914d\u7f6e \u00a7 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528 JINDOSDK_CONF_DIR \u4e3a\u73af\u5883\u53d8\u91cf\u3002\u4f7f\u8be5\u53d8\u91cf\u6307\u5411\u4e00\u4e2a\u672c\u5730\u76ee\u5f55\uff0c\u5728\u76ee\u5f55\u4e0b\u653e\u7f6e\u914d\u7f6e\u6587\u4ef6 jindofs.cfg \uff0c\u4fbf\u53ef\u4ee5\u4e3a\u547d\u4ee4\u884c\u5de5\u5177\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\u3002 jindofs.cfg \u7684\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a [client] fs.oss.accessKeyId = <access key> fs.oss.accessKeySecret = <access secret> fs.oss.endpoint = <OSS-HDFS endpoint> fs.oss.data.lake.storage.enable = true # fs.oss.provider.endpoint = ECS_ROLE # EMR Credential provider for password-free access # fs.oss.provider.format = JSON \u5728\u89e3\u538b\u540e\u7684 conf/ \u76ee\u5f55\u4e0b\u4e5f\u53ef\u4ee5\u627e\u5230\u914d\u7f6e\u793a\u4f8b\u6587\u4ef6 jindofs.cfg.template \u3002 \u7528\u6cd5 \u00a7 \u5728\u547d\u4ee4\u884c\u4e2d\u6267\u884c jindofs \u4e8c\u8fdb\u5236\u6587\u4ef6\uff08\u4f8b\u5982\uff0c cd \u81f3\u6240\u5728\u76ee\u5f55\uff0c\u6267\u884c ./jindofs \u6216\u8005 ./jindofs -help \uff09\uff0c\u5c06\u663e\u793a\u6240\u6709\u652f\u6301\u7684\u547d\u4ee4\uff1a jindofs [-help] [-version] jindofs admin [-help] [-addProxyUser -dlsUri <path> -proxyUser <value> -users|-groups <value> -hosts <value>] [-allowSnapshot -dlsUri <path>] [-addUserGroupsMapping -dlsUri <path> -user <value> -groups <value>] [-disallowSnapshot -dlsUri <path>] [-deleteProxyUser -dlsUri <path> -proxyUser <value>] [-deleteUserGroupsMapping -dlsUri <path> -user <value>] [-dumpMetaInfo -dlsUri <path>] [-dumpInventory <path>] [-getJobProgress <jobId> -dlsUri <path>] [-listJobs -dlsUri <path> [-stages <stages>] [-createTime <createTime>] [-type <type>]] [-listProxyUsers -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-listUserGroupsMappings -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-snapshotDiff -dlsUri <path> -fromSnapshot <value> -toSnapshot <value>] [-setRootPolicy <dlsRootPath> <accessRootPath>] [-unsetRootPolicy <dlsRootPath> <accessRootPath>] [-listAccessPolicies <dlsRootPath>] [-putConfig -dlsUri <path> -conf <key1=value1> -conf <key2=value2> ...] [-getConfig -dlsUri <path> -name <keys>] jindofs fs [-help] [-cat <path>] [-count [-h] <path>] [-cp [-f] <src> <dst>] [-checksum [-mode COMPOSITE_CRC(default)/MD5MD5CRC] [-blockSize <size in bytes (only used in MD5MD5CRC mode, 128MB by default)>] <src> ...] [-chgrp [-R] <group> <path>] [-chmod [-R] <mode> <path>] [-chown [-R] [owner][:[group]] <path>] [-copyFromLocal [-f] <localsrc> <dst>] [-copyToLocal [-f] <src> <localdst>] [-createSnapshot <snapshotDir> [<snapshotName>]] [-checkStoragePolicy -path <path>] [-deleteSnapshot <snapshotDir> <snapshotName>] [-du [-s] [-h] <path> ...] [-expunge [-immediate] <path>] [-get [-f] <src> <localdst>] [-getfacl [-R] <path>] [-getfattr [-R] {-n name | -d} <path>] [-getStoragePolicy -path <path>] [-ls [-R] <path>] [-listPolicies] [-mkdir [-p] <path>] [-mv <src> ... <dst>] [-moveFromLocal <localsrc> ... <dst>] [-moveToLocal <src> <localdst>] [-put [-f] <localsrc> <dst>] [-rm [-skipTrash] [-f] [-r|-R] [-safely] <path>] [-rmdir <path>] [-test -[defsz] <path>] [-renameSnapshot <snapshotDir> <oldName> <newName>] [-stat [format] <path>] [-setStoragePolicy -path <path> -policy <policy> [-restoreDays <restoreDays>]] [-setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>]] [-setfattr {-n name [-v value] | -x name} <path>] [-touchz <path>] [-truncate [-w] <length> <path>] [-unsetStoragePolicy -path <path>] \u53e6\u5916\uff0c\u6240\u6709\u547d\u4ee4\u5747\u652f\u6301 --extraConf key=value \u7684\u65b9\u5f0f\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u4ec5\u5bf9\u8be5\u6b21\u6267\u884c\u6709\u6548\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u5199\u5728\u914d\u7f6e\u6587\u4ef6\u4e2d\u3002 \u793a\u4f8b \u00a7 \u4e0a\u4f20\u6587\u4ef6 \u00a7 ./jindofs fs -put /local/file/path oss://<bucket-name>.<oss-hdfs-endpoint>/remote/file/path \u4f5c\u7528\u662f\u628a\u672c\u5730\u6587\u4ef6 /local/file/path \u4e0a\u4f20\u5230\u8fdc\u7aef\u8def\u5f84 oss://<bucket-name>.<oss-hdfs-endpoint>/remote/file/path \u3002 \u6e05\u5355\u5bfc\u51fa \u00a7 ./jindofs admin -dumpInventory oss://<bucket-name>.<oss-hdfs-endpoint>/ --extraConf fs.oss.accessKeyId=<access key> --extraConf fs.oss.accessKeySecret=<access secret> \u4e0a\u8ff0\u547d\u4ee4\u5411 OSS-HDFS \u670d\u52a1\u53d1\u8d77\u4e86\u4e00\u6b21\u6e05\u5355\u5bfc\u51fa\u4efb\u52a1\uff0c\u5e76\u4e14\u8fd8\u901a\u8fc7 --extraConf key=value \u7684\u65b9\u5f0f\u4e3a\u672c\u6b21\u6267\u884c\u63d0\u4f9b\u4e86\u989d\u5916\u7684\u4e24\u6761\u914d\u7f6e\u4fe1\u606f\u3002JindoFS \u7684\u6240\u6709\u547d\u4ee4\u5747\u652f\u6301 --extraConf \u53c2\u6570\u3002 \u5982\u679c\u547d\u4ee4\u6210\u529f\u6267\u884c\uff0c\u5c06\u8fd4\u56de\uff1a =============Dump Inventory============= Job Id: <job id> Data Location: <oss-hdfs path to inventory file> ....................... FINISHED. \u968f\u540e\u53ef\u5728 Data Location \u6307\u5411\u7684 OSS-HDFS \u8def\u5f84\u627e\u5230\u6e05\u5355\u5bfc\u51fa\u7684\u7ed3\u679c\u6587\u4ef6\uff0c\u6309\u666e\u901a\u6587\u4ef6\u8bbf\u95ee\u5373\u53ef\u3002","title":"JindoFS Cli \u5feb\u901f\u5165\u95e8"},{"location":"jindofs/jindofs_client_tools/#jindofs","text":"","title":"JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357"},{"location":"jindofs/jindofs_client_tools/#_1","text":"JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u662f\u8bbf\u95ee OSS-HDFS \u7684\u53ef\u6267\u884c\u7a0b\u5e8f\uff0c\u529f\u80fd\u7c7b\u4f3c\u4e8e\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u7684 ossutil\u3002\u65e2\u652f\u6301\u5e38\u7528\u7684\u6587\u4ef6\u5143\u6570\u636e\u548c\u8bfb\u5199\u6d41\u64cd\u4f5c\uff0c\u5982\u521b\u5efa\u76ee\u5f55\u3001\u62f7\u8d1d\u6587\u4ef6\uff1b\u4e5f\u652f\u6301 OSS-HDFS \u7279\u6709\u7684\u65b9\u6cd5\uff0c\u5982\u6e05\u5355\u5bfc\u51fa\u7b49\u3002\u91c7\u7528\u4e86 Native \u5b9e\u73b0\uff0c\u76ee\u6807\u662f\u5bf9 OSS-HDFS \u63d0\u4f9b\u9ad8\u6548\u6613\u7528\u7684\u547d\u4ee4\u884c\u652f\u6301\u3002 \u4e0e JindoSDK \u63d0\u4f9b\u7684\u547d\u4ee4\u884c\u5de5\u5177\u76f8\u6bd4\uff0cJindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4e0d\u4ec5\u63d0\u4f9b\u5bf9\u6807 Hdfs shell \u7684\u5e38\u7528\u547d\u4ee4\uff0c\u8fd8\u63d0\u4f9b OSS-HDFS \u7684\u7279\u6709\u65b9\u6cd5\u3002\u7136\u800c\uff0cJindoFS \u547d\u4ee4\u884c\u5de5\u5177\u53ea\u652f\u6301 OSS-HDFS\uff0c\u4e3b\u8981\u4f5c\u4e3a OSS-HDFS \u7684\u7279\u6709\u5de5\u5177\u4f7f\u7528\u3002JindoSDK \u7684\u547d\u4ee4\u884c\u5de5\u5177\u53c2\u89c1 \u300aJindo CLI \u4f7f\u7528\u6307\u5357\u300b","title":"\u4ecb\u7ecd"},{"location":"jindofs/jindofs_client_tools/#_2","text":"\u76ee\u524d\u53ea\u63d0\u4f9b\u4e86 Linux \u7248\u672c\u7684\u547d\u4ee4\u884c\u5de5\u5177\uff0c \u4e0b\u8f7d\u94fe\u63a5 \u4e0b\u8f7d\u540e\u89e3\u538b\u7f29\uff0c\u5728 bin/ \u76ee\u5f55\u4e0b\u5373\u53ef\u627e\u5230\u547d\u4ee4\u884c\u5de5\u5177\u7684\u4e8c\u8fdb\u5236\u6587\u4ef6\uff0c\u6587\u4ef6\u540d\u4e3a jindofs \u3002","title":"\u83b7\u53d6"},{"location":"jindofs/jindofs_client_tools/#_3","text":"JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528 JINDOSDK_CONF_DIR \u4e3a\u73af\u5883\u53d8\u91cf\u3002\u4f7f\u8be5\u53d8\u91cf\u6307\u5411\u4e00\u4e2a\u672c\u5730\u76ee\u5f55\uff0c\u5728\u76ee\u5f55\u4e0b\u653e\u7f6e\u914d\u7f6e\u6587\u4ef6 jindofs.cfg \uff0c\u4fbf\u53ef\u4ee5\u4e3a\u547d\u4ee4\u884c\u5de5\u5177\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\u3002 jindofs.cfg \u7684\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a [client] fs.oss.accessKeyId = <access key> fs.oss.accessKeySecret = <access secret> fs.oss.endpoint = <OSS-HDFS endpoint> fs.oss.data.lake.storage.enable = true # fs.oss.provider.endpoint = ECS_ROLE # EMR Credential provider for password-free access # fs.oss.provider.format = JSON \u5728\u89e3\u538b\u540e\u7684 conf/ \u76ee\u5f55\u4e0b\u4e5f\u53ef\u4ee5\u627e\u5230\u914d\u7f6e\u793a\u4f8b\u6587\u4ef6 jindofs.cfg.template \u3002","title":"\u914d\u7f6e"},{"location":"jindofs/jindofs_client_tools/#_4","text":"\u5728\u547d\u4ee4\u884c\u4e2d\u6267\u884c jindofs \u4e8c\u8fdb\u5236\u6587\u4ef6\uff08\u4f8b\u5982\uff0c cd \u81f3\u6240\u5728\u76ee\u5f55\uff0c\u6267\u884c ./jindofs \u6216\u8005 ./jindofs -help \uff09\uff0c\u5c06\u663e\u793a\u6240\u6709\u652f\u6301\u7684\u547d\u4ee4\uff1a jindofs [-help] [-version] jindofs admin [-help] [-addProxyUser -dlsUri <path> -proxyUser <value> -users|-groups <value> -hosts <value>] [-allowSnapshot -dlsUri <path>] [-addUserGroupsMapping -dlsUri <path> -user <value> -groups <value>] [-disallowSnapshot -dlsUri <path>] [-deleteProxyUser -dlsUri <path> -proxyUser <value>] [-deleteUserGroupsMapping -dlsUri <path> -user <value>] [-dumpMetaInfo -dlsUri <path>] [-dumpInventory <path>] [-getJobProgress <jobId> -dlsUri <path>] [-listJobs -dlsUri <path> [-stages <stages>] [-createTime <createTime>] [-type <type>]] [-listProxyUsers -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-listUserGroupsMappings -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-snapshotDiff -dlsUri <path> -fromSnapshot <value> -toSnapshot <value>] [-setRootPolicy <dlsRootPath> <accessRootPath>] [-unsetRootPolicy <dlsRootPath> <accessRootPath>] [-listAccessPolicies <dlsRootPath>] [-putConfig -dlsUri <path> -conf <key1=value1> -conf <key2=value2> ...] [-getConfig -dlsUri <path> -name <keys>] jindofs fs [-help] [-cat <path>] [-count [-h] <path>] [-cp [-f] <src> <dst>] [-checksum [-mode COMPOSITE_CRC(default)/MD5MD5CRC] [-blockSize <size in bytes (only used in MD5MD5CRC mode, 128MB by default)>] <src> ...] [-chgrp [-R] <group> <path>] [-chmod [-R] <mode> <path>] [-chown [-R] [owner][:[group]] <path>] [-copyFromLocal [-f] <localsrc> <dst>] [-copyToLocal [-f] <src> <localdst>] [-createSnapshot <snapshotDir> [<snapshotName>]] [-checkStoragePolicy -path <path>] [-deleteSnapshot <snapshotDir> <snapshotName>] [-du [-s] [-h] <path> ...] [-expunge [-immediate] <path>] [-get [-f] <src> <localdst>] [-getfacl [-R] <path>] [-getfattr [-R] {-n name | -d} <path>] [-getStoragePolicy -path <path>] [-ls [-R] <path>] [-listPolicies] [-mkdir [-p] <path>] [-mv <src> ... <dst>] [-moveFromLocal <localsrc> ... <dst>] [-moveToLocal <src> <localdst>] [-put [-f] <localsrc> <dst>] [-rm [-skipTrash] [-f] [-r|-R] [-safely] <path>] [-rmdir <path>] [-test -[defsz] <path>] [-renameSnapshot <snapshotDir> <oldName> <newName>] [-stat [format] <path>] [-setStoragePolicy -path <path> -policy <policy> [-restoreDays <restoreDays>]] [-setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>]] [-setfattr {-n name [-v value] | -x name} <path>] [-touchz <path>] [-truncate [-w] <length> <path>] [-unsetStoragePolicy -path <path>] \u53e6\u5916\uff0c\u6240\u6709\u547d\u4ee4\u5747\u652f\u6301 --extraConf key=value \u7684\u65b9\u5f0f\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u4ec5\u5bf9\u8be5\u6b21\u6267\u884c\u6709\u6548\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u5199\u5728\u914d\u7f6e\u6587\u4ef6\u4e2d\u3002","title":"\u7528\u6cd5"},{"location":"jindofs/jindofs_client_tools/#_5","text":"","title":"\u793a\u4f8b"},{"location":"jindofs/jindofs_client_tools/#_6","text":"./jindofs fs -put /local/file/path oss://<bucket-name>.<oss-hdfs-endpoint>/remote/file/path \u4f5c\u7528\u662f\u628a\u672c\u5730\u6587\u4ef6 /local/file/path \u4e0a\u4f20\u5230\u8fdc\u7aef\u8def\u5f84 oss://<bucket-name>.<oss-hdfs-endpoint>/remote/file/path \u3002","title":"\u4e0a\u4f20\u6587\u4ef6"},{"location":"jindofs/jindofs_client_tools/#_7","text":"./jindofs admin -dumpInventory oss://<bucket-name>.<oss-hdfs-endpoint>/ --extraConf fs.oss.accessKeyId=<access key> --extraConf fs.oss.accessKeySecret=<access secret> \u4e0a\u8ff0\u547d\u4ee4\u5411 OSS-HDFS \u670d\u52a1\u53d1\u8d77\u4e86\u4e00\u6b21\u6e05\u5355\u5bfc\u51fa\u4efb\u52a1\uff0c\u5e76\u4e14\u8fd8\u901a\u8fc7 --extraConf key=value \u7684\u65b9\u5f0f\u4e3a\u672c\u6b21\u6267\u884c\u63d0\u4f9b\u4e86\u989d\u5916\u7684\u4e24\u6761\u914d\u7f6e\u4fe1\u606f\u3002JindoFS \u7684\u6240\u6709\u547d\u4ee4\u5747\u652f\u6301 --extraConf \u53c2\u6570\u3002 \u5982\u679c\u547d\u4ee4\u6210\u529f\u6267\u884c\uff0c\u5c06\u8fd4\u56de\uff1a =============Dump Inventory============= Job Id: <job id> Data Location: <oss-hdfs path to inventory file> ....................... FINISHED. \u968f\u540e\u53ef\u5728 Data Location \u6307\u5411\u7684 OSS-HDFS \u8def\u5f84\u627e\u5230\u6e05\u5355\u5bfc\u51fa\u7684\u7ed3\u679c\u6587\u4ef6\uff0c\u6309\u666e\u901a\u6587\u4ef6\u8bbf\u95ee\u5373\u53ef\u3002","title":"\u6e05\u5355\u5bfc\u51fa"},{"location":"jindofs/jindofs_java_sdk/","text":"JindoFS Java SDK \u4f7f\u7528\u6307\u5357 \u00a7 \u4ecb\u7ecd \u00a7 JindoFS Java SDK \u63d0\u4f9b\u4e86\u8bbf\u95ee OSS-HDFS \u6570\u636e\u6e56\u5b58\u50a8\u670d\u52a1\u7684 Java API\u3002\u5b83\u652f\u6301\u5e38\u89c1\u7684\u6587\u4ef6\u7cfb\u7edf\u5143\u6570\u636e\u63a5\u53e3\u548c\u6d41\u5f0f\u6570\u636e\u64cd\u4f5c\uff0c\u5982\u521b\u5efa\u76ee\u5f55\u3001\u8bfb\u5199\u6216\u5220\u9664\u6587\u4ef6\uff1b\u540c\u65f6\u4e5f\u652f\u6301 OSS-HDFS \u7279\u6709\u7684\u65b9\u6cd5\uff0c\u5982\u6e05\u5355\u5bfc\u51fa\u7b49\u3002 JindoFS Java SDK \u5e76\u672a\u63d0\u4f9b\u57fa\u4e8e Hadoop-Compatible FileSystem\uff08HCFS\uff09\u7684\u5b9e\u73b0\u3002\u5982\u679c\u5e0c\u671b\u5728 Hadoop \u751f\u6001\u4e2d\u4ee5 HCFS \u6807\u51c6\u63a5\u53e3\u8bbf\u95ee OSS-HDFS\uff0c\u8bf7\u53c2\u8003 \u300aHadoop \u73af\u5883\u4f7f\u7528 JindoSDK \u5feb\u901f\u5165\u95e8\u300b \u672c\u6587\u6863\u57fa\u4e8e JindoFS Java SDK 6.2.5 \u7248\u672c\u7f16\u5199\u3002 \u4e0e JindoSDK \u7684\u533a\u522b \u00a7 JindoSDK \u540c\u6837\u63d0\u4f9b\u4e86\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684 Java API\u3002\u4e0e\u4e4b\u76f8\u6bd4\uff0cJindoFS Java SDK \u6709\u4e09\u4e2a\u4e3b\u8981\u533a\u522b\uff1a 1. JindoFS Java SDK \u4e0d\u8003\u8651\u539f\u751f\u652f\u6301 Hadoop \u751f\u6001\uff0c\u4e0d\u63d0\u4f9b\u57fa\u4e8e HCFS \u7684\u5b9e\u73b0\uff0c\u4e5f\u4e0d\u4f1a\u4ece Hadoop core-site \u8bfb\u53d6\u914d\u7f6e\u3002 2. JindoFS Java SDK \u53ea\u652f\u6301 OSS-HDFS\uff0c\u800c JindoSDK \u9664\u4e86\u652f\u6301 OSS-HDFS \u4ee5\u5916\uff0c\u8fd8\u53ef\u4ee5\u652f\u6301\u666e\u901a OSS\u3001S3\u3001HDFS \u7b49\u591a\u79cd\u5b58\u50a8\u670d\u52a1\u3002 3. JindoFS Java SDK \u63d0\u4f9b\u4e86\u4e00\u4e9b OSS-HDFS \u7279\u6709\u7684\u63a5\u53e3\uff0c\u8fd9\u4e9b\u63a5\u53e3\u53ef\u80fd\u662f JindoSDK \u672a\u63d0\u4f9b\u7684\uff0c\u4f8b\u5982\u6e05\u5355\u5bfc\u51fa\u7b49\u3002 \u5982\u679c\u5e0c\u671b\u4f7f\u7528 HCFS \u7684\u6587\u4ef6\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u300aHadoop \u73af\u5883\u4f7f\u7528 JindoSDK \u5feb\u901f\u5165\u95e8\u300b \uff0c\u5bf9 Hadoop \u751f\u6001 HCFS \u7684\u4ecb\u7ecd\u8bf7\u53c2\u9605 Apache Hadoop \u6587\u6863\u3002 \u83b7\u53d6 \u00a7 JindoFS Java SDK \u7684\u4f7f\u7528\u6709\u5e73\u53f0\u9650\u5236\uff0c\u76ee\u524d\u53ea\u63d0\u4f9b\u4e86 Linux x86 \u5e73\u53f0\u7684\u7248\u672c\uff0c \u4e0b\u8f7d\u94fe\u63a5 \u6ce8\uff1a\u5982\u679c\u5e0c\u671b\u5728 MacOS\u3001Ubuntu\u3001\u6216\u8005 Arch \u67b6\u6784\u4e0b\u4f7f\u7528 JindoFS Java SDK\uff0c\u8bf7\u8054\u7cfb JindoFS \u56e2\u961f\u63d0\u4f9b\u652f\u6301\u3002 \u4e0b\u8f7d\u540e\u89e3\u538b\u7f29\uff0c\u5728 lib/ \u76ee\u5f55\u4e0b\u5373\u53ef\u627e\u5230 JindoFS Java SDK \u7684\u8d44\u6e90\uff1a - jindofs-core-${version}.jar - jindofs-sdk-${version}.jar Maven \u00a7 \u76ee\u524d\u6682\u672a\u63d0\u4f9b\u53ef\u4ee5\u516c\u5171\u8bbf\u95ee\u7684 Maven \u4ed3\u5e93\u3002\u60a8\u53ef\u4ee5\u672c\u5730\u4f9d\u8d56\u8fd9\u4e24\u4e2a jar \u6587\u4ef6\uff0c\u6216\u8005\u5c06\u5b83\u4eec\u653e\u7f6e\u5728\u4e00\u4e2a\u60a8\u65b9\u4fbf\u8bbf\u95ee\u7684 Maven \u4ed3\u5e93\u4e2d\u3002 Maven \u4f9d\u8d56\u5199\u4f5c\uff1a <dependencies> <dependency> <groupId>com.aliyun.jindodata.jindofs</groupId> <artifactId>jindofs-core</artifactId> <version>${version}</version> </dependency> <dependency> <groupId>com.aliyun.jindodata.jindofs</groupId> <artifactId>jindofs-sdk</artifactId> <version>${version}</version> </dependency> </dependencies> \u5176\u4e2d ${version} \u6839\u636e\u5b9e\u9645\u7684\u7248\u672c\u53f7\u586b\u5199\u3002 \u4f7f\u7528 \u00a7 JindoFS Java SDK \u63d0\u4f9b\u4e86 JfsFileSystem \u7684\u63a5\u53e3\uff0c\u4ee5\u53ca\u4e00\u4e2a JfsFileSystemImpl \u7684\u5b9e\u73b0\u3002\u5ba2\u6237\u7aef\u521d\u59cb\u5316\u7684\u65b9\u5f0f\u4e3a\uff1a JfsFileSystem fs = new JfsFileSystemImpl(); fs.initialize(uri, user, conf); \u5176\u4e2d\uff1a 1. uri \u6307\u5411 OSS-HDFS \u7684\u521d\u59cb\u5316\u5730\u5740\uff0c\u4f8b\u5982 oss://<your-oss-hdfs-bucket>.<oss-hdfs-endpoint>/init-path 2. user \u8868\u793a\u5ba2\u6237\u7aef\u4ee3\u8868\u7684\u7528\u6237\u540d\u79f0\u3002 3. conf \u662f\u4e00\u4e2a JfsConfiguration \u5bf9\u8c61\u3002 JfsConfiguration \u662f\u5ba2\u6237\u7aef\u63a5\u53d7\u914d\u7f6e\u7684\u552f\u4e00\u9014\u5f84\uff0cJindoFS Java SDK \u672c\u8eab\u4e0d\u4f1a\u8bfb\u53d6\u4efb\u4f55\u914d\u7f6e\u6587\u4ef6\u3002\u901a\u8fc7 JfsConfiguration \u53ef\u4ee5\u8bbe\u7f6e\u591a\u79cd\u914d\u7f6e\u53c2\u6570\uff0c\u5176\u4e2d JfsConfiguration.JfsAccessInfo \u5305\u542b\u8bbf\u95ee OSS-HDFS \u5fc5\u8981\u7684\u4fe1\u606f\uff0c\u5176\u4ed6\u914d\u7f6e\u9879\u5747\u53ef\u4ee5\u91c7\u7528\u9ed8\u8ba4\u503c\u3002 \u6784\u9020 JfsConfiguration.JfsAccessInfo \u5e76\u8fdb\u4e00\u6b65\u521b\u5efa JfsConfiguration \u7684\u65b9\u6cd5\u5982\u4e0b\uff1a JfsConfiguration.JfsAccessInfo accessInfo = new JfsConfiguration.JfsAccessInfo(); accessInfo.accessKeyId = \"your access key ID\"; accessInfo.accessKeySecret = \"your access key secret\"; accessInfo.endpoint = \"oss-hdfs-endpoint\"; JfsConfiguration conf = new JfsConfiguration.Builder() .setDefaultAccessInfo(accessInfo) .build(); \u6ce8\u610f\u4e8b\u9879 \u00a7 \u4e0e Hadoop \u751f\u6001\u7684\u4f7f\u7528\u4e60\u60ef\u76f8\u6096\u7684\u662f\uff0c JfsFileSystem \u7684\u5b9e\u4f8b\u6ca1\u6709\u5168\u5c40\u5171\u4eab\u7684\u673a\u5236\uff0c\u4e5f\u4e0d\u63d0\u4f9b\u4ece\u8def\u5f84\u83b7\u53d6 JfsFileSystem \u5b9e\u4f8b\u7684\u65b9\u6cd5\u3002\u5e76\u4e14\uff0c\u6bcf\u4e00\u4e2a\u65b0\u521b\u5efa\u7684 JfsFileSystem \u5b9e\u4f8b\u5728\u4f7f\u7528\u5b8c\u4e4b\u540e\u90fd\u9700\u8981\u4e3b\u52a8\u8c03\u7528 close \u5173\u95ed\uff0c\u5426\u5219\u53ef\u80fd\u4f1a\u4ea7\u751f\u8d44\u6e90\u6cc4\u9732\u3002\u53e6\u4e00\u65b9\u9762\uff0c\u5bf9\u4e8e\u8bfb\u5199\u6d41 JfsInputStream \u4e0e JfsOutputStream \u7684\u5bf9\u8c61\uff0c\u5982\u679c\u4e0d\u8c03\u7528 close \u5e76\u6ca1\u6709\u8d44\u6e90\u6cc4\u9732\u4e4b\u865e\u3002 \u793a\u4f8b \u00a7 \u4e0b\u9762\u662f\u4e00\u4e2a\u5b8c\u6574\u7684\u4ee3\u7801\u793a\u4f8b\uff0c\u7528\u6765\u6f14\u793a\u5982\u4f55\u521d\u59cb\u5316\u4e00\u4e2a JfsFileSystem \u5bf9\u8c61\uff0c\u5e76\u4e14\u5728 OSS-HDFS \u4e2d\u521b\u5efa\u4e00\u4e2a\u76ee\u5f55\uff1a package com.aliyun.jindodata.jfssdk.demo; import com.aliyun.jindodata.jfssdk.JfsConfiguration; import com.aliyun.jindodata.jfssdk.JfsFileSystem; import com.aliyun.jindodata.jfssdk.JfsFileSystemImpl; import java.io.IOException; public class Demo { public static void main(String[] args) throws IOException { JfsConfiguration.JfsAccessInfo accessInfo = new JfsConfiguration.JfsAccessInfo(); accessInfo.accessKeyId = \"your access key ID\"; accessInfo.accessKeySecret = \"your access key secret\"; accessInfo.endpoint = \"oss-hdfs-endpoint\"; JfsConfiguration conf = new JfsConfiguration.Builder() .setDefaultAccessInfo(accessInfo) .build(); String uri = \"oss://<your-oss-hdfs-bucket>.<oss-hdfs-endpoint>/init-uri\"; try (JfsFileSystem fs = new JfsFileSystemImpl()) { fs.initialize(uri, \"hadoop\", conf); fs.mkdirs(\"oss://<your-oss-hdfs-bucket>.<oss-hdfs-endpoint>/a-new-dir\"); } } } \u5176\u4e2d\uff0c your-oss-hdfs-bucket \u3001 your access key ID \u3001 your access key secret \u3001 oss-hdfs-endpoint \u6839\u636e\u5b9e\u9645\u503c\u586b\u5199\u3002","title":"JindoFS Java SDK \u4f7f\u7528\u6307\u5357"},{"location":"jindofs/jindofs_java_sdk/#jindofs-java-sdk","text":"","title":"JindoFS Java SDK \u4f7f\u7528\u6307\u5357"},{"location":"jindofs/jindofs_java_sdk/#_1","text":"JindoFS Java SDK \u63d0\u4f9b\u4e86\u8bbf\u95ee OSS-HDFS \u6570\u636e\u6e56\u5b58\u50a8\u670d\u52a1\u7684 Java API\u3002\u5b83\u652f\u6301\u5e38\u89c1\u7684\u6587\u4ef6\u7cfb\u7edf\u5143\u6570\u636e\u63a5\u53e3\u548c\u6d41\u5f0f\u6570\u636e\u64cd\u4f5c\uff0c\u5982\u521b\u5efa\u76ee\u5f55\u3001\u8bfb\u5199\u6216\u5220\u9664\u6587\u4ef6\uff1b\u540c\u65f6\u4e5f\u652f\u6301 OSS-HDFS \u7279\u6709\u7684\u65b9\u6cd5\uff0c\u5982\u6e05\u5355\u5bfc\u51fa\u7b49\u3002 JindoFS Java SDK \u5e76\u672a\u63d0\u4f9b\u57fa\u4e8e Hadoop-Compatible FileSystem\uff08HCFS\uff09\u7684\u5b9e\u73b0\u3002\u5982\u679c\u5e0c\u671b\u5728 Hadoop \u751f\u6001\u4e2d\u4ee5 HCFS \u6807\u51c6\u63a5\u53e3\u8bbf\u95ee OSS-HDFS\uff0c\u8bf7\u53c2\u8003 \u300aHadoop \u73af\u5883\u4f7f\u7528 JindoSDK \u5feb\u901f\u5165\u95e8\u300b \u672c\u6587\u6863\u57fa\u4e8e JindoFS Java SDK 6.2.5 \u7248\u672c\u7f16\u5199\u3002","title":"\u4ecb\u7ecd"},{"location":"jindofs/jindofs_java_sdk/#jindosdk","text":"JindoSDK \u540c\u6837\u63d0\u4f9b\u4e86\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684 Java API\u3002\u4e0e\u4e4b\u76f8\u6bd4\uff0cJindoFS Java SDK \u6709\u4e09\u4e2a\u4e3b\u8981\u533a\u522b\uff1a 1. JindoFS Java SDK \u4e0d\u8003\u8651\u539f\u751f\u652f\u6301 Hadoop \u751f\u6001\uff0c\u4e0d\u63d0\u4f9b\u57fa\u4e8e HCFS \u7684\u5b9e\u73b0\uff0c\u4e5f\u4e0d\u4f1a\u4ece Hadoop core-site \u8bfb\u53d6\u914d\u7f6e\u3002 2. JindoFS Java SDK \u53ea\u652f\u6301 OSS-HDFS\uff0c\u800c JindoSDK \u9664\u4e86\u652f\u6301 OSS-HDFS \u4ee5\u5916\uff0c\u8fd8\u53ef\u4ee5\u652f\u6301\u666e\u901a OSS\u3001S3\u3001HDFS \u7b49\u591a\u79cd\u5b58\u50a8\u670d\u52a1\u3002 3. JindoFS Java SDK \u63d0\u4f9b\u4e86\u4e00\u4e9b OSS-HDFS \u7279\u6709\u7684\u63a5\u53e3\uff0c\u8fd9\u4e9b\u63a5\u53e3\u53ef\u80fd\u662f JindoSDK \u672a\u63d0\u4f9b\u7684\uff0c\u4f8b\u5982\u6e05\u5355\u5bfc\u51fa\u7b49\u3002 \u5982\u679c\u5e0c\u671b\u4f7f\u7528 HCFS \u7684\u6587\u4ef6\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u300aHadoop \u73af\u5883\u4f7f\u7528 JindoSDK \u5feb\u901f\u5165\u95e8\u300b \uff0c\u5bf9 Hadoop \u751f\u6001 HCFS \u7684\u4ecb\u7ecd\u8bf7\u53c2\u9605 Apache Hadoop \u6587\u6863\u3002","title":"\u4e0e JindoSDK \u7684\u533a\u522b"},{"location":"jindofs/jindofs_java_sdk/#_2","text":"JindoFS Java SDK \u7684\u4f7f\u7528\u6709\u5e73\u53f0\u9650\u5236\uff0c\u76ee\u524d\u53ea\u63d0\u4f9b\u4e86 Linux x86 \u5e73\u53f0\u7684\u7248\u672c\uff0c \u4e0b\u8f7d\u94fe\u63a5 \u6ce8\uff1a\u5982\u679c\u5e0c\u671b\u5728 MacOS\u3001Ubuntu\u3001\u6216\u8005 Arch \u67b6\u6784\u4e0b\u4f7f\u7528 JindoFS Java SDK\uff0c\u8bf7\u8054\u7cfb JindoFS \u56e2\u961f\u63d0\u4f9b\u652f\u6301\u3002 \u4e0b\u8f7d\u540e\u89e3\u538b\u7f29\uff0c\u5728 lib/ \u76ee\u5f55\u4e0b\u5373\u53ef\u627e\u5230 JindoFS Java SDK \u7684\u8d44\u6e90\uff1a - jindofs-core-${version}.jar - jindofs-sdk-${version}.jar","title":"\u83b7\u53d6"},{"location":"jindofs/jindofs_java_sdk/#maven","text":"\u76ee\u524d\u6682\u672a\u63d0\u4f9b\u53ef\u4ee5\u516c\u5171\u8bbf\u95ee\u7684 Maven \u4ed3\u5e93\u3002\u60a8\u53ef\u4ee5\u672c\u5730\u4f9d\u8d56\u8fd9\u4e24\u4e2a jar \u6587\u4ef6\uff0c\u6216\u8005\u5c06\u5b83\u4eec\u653e\u7f6e\u5728\u4e00\u4e2a\u60a8\u65b9\u4fbf\u8bbf\u95ee\u7684 Maven \u4ed3\u5e93\u4e2d\u3002 Maven \u4f9d\u8d56\u5199\u4f5c\uff1a <dependencies> <dependency> <groupId>com.aliyun.jindodata.jindofs</groupId> <artifactId>jindofs-core</artifactId> <version>${version}</version> </dependency> <dependency> <groupId>com.aliyun.jindodata.jindofs</groupId> <artifactId>jindofs-sdk</artifactId> <version>${version}</version> </dependency> </dependencies> \u5176\u4e2d ${version} \u6839\u636e\u5b9e\u9645\u7684\u7248\u672c\u53f7\u586b\u5199\u3002","title":"Maven"},{"location":"jindofs/jindofs_java_sdk/#_3","text":"JindoFS Java SDK \u63d0\u4f9b\u4e86 JfsFileSystem \u7684\u63a5\u53e3\uff0c\u4ee5\u53ca\u4e00\u4e2a JfsFileSystemImpl \u7684\u5b9e\u73b0\u3002\u5ba2\u6237\u7aef\u521d\u59cb\u5316\u7684\u65b9\u5f0f\u4e3a\uff1a JfsFileSystem fs = new JfsFileSystemImpl(); fs.initialize(uri, user, conf); \u5176\u4e2d\uff1a 1. uri \u6307\u5411 OSS-HDFS \u7684\u521d\u59cb\u5316\u5730\u5740\uff0c\u4f8b\u5982 oss://<your-oss-hdfs-bucket>.<oss-hdfs-endpoint>/init-path 2. user \u8868\u793a\u5ba2\u6237\u7aef\u4ee3\u8868\u7684\u7528\u6237\u540d\u79f0\u3002 3. conf \u662f\u4e00\u4e2a JfsConfiguration \u5bf9\u8c61\u3002 JfsConfiguration \u662f\u5ba2\u6237\u7aef\u63a5\u53d7\u914d\u7f6e\u7684\u552f\u4e00\u9014\u5f84\uff0cJindoFS Java SDK \u672c\u8eab\u4e0d\u4f1a\u8bfb\u53d6\u4efb\u4f55\u914d\u7f6e\u6587\u4ef6\u3002\u901a\u8fc7 JfsConfiguration \u53ef\u4ee5\u8bbe\u7f6e\u591a\u79cd\u914d\u7f6e\u53c2\u6570\uff0c\u5176\u4e2d JfsConfiguration.JfsAccessInfo \u5305\u542b\u8bbf\u95ee OSS-HDFS \u5fc5\u8981\u7684\u4fe1\u606f\uff0c\u5176\u4ed6\u914d\u7f6e\u9879\u5747\u53ef\u4ee5\u91c7\u7528\u9ed8\u8ba4\u503c\u3002 \u6784\u9020 JfsConfiguration.JfsAccessInfo \u5e76\u8fdb\u4e00\u6b65\u521b\u5efa JfsConfiguration \u7684\u65b9\u6cd5\u5982\u4e0b\uff1a JfsConfiguration.JfsAccessInfo accessInfo = new JfsConfiguration.JfsAccessInfo(); accessInfo.accessKeyId = \"your access key ID\"; accessInfo.accessKeySecret = \"your access key secret\"; accessInfo.endpoint = \"oss-hdfs-endpoint\"; JfsConfiguration conf = new JfsConfiguration.Builder() .setDefaultAccessInfo(accessInfo) .build();","title":"\u4f7f\u7528"},{"location":"jindofs/jindofs_java_sdk/#_4","text":"\u4e0e Hadoop \u751f\u6001\u7684\u4f7f\u7528\u4e60\u60ef\u76f8\u6096\u7684\u662f\uff0c JfsFileSystem \u7684\u5b9e\u4f8b\u6ca1\u6709\u5168\u5c40\u5171\u4eab\u7684\u673a\u5236\uff0c\u4e5f\u4e0d\u63d0\u4f9b\u4ece\u8def\u5f84\u83b7\u53d6 JfsFileSystem \u5b9e\u4f8b\u7684\u65b9\u6cd5\u3002\u5e76\u4e14\uff0c\u6bcf\u4e00\u4e2a\u65b0\u521b\u5efa\u7684 JfsFileSystem \u5b9e\u4f8b\u5728\u4f7f\u7528\u5b8c\u4e4b\u540e\u90fd\u9700\u8981\u4e3b\u52a8\u8c03\u7528 close \u5173\u95ed\uff0c\u5426\u5219\u53ef\u80fd\u4f1a\u4ea7\u751f\u8d44\u6e90\u6cc4\u9732\u3002\u53e6\u4e00\u65b9\u9762\uff0c\u5bf9\u4e8e\u8bfb\u5199\u6d41 JfsInputStream \u4e0e JfsOutputStream \u7684\u5bf9\u8c61\uff0c\u5982\u679c\u4e0d\u8c03\u7528 close \u5e76\u6ca1\u6709\u8d44\u6e90\u6cc4\u9732\u4e4b\u865e\u3002","title":"\u6ce8\u610f\u4e8b\u9879"},{"location":"jindofs/jindofs_java_sdk/#_5","text":"\u4e0b\u9762\u662f\u4e00\u4e2a\u5b8c\u6574\u7684\u4ee3\u7801\u793a\u4f8b\uff0c\u7528\u6765\u6f14\u793a\u5982\u4f55\u521d\u59cb\u5316\u4e00\u4e2a JfsFileSystem \u5bf9\u8c61\uff0c\u5e76\u4e14\u5728 OSS-HDFS \u4e2d\u521b\u5efa\u4e00\u4e2a\u76ee\u5f55\uff1a package com.aliyun.jindodata.jfssdk.demo; import com.aliyun.jindodata.jfssdk.JfsConfiguration; import com.aliyun.jindodata.jfssdk.JfsFileSystem; import com.aliyun.jindodata.jfssdk.JfsFileSystemImpl; import java.io.IOException; public class Demo { public static void main(String[] args) throws IOException { JfsConfiguration.JfsAccessInfo accessInfo = new JfsConfiguration.JfsAccessInfo(); accessInfo.accessKeyId = \"your access key ID\"; accessInfo.accessKeySecret = \"your access key secret\"; accessInfo.endpoint = \"oss-hdfs-endpoint\"; JfsConfiguration conf = new JfsConfiguration.Builder() .setDefaultAccessInfo(accessInfo) .build(); String uri = \"oss://<your-oss-hdfs-bucket>.<oss-hdfs-endpoint>/init-uri\"; try (JfsFileSystem fs = new JfsFileSystemImpl()) { fs.initialize(uri, \"hadoop\", conf); fs.mkdirs(\"oss://<your-oss-hdfs-bucket>.<oss-hdfs-endpoint>/a-new-dir\"); } } } \u5176\u4e2d\uff0c your-oss-hdfs-bucket \u3001 your access key ID \u3001 your access key secret \u3001 oss-hdfs-endpoint \u6839\u636e\u5b9e\u9645\u503c\u586b\u5199\u3002","title":"\u793a\u4f8b"},{"location":"jindofs/jindofs_protected_directories/","text":"\u80cc\u666f \u00a7 JindoFS \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u5f53\u524d\u53ef\u4ee5\u901a\u8fc7\u547d\u4ee4\u884c\u547d\u4ee4\u7684\u65b9\u5f0f\u8fdb\u884c\u64cd\u4f5c\uff0c\u53ef\u4ee5\u663e\u793a\u6b63\u5728\u4fdd\u62a4\u7684\u76ee\u5f55\u5217\u8868\uff0c\u6216\u8005\u4fee\u6539\u4fdd\u62a4\u7684\u76ee\u5f55\u3002\u5c06\u6765\u8fd8\u4f1a\u901a\u8fc7\u4ea7\u54c1\u754c\u9762\u7684\u65b9\u5f0f\uff0c\u63d0\u4f9b\u66f4\u52a0\u4fbf\u6377\u6613\u7528\u7684\u67e5\u770b\u548c\u8bbe\u7f6e\u65b9\u6cd5\u3002 \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u63cf\u8ff0 \u00a7 JindoFS \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\uff0c\u4e3b\u8981\u9632\u6b62\u7528\u6237\u8bef\u5220\u76ee\u5f55\uff0c\u88ab\u4fdd\u62a4\u7684\u76ee\u5f55\u53ea\u8981\u8be5\u76ee\u5f55\u4e0b\u5b58\u5728\u76ee\u5f55\u6216\u8005\u6587\u4ef6\uff0c\u4fdd\u62a4\u76ee\u5f55\u7684\u5220\u9664\u4ee5\u53ca\u91cd\u547d\u540d\u64cd\u4f5c\u4f1a\u62a5\u9519\uff0c\u53ea\u6709\u5f53\u88ab\u4fdd\u62a4\u7684\u76ee\u5f55\u4e0b\u4e0d\u5b58\u5728\u4efb\u4f55\u76ee\u5f55\u6216\u8005\u6587\u4ef6\uff0c\u4fdd\u62a4\u76ee\u5f55\u624d\u4f1a\u88ab\u5141\u8bb8\u5220\u9664\u6216\u8005\u91cd\u547d\u540d\u3002 JindoFS \u547d\u4ee4\u884c\u7b80\u4ecb \u00a7 \u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u9700\u8981\u7528\u5230\u7684\u662f -putConfig \u4e0e -getConfig \u547d\u4ee4\uff0c\u5177\u4f53\u7528\u6cd5\u5c06\u5728\u4e0b\u6587\u4ecb\u7ecd\u3002 \u914d\u7f6e\u8bbf\u95ee AK \u00a7 \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u9700\u8981\u914d\u7f6e\u8fde\u63a5 OSS-HDFS \u6240\u9700\u7684 AK \u4fe1\u606f\u3002\u6709\u4e24\u79cd\u65b9\u6cd5\u53ef\u4ee5\u8fdb\u884c\u914d\u7f6e\uff1a \u914d\u7f6e\u6587\u4ef6\uff08\u63a8\u8350\uff09 \u00a7 \u8bbe\u7f6e\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6307\u5411\u67d0\u4e2a\u76ee\u5f55\uff0c\u7136\u540e\u5728\u76ee\u5f55\u4e0b\u521b\u5efa\u540d\u4e3a jindofs.cfg \u7684\u6587\u4ef6\uff0c\u7136\u540e\u5728\u6587\u4ef6\u91cc\u5199\u5165\u5982\u4e0b\u5185\u5bb9\uff1a [client] fs.oss.accessKeyId = <key> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK key fs.oss.accessKeySecret = <secret> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK secret fs.oss.bucket.<bucket>.accessKeyId = <key> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK key\uff0c\u4f18\u5148\u7ea7\u9ad8\u4e8e\u9ed8\u8ba4 fs.oss.bucket.<bucket>.accessKeySecret = <secret> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK secret --extraConf \u00a7 \u547d\u4ee4\u884c\u5de5\u5177\u7684\u6240\u6709\u6307\u4ee4\u5747\u53ef\u901a\u8fc7 --extraConf \u9009\u9879\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u914d\u7f6e\u6587\u4ef6\uff0c\u4f8b\u5982\uff1a jindofs admin -getConfig -dlsUri <path> -name <keys> \\ --extraConf fs.oss.accessKeyId=<AK key> \\ --extraConf fs.oss.accessKeySecret=<AK secret> \u5c06\u7ed9\u547d\u4ee4\u63d0\u4f9b\u989d\u5916\u7684 AK key \u4e0e AK secret \u914d\u7f6e\u3002 \u8bbe\u7f6e\u4fdd\u62a4\u76ee\u5f55 \u00a7 \u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf fs.protected.directories=/path/to/dir1,/path/to/dir2 \u4e0a\u8ff0\u547d\u4ee4\u5bf9\u540d\u4e3a \uff08endpoint \u4e3a \uff09\u7684\u6876\u914d\u7f6e\u4e86\u4e24\u4e2a\u4fdd\u62a4\u76ee\u5f55\uff1a /path/to/dir1 /path/to/dir2 \u5982\u793a\u4f8b\u4e2d\u7684\u683c\u5f0f\uff0c\u8fd9\u4e9b\u4fdd\u62a4\u76ee\u5f55\u7684\u8def\u5f84\u9700\u8981\u5199\u5728\u4e00\u8d77\uff0c\u9017\u53f7\u9694\u5f00\u3002\u8def\u5f84\u4e0d\u4ee5 oss:// \u5f00\u5934\uff0c\u800c\u662f\u4f7f\u7528\u6876\u5185\u4ee5 / \u5f00\u5934\u7684\u7edd\u5bf9\u8def\u5f84\u3002\u5982\u679c\u547d\u4ee4\u5931\u8d25\uff0c\u5c06\u663e\u793a\u5931\u8d25\u4fe1\u606f\u3002\u5982\u679c\u6ca1\u6709\u4efb\u4f55\u4fe1\u606f\u663e\u793a\uff0c\u8868\u660e\u6267\u884c\u6210\u529f\u3002\u547d\u4ee4\u6267\u884c\u6210\u529f\u4e4b\u540e\uff0c\u5bf9\u8fd9\u4e9b\u76ee\u5f55\u7684\u4fdd\u62a4\u529f\u80fd\u5c06\u5728 30 \u79d2\u5185\u6b63\u5f0f\u751f\u6548\u3002 \u67e5\u770b\u4fdd\u62a4\u76ee\u5f55 \u00a7 \u547d\u4ee4\uff1a jindofs admin -getConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -name fs.protected.directories \u4e0a\u8ff0\u547d\u4ee4\u8fd4\u56de\u4e86\u540d\u4e3a \uff08endpoint \u4e3a \uff09\u7684\u6876\u7684\u4fdd\u62a4\u76ee\u5f55\u5217\u8868\uff0c\u8fd4\u56de\u503c\u793a\u4f8b\uff1a fs.protected.directories: /path/to/dir1,/path/to/dir2 \u8868\u660e\u8fd9\u4e9b\u76ee\u5f55\u76ee\u524d\u6b63\u5728\u4fdd\u62a4\u5f53\u4e2d\uff1a /path/to/dir1 /path/to/dir2 \u5220\u9664\u6240\u6709\u4fdd\u62a4\u76ee\u5f55 \u00a7 \u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf fs.protected.directories= \u4e0a\u8ff0\u547d\u4ee4\u5220\u9664\u4e86\u540d\u4e3a \\<bucket>\uff08endpoint \u4e3a \\<oss-hdfs-endpoint>\uff09\u7684\u6876\u7684\u6240\u6709\u4fdd\u62a4\u76ee\u5f55\uff0c\u6b63\u5f0f\u751f\u6548\u540e\uff08\u547d\u4ee4\u8fd4\u56de\u540e 30 \u79d2\u5185\uff09\uff0c\u8be5\u6876\u5c06\u4e0d\u518d\u6709\u4efb\u4f55\u76ee\u5f55\u53d7\u5230\u4fdd\u62a4\u3002","title":"Jindofs protected directories"},{"location":"jindofs/jindofs_protected_directories/#_1","text":"JindoFS \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u5f53\u524d\u53ef\u4ee5\u901a\u8fc7\u547d\u4ee4\u884c\u547d\u4ee4\u7684\u65b9\u5f0f\u8fdb\u884c\u64cd\u4f5c\uff0c\u53ef\u4ee5\u663e\u793a\u6b63\u5728\u4fdd\u62a4\u7684\u76ee\u5f55\u5217\u8868\uff0c\u6216\u8005\u4fee\u6539\u4fdd\u62a4\u7684\u76ee\u5f55\u3002\u5c06\u6765\u8fd8\u4f1a\u901a\u8fc7\u4ea7\u54c1\u754c\u9762\u7684\u65b9\u5f0f\uff0c\u63d0\u4f9b\u66f4\u52a0\u4fbf\u6377\u6613\u7528\u7684\u67e5\u770b\u548c\u8bbe\u7f6e\u65b9\u6cd5\u3002","title":"\u80cc\u666f"},{"location":"jindofs/jindofs_protected_directories/#_2","text":"JindoFS \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\uff0c\u4e3b\u8981\u9632\u6b62\u7528\u6237\u8bef\u5220\u76ee\u5f55\uff0c\u88ab\u4fdd\u62a4\u7684\u76ee\u5f55\u53ea\u8981\u8be5\u76ee\u5f55\u4e0b\u5b58\u5728\u76ee\u5f55\u6216\u8005\u6587\u4ef6\uff0c\u4fdd\u62a4\u76ee\u5f55\u7684\u5220\u9664\u4ee5\u53ca\u91cd\u547d\u540d\u64cd\u4f5c\u4f1a\u62a5\u9519\uff0c\u53ea\u6709\u5f53\u88ab\u4fdd\u62a4\u7684\u76ee\u5f55\u4e0b\u4e0d\u5b58\u5728\u4efb\u4f55\u76ee\u5f55\u6216\u8005\u6587\u4ef6\uff0c\u4fdd\u62a4\u76ee\u5f55\u624d\u4f1a\u88ab\u5141\u8bb8\u5220\u9664\u6216\u8005\u91cd\u547d\u540d\u3002","title":"\u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u63cf\u8ff0"},{"location":"jindofs/jindofs_protected_directories/#jindofs","text":"\u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u9700\u8981\u7528\u5230\u7684\u662f -putConfig \u4e0e -getConfig \u547d\u4ee4\uff0c\u5177\u4f53\u7528\u6cd5\u5c06\u5728\u4e0b\u6587\u4ecb\u7ecd\u3002","title":"JindoFS \u547d\u4ee4\u884c\u7b80\u4ecb"},{"location":"jindofs/jindofs_protected_directories/#ak","text":"\u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u9700\u8981\u914d\u7f6e\u8fde\u63a5 OSS-HDFS \u6240\u9700\u7684 AK \u4fe1\u606f\u3002\u6709\u4e24\u79cd\u65b9\u6cd5\u53ef\u4ee5\u8fdb\u884c\u914d\u7f6e\uff1a","title":"\u914d\u7f6e\u8bbf\u95ee AK"},{"location":"jindofs/jindofs_protected_directories/#_3","text":"\u8bbe\u7f6e\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6307\u5411\u67d0\u4e2a\u76ee\u5f55\uff0c\u7136\u540e\u5728\u76ee\u5f55\u4e0b\u521b\u5efa\u540d\u4e3a jindofs.cfg \u7684\u6587\u4ef6\uff0c\u7136\u540e\u5728\u6587\u4ef6\u91cc\u5199\u5165\u5982\u4e0b\u5185\u5bb9\uff1a [client] fs.oss.accessKeyId = <key> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK key fs.oss.accessKeySecret = <secret> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK secret fs.oss.bucket.<bucket>.accessKeyId = <key> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK key\uff0c\u4f18\u5148\u7ea7\u9ad8\u4e8e\u9ed8\u8ba4 fs.oss.bucket.<bucket>.accessKeySecret = <secret> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK secret","title":"\u914d\u7f6e\u6587\u4ef6\uff08\u63a8\u8350\uff09"},{"location":"jindofs/jindofs_protected_directories/#-extraconf","text":"\u547d\u4ee4\u884c\u5de5\u5177\u7684\u6240\u6709\u6307\u4ee4\u5747\u53ef\u901a\u8fc7 --extraConf \u9009\u9879\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u914d\u7f6e\u6587\u4ef6\uff0c\u4f8b\u5982\uff1a jindofs admin -getConfig -dlsUri <path> -name <keys> \\ --extraConf fs.oss.accessKeyId=<AK key> \\ --extraConf fs.oss.accessKeySecret=<AK secret> \u5c06\u7ed9\u547d\u4ee4\u63d0\u4f9b\u989d\u5916\u7684 AK key \u4e0e AK secret \u914d\u7f6e\u3002","title":"--extraConf"},{"location":"jindofs/jindofs_protected_directories/#_4","text":"\u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf fs.protected.directories=/path/to/dir1,/path/to/dir2 \u4e0a\u8ff0\u547d\u4ee4\u5bf9\u540d\u4e3a \uff08endpoint \u4e3a \uff09\u7684\u6876\u914d\u7f6e\u4e86\u4e24\u4e2a\u4fdd\u62a4\u76ee\u5f55\uff1a /path/to/dir1 /path/to/dir2 \u5982\u793a\u4f8b\u4e2d\u7684\u683c\u5f0f\uff0c\u8fd9\u4e9b\u4fdd\u62a4\u76ee\u5f55\u7684\u8def\u5f84\u9700\u8981\u5199\u5728\u4e00\u8d77\uff0c\u9017\u53f7\u9694\u5f00\u3002\u8def\u5f84\u4e0d\u4ee5 oss:// \u5f00\u5934\uff0c\u800c\u662f\u4f7f\u7528\u6876\u5185\u4ee5 / \u5f00\u5934\u7684\u7edd\u5bf9\u8def\u5f84\u3002\u5982\u679c\u547d\u4ee4\u5931\u8d25\uff0c\u5c06\u663e\u793a\u5931\u8d25\u4fe1\u606f\u3002\u5982\u679c\u6ca1\u6709\u4efb\u4f55\u4fe1\u606f\u663e\u793a\uff0c\u8868\u660e\u6267\u884c\u6210\u529f\u3002\u547d\u4ee4\u6267\u884c\u6210\u529f\u4e4b\u540e\uff0c\u5bf9\u8fd9\u4e9b\u76ee\u5f55\u7684\u4fdd\u62a4\u529f\u80fd\u5c06\u5728 30 \u79d2\u5185\u6b63\u5f0f\u751f\u6548\u3002","title":"\u8bbe\u7f6e\u4fdd\u62a4\u76ee\u5f55"},{"location":"jindofs/jindofs_protected_directories/#_5","text":"\u547d\u4ee4\uff1a jindofs admin -getConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -name fs.protected.directories \u4e0a\u8ff0\u547d\u4ee4\u8fd4\u56de\u4e86\u540d\u4e3a \uff08endpoint \u4e3a \uff09\u7684\u6876\u7684\u4fdd\u62a4\u76ee\u5f55\u5217\u8868\uff0c\u8fd4\u56de\u503c\u793a\u4f8b\uff1a fs.protected.directories: /path/to/dir1,/path/to/dir2 \u8868\u660e\u8fd9\u4e9b\u76ee\u5f55\u76ee\u524d\u6b63\u5728\u4fdd\u62a4\u5f53\u4e2d\uff1a /path/to/dir1 /path/to/dir2","title":"\u67e5\u770b\u4fdd\u62a4\u76ee\u5f55"},{"location":"jindofs/jindofs_protected_directories/#_6","text":"\u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf fs.protected.directories= \u4e0a\u8ff0\u547d\u4ee4\u5220\u9664\u4e86\u540d\u4e3a \\<bucket>\uff08endpoint \u4e3a \\<oss-hdfs-endpoint>\uff09\u7684\u6876\u7684\u6240\u6709\u4fdd\u62a4\u76ee\u5f55\uff0c\u6b63\u5f0f\u751f\u6548\u540e\uff08\u547d\u4ee4\u8fd4\u56de\u540e 30 \u79d2\u5185\uff09\uff0c\u8be5\u6876\u5c06\u4e0d\u518d\u6709\u4efb\u4f55\u76ee\u5f55\u53d7\u5230\u4fdd\u62a4\u3002","title":"\u5220\u9664\u6240\u6709\u4fdd\u62a4\u76ee\u5f55"},{"location":"jindofs/jindofs_storagepolicy_restore/","text":"\u4f7f\u7528 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u5b9e\u73b0\u51b7\u70ed\u5206\u5c42\u5b58\u50a8 \u00a7 \u6982\u8ff0 \u00a7 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u662f\u8bbf\u95ee OSS-HDFS \u7684\u53ef\u6267\u884c\u7a0b\u5e8f\uff0c\u652f\u6301\u5e38\u89c1\u6587\u4ef6\u5143\u6570\u636e\u53ca\u8bfb\u5199\u64cd\u4f5c\uff0c\u4ee5\u53ca\u670d\u52a1\u7279\u6709\u65b9\u6cd5\u3002\u66f4\u591a\u4ecb\u7ecd\uff0c\u4ee5\u53ca\u5de5\u5177\u7684\u83b7\u53d6\u548c\u914d\u7f6e\u65b9\u6cd5\uff0c\u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 OSS-HDFS \u670d\u52a1\u7684\u4ecb\u7ecd\uff0c\u53c2\u89c1 OSS-HDFS\u670d\u52a1\u6982\u8ff0 \u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u4f7f\u7528 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u5b9e\u73b0 OSS-HDFS \u670d\u52a1\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8\u529f\u80fd\uff0c\u4ee5\u52a9\u4e8e\u66f4\u597d\u5730\u6cbb\u7406\u6570\u636e\uff0c\u7ba1\u7406\u6210\u672c\u3002 \u5b58\u50a8\u7c7b\u578b \u00a7 OSS-HDFS \u670d\u52a1\u76ee\u524d\u652f\u6301\u516d\u79cd\u5b58\u50a8\u7c7b\u578b\uff0c\u4f9d\u6b21\u4e3a\uff1a \u5b58\u50a8\u7c7b\u578b \u63cf\u8ff0 CLOUD_STD \u6807\u51c6\u5b58\u50a8\uff0c\u9ed8\u8ba4\u7684\u5b58\u50a8\u7c7b\u578b\uff0c\u70ed\u6570\u636e\uff0c\u5b58\u50a8\u8d39\u7528\u6700\u9ad8 CLOUD_IA \u4f4e\u9891\u5b58\u50a8\uff0c\u504f\u51b7\u7684\u6570\u636e\uff0c\u53ef\u76f4\u63a5\u8bbf\u95ee\uff0c\u5b58\u50a8\u8d39\u7528\u8f83\u4f4e\uff0c\u8bbf\u95ee\u8d39\u7528\u8f83\u9ad8 CLOUD_AR \u5f52\u6863\u5b58\u50a8\uff0c\u51b7\u6570\u636e\uff0c\u53ea\u6709\u4e34\u65f6\u89e3\u51bb\u540e\u624d\u80fd\u8bbf\u95ee\uff0c\u5b58\u50a8\u8d39\u7528\u4f4e CLOUD_COLD_AR \u51b7\u5f52\u6863\u5b58\u50a8\uff0c\u6bd4\u5f52\u6863\u66f4\u51b7\u7684\u7c7b\u578b\uff0c\u53ea\u6709\u4e34\u65f6\u89e3\u51bb\u540e\u624d\u80fd\u8bbf\u95ee\uff0c\u5b58\u50a8\u8d39\u7528\u5f88\u4f4e CLOUD_AR_RESTORED \u5f52\u6863\u5b58\u50a8\u4e34\u65f6\u89e3\u51bb\u540e\u7684\u7c7b\u578b\uff0c\u6709\u8fc7\u671f\u65f6\u95f4 CLOUD_COLD_AR_RESTORED \u51b7\u5f52\u6863\u5b58\u50a8\u4e34\u65f6\u89e3\u51bb\u540e\u7684\u7c7b\u578b\uff0c\u6709\u8fc7\u671f\u65f6\u95f4 \u6570\u636e\u5f52\u6863 \u00a7 \u5bf9\u4e8e\u539f\u672c\u5b58\u50a8\u7c7b\u578b\u4e3a \u201c\u6807\u51c6\u201d \u6216 \u201c\u4f4e\u9891\u201d \u7684\u6570\u636e\uff0c\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0b\u547d\u4ee4\u89e6\u53d1\u5bf9\u6570\u636e\u7684\u5f52\u6863\u6307\u4ee4\uff1a jindofs fs -setStoragePolicy -path <path> -policy <policy> \u5176\u4e2d\uff1a * <path> \u53ef\u4ee5\u662f\u6587\u4ef6\u6216\u76ee\u5f55\uff0c\u4e0e\u5f00\u6e90 HDFS \u7684 setStoragePolicy \u529f\u80fd\u4e00\u81f4\u3002 * <policy> \u4e3a\u76ee\u6807\u5b58\u50a8\u7c7b\u578b\uff0c CLOUD_AR \u4e3a \u201c\u5f52\u6863\u201d\uff0c CLOUD_COLD_AR \u4e3a \u201c\u51b7\u5f52\u6863\u201d\u3002 \u5982\u679c\u547d\u4ee4\u6b63\u786e\u6267\u884c\uff0c\u5c06\u8fd4\u56de\u5982\u4e0b\u4fe1\u606f\uff1a Successfully Set StoragePolicy for <path> with policy: <policy> \u68c0\u67e5\u5f52\u6863\u4efb\u52a1 \u00a7 \u4e0a\u4e00\u5219\u547d\u4ee4\u89e6\u53d1\u4e86\u6570\u636e\u5f52\u6863\u7684\u540e\u53f0\u4efb\u52a1\u3002\u53ef\u4ee5\u901a\u8fc7\u4e0b\u5217\u6307\u4ee4\u68c0\u67e5\u8fd9\u4e2a\u4efb\u52a1\u662f\u5426\u5b8c\u6210\uff1a jindofs fs -checkStoragePolicy -path <path> \u8fd4\u56de\u7684\u5185\u5bb9\u793a\u4f8b\u4e3a\uff1a The status storage policy set/unset job for <path> is FINALIZED \u5176\u4e2d FINALIZED \u5373\u4e3a \u201c\u5df2\u5b8c\u6210\u201d\u3002\u5176\u4ed6\u53ef\u80fd\u7684\u72b6\u6001\u8fd8\u6709 PENDING \u3001 PROCESSING \u3001 SUBMITTED \uff0c\u5747\u4e3a\u672a\u5b8c\u6210\u72b6\u6001\uff0c\u9700\u7ee7\u7eed\u7b49\u5f85\u3002 \u6570\u636e\u89e3\u51bb \u00a7 \u5bf9\u4e8e\u5f52\u6863\u7684\u6570\u636e\uff0c\u5982\u679c\u9700\u8981\u8bbf\u95ee\uff0c\u5fc5\u987b\u5148\u8fdb\u884c\u89e3\u51bb\u3002\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0b\u547d\u4ee4\u89e6\u53d1\u5bf9\u6570\u636e\u7684\u4e34\u65f6\u89e3\u51bb\uff1a jindofs fs -setStoragePolicy -path <path> -policy <policy> -restoreDays <restoreDays> \u5176\u4e2d\uff1a * \u5982\u679c\u6570\u636e\u539f\u672c\u4e3a\u5f52\u6863\uff08 CLOUD_AR \uff09\uff0c\u90a3\u4e48 <policy> \u5fc5\u987b\u4e3a CLOUD_AR_RESTORED \uff1b\u5982\u679c\u539f\u672c\u4e3a\u51b7\u5f52\u6863\uff08 CLOUD_COLD_AR \uff09\uff0c\u5219 <policy> \u5fc5\u987b\u4e3a CLOUD_COLD_AR_RESTORED \u3002 * <restoreDays> \u6307\u5b9a\u89e3\u51bb\u7684\u5929\u6570\u3002\u5f52\u6863\u652f\u6301 1 - 7 \u5929\uff0c\u51b7\u5f52\u6863\u652f\u6301 1 - 365 \u5929\u3002\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4 1 \u5929\u3002 * \u53ef\u4ee5\u7528 jindofs fs -checkStoragePolicy -path <path> \u67e5\u770b\u4e34\u65f6\u89e3\u51bb\u4efb\u52a1\u662f\u5426\u5b8c\u6210\u3002 * \u6570\u636e\u4ece\u5f52\u6863\uff08 CLOUD_AR \uff09\u6216\u8005\u51b7\u5f52\u6863( CLOUD_COLD_AR ) \u6062\u590d\u6210\u4f4e\u9891\uff08 CLOUD_IA \uff09\u6216\u8005\u6807\u51c6 ( CLOUD_STD ) \u5b58\u5728\u9650\u5236\uff0c\u5355\u6b21\u63d0\u4ea4\u6570\u636e\u91cf\u4e0d\u80fd\u8d85\u8fc75TB\uff0c \u540c\u65f6\u5904\u4e8e\u6267\u884c\u72b6\u6001\u7684\u6570\u636e\u91cf\u4e0d\u80fd\u8d85\u8fc750TB\u3002 \u4e34\u65f6\u89e3\u51bb\u6709\u4e0b\u5217\u6ce8\u610f\u4e8b\u9879\uff1a * \u7528 CLOUD_AR \u6216\u8005 CLOUD_COLD_AR \u7684\u7b56\u7565\u8fdb\u884c\u5f52\u6863\u4e4b\u540e\uff0c\u5fc5\u987b\u95f4\u9694\u8d85\u8fc7 2 \u5929\u624d\u53ef\u8fdb\u884c\u89e3\u51bb\u3002 * \u4e34\u65f6\u89e3\u51bb\u4efb\u52a1\u5b8c\u6210\u540e\uff0c\u6570\u636e\u4ecd\u4e0d\u80fd\u7acb\u523b\u53ef\u8bfb\u3002\u901a\u5e38\u5f52\u6863\u7c7b\u578b\u9700\u8981\u6570\u5206\u949f\u540e\u53ef\u8bfb\uff0c\u51b7\u5f52\u6863\u7c7b\u578b\u5219\u9700\u8981\u6570\u5c0f\u65f6\u3002 * \u4e34\u65f6\u89e3\u51bb\u6709\u5929\u6570\u9650\u5236\uff0c\u7531 <restoreDays> \u6307\u5b9a\uff0c\u8d85\u51fa\u5219\u56de\u5230\u4e0d\u53ef\u8bfb\u72b6\u6001\u3002 * \u5904\u4e8e\u4e34\u65f6\u89e3\u51bb\u72b6\u6001\u65f6\uff0c\u4ecd\u53ef\u4ee5\u518d\u6b21\u8fdb\u884c\u89e3\u51bb\uff0c\u4f46\u662f\u95f4\u9694\u5fc5\u987b\u8d85\u8fc7 2 \u5929\u4ee5\u4e0a\u3002","title":"\u4f7f\u7528 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u5b9e\u73b0\u51b7\u70ed\u5206\u5c42\u5b58\u50a8"},{"location":"jindofs/jindofs_storagepolicy_restore/#jindofs","text":"","title":"\u4f7f\u7528 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u5b9e\u73b0\u51b7\u70ed\u5206\u5c42\u5b58\u50a8"},{"location":"jindofs/jindofs_storagepolicy_restore/#_1","text":"JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u662f\u8bbf\u95ee OSS-HDFS \u7684\u53ef\u6267\u884c\u7a0b\u5e8f\uff0c\u652f\u6301\u5e38\u89c1\u6587\u4ef6\u5143\u6570\u636e\u53ca\u8bfb\u5199\u64cd\u4f5c\uff0c\u4ee5\u53ca\u670d\u52a1\u7279\u6709\u65b9\u6cd5\u3002\u66f4\u591a\u4ecb\u7ecd\uff0c\u4ee5\u53ca\u5de5\u5177\u7684\u83b7\u53d6\u548c\u914d\u7f6e\u65b9\u6cd5\uff0c\u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 OSS-HDFS \u670d\u52a1\u7684\u4ecb\u7ecd\uff0c\u53c2\u89c1 OSS-HDFS\u670d\u52a1\u6982\u8ff0 \u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u4f7f\u7528 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u5b9e\u73b0 OSS-HDFS \u670d\u52a1\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8\u529f\u80fd\uff0c\u4ee5\u52a9\u4e8e\u66f4\u597d\u5730\u6cbb\u7406\u6570\u636e\uff0c\u7ba1\u7406\u6210\u672c\u3002","title":"\u6982\u8ff0"},{"location":"jindofs/jindofs_storagepolicy_restore/#_2","text":"OSS-HDFS \u670d\u52a1\u76ee\u524d\u652f\u6301\u516d\u79cd\u5b58\u50a8\u7c7b\u578b\uff0c\u4f9d\u6b21\u4e3a\uff1a \u5b58\u50a8\u7c7b\u578b \u63cf\u8ff0 CLOUD_STD \u6807\u51c6\u5b58\u50a8\uff0c\u9ed8\u8ba4\u7684\u5b58\u50a8\u7c7b\u578b\uff0c\u70ed\u6570\u636e\uff0c\u5b58\u50a8\u8d39\u7528\u6700\u9ad8 CLOUD_IA \u4f4e\u9891\u5b58\u50a8\uff0c\u504f\u51b7\u7684\u6570\u636e\uff0c\u53ef\u76f4\u63a5\u8bbf\u95ee\uff0c\u5b58\u50a8\u8d39\u7528\u8f83\u4f4e\uff0c\u8bbf\u95ee\u8d39\u7528\u8f83\u9ad8 CLOUD_AR \u5f52\u6863\u5b58\u50a8\uff0c\u51b7\u6570\u636e\uff0c\u53ea\u6709\u4e34\u65f6\u89e3\u51bb\u540e\u624d\u80fd\u8bbf\u95ee\uff0c\u5b58\u50a8\u8d39\u7528\u4f4e CLOUD_COLD_AR \u51b7\u5f52\u6863\u5b58\u50a8\uff0c\u6bd4\u5f52\u6863\u66f4\u51b7\u7684\u7c7b\u578b\uff0c\u53ea\u6709\u4e34\u65f6\u89e3\u51bb\u540e\u624d\u80fd\u8bbf\u95ee\uff0c\u5b58\u50a8\u8d39\u7528\u5f88\u4f4e CLOUD_AR_RESTORED \u5f52\u6863\u5b58\u50a8\u4e34\u65f6\u89e3\u51bb\u540e\u7684\u7c7b\u578b\uff0c\u6709\u8fc7\u671f\u65f6\u95f4 CLOUD_COLD_AR_RESTORED \u51b7\u5f52\u6863\u5b58\u50a8\u4e34\u65f6\u89e3\u51bb\u540e\u7684\u7c7b\u578b\uff0c\u6709\u8fc7\u671f\u65f6\u95f4","title":"\u5b58\u50a8\u7c7b\u578b"},{"location":"jindofs/jindofs_storagepolicy_restore/#_3","text":"\u5bf9\u4e8e\u539f\u672c\u5b58\u50a8\u7c7b\u578b\u4e3a \u201c\u6807\u51c6\u201d \u6216 \u201c\u4f4e\u9891\u201d \u7684\u6570\u636e\uff0c\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0b\u547d\u4ee4\u89e6\u53d1\u5bf9\u6570\u636e\u7684\u5f52\u6863\u6307\u4ee4\uff1a jindofs fs -setStoragePolicy -path <path> -policy <policy> \u5176\u4e2d\uff1a * <path> \u53ef\u4ee5\u662f\u6587\u4ef6\u6216\u76ee\u5f55\uff0c\u4e0e\u5f00\u6e90 HDFS \u7684 setStoragePolicy \u529f\u80fd\u4e00\u81f4\u3002 * <policy> \u4e3a\u76ee\u6807\u5b58\u50a8\u7c7b\u578b\uff0c CLOUD_AR \u4e3a \u201c\u5f52\u6863\u201d\uff0c CLOUD_COLD_AR \u4e3a \u201c\u51b7\u5f52\u6863\u201d\u3002 \u5982\u679c\u547d\u4ee4\u6b63\u786e\u6267\u884c\uff0c\u5c06\u8fd4\u56de\u5982\u4e0b\u4fe1\u606f\uff1a Successfully Set StoragePolicy for <path> with policy: <policy>","title":"\u6570\u636e\u5f52\u6863"},{"location":"jindofs/jindofs_storagepolicy_restore/#_4","text":"\u4e0a\u4e00\u5219\u547d\u4ee4\u89e6\u53d1\u4e86\u6570\u636e\u5f52\u6863\u7684\u540e\u53f0\u4efb\u52a1\u3002\u53ef\u4ee5\u901a\u8fc7\u4e0b\u5217\u6307\u4ee4\u68c0\u67e5\u8fd9\u4e2a\u4efb\u52a1\u662f\u5426\u5b8c\u6210\uff1a jindofs fs -checkStoragePolicy -path <path> \u8fd4\u56de\u7684\u5185\u5bb9\u793a\u4f8b\u4e3a\uff1a The status storage policy set/unset job for <path> is FINALIZED \u5176\u4e2d FINALIZED \u5373\u4e3a \u201c\u5df2\u5b8c\u6210\u201d\u3002\u5176\u4ed6\u53ef\u80fd\u7684\u72b6\u6001\u8fd8\u6709 PENDING \u3001 PROCESSING \u3001 SUBMITTED \uff0c\u5747\u4e3a\u672a\u5b8c\u6210\u72b6\u6001\uff0c\u9700\u7ee7\u7eed\u7b49\u5f85\u3002","title":"\u68c0\u67e5\u5f52\u6863\u4efb\u52a1"},{"location":"jindofs/jindofs_storagepolicy_restore/#_5","text":"\u5bf9\u4e8e\u5f52\u6863\u7684\u6570\u636e\uff0c\u5982\u679c\u9700\u8981\u8bbf\u95ee\uff0c\u5fc5\u987b\u5148\u8fdb\u884c\u89e3\u51bb\u3002\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0b\u547d\u4ee4\u89e6\u53d1\u5bf9\u6570\u636e\u7684\u4e34\u65f6\u89e3\u51bb\uff1a jindofs fs -setStoragePolicy -path <path> -policy <policy> -restoreDays <restoreDays> \u5176\u4e2d\uff1a * \u5982\u679c\u6570\u636e\u539f\u672c\u4e3a\u5f52\u6863\uff08 CLOUD_AR \uff09\uff0c\u90a3\u4e48 <policy> \u5fc5\u987b\u4e3a CLOUD_AR_RESTORED \uff1b\u5982\u679c\u539f\u672c\u4e3a\u51b7\u5f52\u6863\uff08 CLOUD_COLD_AR \uff09\uff0c\u5219 <policy> \u5fc5\u987b\u4e3a CLOUD_COLD_AR_RESTORED \u3002 * <restoreDays> \u6307\u5b9a\u89e3\u51bb\u7684\u5929\u6570\u3002\u5f52\u6863\u652f\u6301 1 - 7 \u5929\uff0c\u51b7\u5f52\u6863\u652f\u6301 1 - 365 \u5929\u3002\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4 1 \u5929\u3002 * \u53ef\u4ee5\u7528 jindofs fs -checkStoragePolicy -path <path> \u67e5\u770b\u4e34\u65f6\u89e3\u51bb\u4efb\u52a1\u662f\u5426\u5b8c\u6210\u3002 * \u6570\u636e\u4ece\u5f52\u6863\uff08 CLOUD_AR \uff09\u6216\u8005\u51b7\u5f52\u6863( CLOUD_COLD_AR ) \u6062\u590d\u6210\u4f4e\u9891\uff08 CLOUD_IA \uff09\u6216\u8005\u6807\u51c6 ( CLOUD_STD ) \u5b58\u5728\u9650\u5236\uff0c\u5355\u6b21\u63d0\u4ea4\u6570\u636e\u91cf\u4e0d\u80fd\u8d85\u8fc75TB\uff0c \u540c\u65f6\u5904\u4e8e\u6267\u884c\u72b6\u6001\u7684\u6570\u636e\u91cf\u4e0d\u80fd\u8d85\u8fc750TB\u3002 \u4e34\u65f6\u89e3\u51bb\u6709\u4e0b\u5217\u6ce8\u610f\u4e8b\u9879\uff1a * \u7528 CLOUD_AR \u6216\u8005 CLOUD_COLD_AR \u7684\u7b56\u7565\u8fdb\u884c\u5f52\u6863\u4e4b\u540e\uff0c\u5fc5\u987b\u95f4\u9694\u8d85\u8fc7 2 \u5929\u624d\u53ef\u8fdb\u884c\u89e3\u51bb\u3002 * \u4e34\u65f6\u89e3\u51bb\u4efb\u52a1\u5b8c\u6210\u540e\uff0c\u6570\u636e\u4ecd\u4e0d\u80fd\u7acb\u523b\u53ef\u8bfb\u3002\u901a\u5e38\u5f52\u6863\u7c7b\u578b\u9700\u8981\u6570\u5206\u949f\u540e\u53ef\u8bfb\uff0c\u51b7\u5f52\u6863\u7c7b\u578b\u5219\u9700\u8981\u6570\u5c0f\u65f6\u3002 * \u4e34\u65f6\u89e3\u51bb\u6709\u5929\u6570\u9650\u5236\uff0c\u7531 <restoreDays> \u6307\u5b9a\uff0c\u8d85\u51fa\u5219\u56de\u5230\u4e0d\u53ef\u8bfb\u72b6\u6001\u3002 * \u5904\u4e8e\u4e34\u65f6\u89e3\u51bb\u72b6\u6001\u65f6\uff0c\u4ecd\u53ef\u4ee5\u518d\u6b21\u8fdb\u884c\u89e3\u51bb\uff0c\u4f46\u662f\u95f4\u9694\u5fc5\u987b\u8d85\u8fc7 2 \u5929\u4ee5\u4e0a\u3002","title":"\u6570\u636e\u89e3\u51bb"},{"location":"jindofs/jindofs_trash_quickstart/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u56de\u6536\u7ad9\u4f7f\u7528\u8bf4\u660e \u00a7 (\u4ece JindoData 4.5.1 \u5f00\u59cb\u652f\u6301) \u4ecb\u7ecd \u00a7 \u5f53\u4ece OSS-HDFS \u670d\u52a1\u5220\u9664\u6587\u4ef6\u65f6\uff0c\u6587\u4ef6\u4e0d\u662f\u7acb\u5373\u88ab\u7acb\u5373\u5220\u9664\u3002\u88ab\u5220\u9664\u7684\u6587\u4ef6\u88ab\u632a\u5230\u4e86 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\u3002\u5f53\u7ecf\u8fc730\u5206\u949f\u540e\uff0cCurrent\u76ee\u5f55\u4f1a\u88ab\u632a\u5230 ```bash /user/ /.Trash/ \u4e5f\u5c31\u662f\u8bf4\u67d0\u4e00\u65f6\u95f4\u6bb5\u5185\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u4f1a\u88ab\u5f52\u7c7b\u5230\u4e00\u4e2a\u5e26\u65f6\u95f4\u6233\u7684\u76ee\u5f55\u4e0b\uff0c\u8868\u793a\u662f\u5728\u8fd9\u4e2a\u65f6\u95f4\u5185\u88ab\u5220\u9664\u7684\uff0c\u76f8\u5f53\u4e8e\u4e00\u6b21checkpoint\u3002\u5f53\u7ecf\u8fc7**7\u5929**\u540e\uff0c\u8fd9\u4e2a<timestamp>\u76ee\u5f55\u5c06\u4f1a\u88ab\u6c38\u8fdc\u5220\u9664\u3002\u56e0\u6b64\u57287\u5929\u5185\uff0c\u60a8\u6709\u673a\u4f1a\u4ece.Trash\u76ee\u5f55\u4e0b\u627e\u5230\u5bf9\u5e94\u65f6\u523b\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u5c06\u5176\u4ece.Trash\u632a\u51fa\uff0c\u4ece\u800c\u6062\u590d\u5b83\u3002 \u6ce8\u610f\uff1a\u8fd9\u9879\u529f\u80fd\uff0c\u662f\u7531\u5ba2\u6237\u7aef\u548c\u670d\u52a1\u7aef\u914d\u5408\uff0c\u65b9\u80fd\u5f62\u6210\u56de\u6536\u7ad9\u7684\u529f\u80fd\u3002\u670d\u52a1\u7aef\u53ea\u8d1f\u8d23\u7ef4\u62a4 ```/user/<username>/.Trash``` \u7684\u5b9a\u65f6\u6e05\u7406\uff0c\u670d\u52a1\u7aef\u7684\u5b9a\u65f6\u6e05\u7406\u662f\u9ed8\u8ba4\u5f00\u542f\u7684\u3002\u5ba2\u6237\u7aef\u8d1f\u8d23\u5c06\u5f85\u5220\u9664\u6587\u4ef6\u632a\u5230.Trash\u76ee\u5f55\u4e0b\u3002 ## Hadoop FileSystem Shell \u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd ```bash hadoop fs -rm oss://bucket/a/b/c \u5ba2\u6237\u7aef\u7684 Hadoop Shell \u547d\u4ee4\u9ed8\u8ba4\u4e0d\u5f00\u542fTrash\u529f\u80fd\uff0c\u56e0\u6b64\u9700\u8981\u5728 core-site.xml \u91cc\u6dfb\u52a0\u4e00\u6761\u914d\u7f6e <property> <name>fs.trash.interval</name> <value>1440</value> </property> (\u8be5\u503c\u53ea\u9700\u5927\u4e8e0\u5373\u53ef) \u6b64\u65f6\u5728\u5ba2\u6237\u7aef\u4f1a\u81ea\u52a8\u5c06rm\u547d\u4ee4\u8f6c\u6362\u4e3a\u4e00\u6761 hadoop fs -mv oss://bucket/a/b/c /user/<username>/.Trash/Current/a/b/c \u547d\u4ee4\uff0c\u56e0\u6b64\uff0c\u60a8\u4e0d\u9700\u8981\u611f\u77e5\u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u670d\u52a1\u7aef\u4f1a\u8d1f\u8d23\u6e05\u7406\u3002 \u5982\u679c\u60a8\u60f3\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\uff0c\u91ca\u653e\u7a7a\u95f4\uff0c\u53ef\u4ee5\u6dfb\u52a0 -skipTrash \u53c2\u6570\uff0c\u6b64\u65f6\u5c06\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\u3002 Hadoop \u751f\u6001\u7ec4\u4ef6\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd \u00a7 Hive/Spark/Flink \u7b49\u7ec4\u4ef6\u5e76\u4e0d\u611f\u77e5 OSS-HDFS \u670d\u52a1\u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u4f7f\u7528 FileSystem(HCFS) \u7684 delete \u63a5\u53e3\u610f\u5473\u7740\u7acb\u5373\u5220\u9664\u3002\u56e0\u6b64\u5982\u679c\u60f3\u8981\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd\uff0c\u9700\u8981\u663e\u5f0f\u5730\u8c03\u7528 FileSystem \u7684 rename \u63a5\u53e3\uff0c\u5c06\u76ee\u6807\u6587\u4ef6\u624b\u52a8\u632a\u5230 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\uff0c\u7531 OSS-HDFS \u670d\u52a1\u7aef\u8d1f\u8d23\u5b9a\u671f\u5220\u9664\u3002\u8fd9\u4e00\u70b9\u4e0a OSS-HDFS \u91c7\u53d6\u4e86\u8ddf\u5f00\u6e90 Hadoop \u76f8\u4f3c\u7684\u7b56\u7565\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u56de\u6536\u7ad9\u4f7f\u7528\u8bf4\u660e"},{"location":"jindofs/jindofs_trash_quickstart/#oss-hdfs-jindofs","text":"(\u4ece JindoData 4.5.1 \u5f00\u59cb\u652f\u6301)","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u56de\u6536\u7ad9\u4f7f\u7528\u8bf4\u660e"},{"location":"jindofs/jindofs_trash_quickstart/#_1","text":"\u5f53\u4ece OSS-HDFS \u670d\u52a1\u5220\u9664\u6587\u4ef6\u65f6\uff0c\u6587\u4ef6\u4e0d\u662f\u7acb\u5373\u88ab\u7acb\u5373\u5220\u9664\u3002\u88ab\u5220\u9664\u7684\u6587\u4ef6\u88ab\u632a\u5230\u4e86 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\u3002\u5f53\u7ecf\u8fc730\u5206\u949f\u540e\uff0cCurrent\u76ee\u5f55\u4f1a\u88ab\u632a\u5230 ```bash /user/ /.Trash/ \u4e5f\u5c31\u662f\u8bf4\u67d0\u4e00\u65f6\u95f4\u6bb5\u5185\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u4f1a\u88ab\u5f52\u7c7b\u5230\u4e00\u4e2a\u5e26\u65f6\u95f4\u6233\u7684\u76ee\u5f55\u4e0b\uff0c\u8868\u793a\u662f\u5728\u8fd9\u4e2a\u65f6\u95f4\u5185\u88ab\u5220\u9664\u7684\uff0c\u76f8\u5f53\u4e8e\u4e00\u6b21checkpoint\u3002\u5f53\u7ecf\u8fc7**7\u5929**\u540e\uff0c\u8fd9\u4e2a<timestamp>\u76ee\u5f55\u5c06\u4f1a\u88ab\u6c38\u8fdc\u5220\u9664\u3002\u56e0\u6b64\u57287\u5929\u5185\uff0c\u60a8\u6709\u673a\u4f1a\u4ece.Trash\u76ee\u5f55\u4e0b\u627e\u5230\u5bf9\u5e94\u65f6\u523b\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u5c06\u5176\u4ece.Trash\u632a\u51fa\uff0c\u4ece\u800c\u6062\u590d\u5b83\u3002 \u6ce8\u610f\uff1a\u8fd9\u9879\u529f\u80fd\uff0c\u662f\u7531\u5ba2\u6237\u7aef\u548c\u670d\u52a1\u7aef\u914d\u5408\uff0c\u65b9\u80fd\u5f62\u6210\u56de\u6536\u7ad9\u7684\u529f\u80fd\u3002\u670d\u52a1\u7aef\u53ea\u8d1f\u8d23\u7ef4\u62a4 ```/user/<username>/.Trash``` \u7684\u5b9a\u65f6\u6e05\u7406\uff0c\u670d\u52a1\u7aef\u7684\u5b9a\u65f6\u6e05\u7406\u662f\u9ed8\u8ba4\u5f00\u542f\u7684\u3002\u5ba2\u6237\u7aef\u8d1f\u8d23\u5c06\u5f85\u5220\u9664\u6587\u4ef6\u632a\u5230.Trash\u76ee\u5f55\u4e0b\u3002 ## Hadoop FileSystem Shell \u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd ```bash hadoop fs -rm oss://bucket/a/b/c \u5ba2\u6237\u7aef\u7684 Hadoop Shell \u547d\u4ee4\u9ed8\u8ba4\u4e0d\u5f00\u542fTrash\u529f\u80fd\uff0c\u56e0\u6b64\u9700\u8981\u5728 core-site.xml \u91cc\u6dfb\u52a0\u4e00\u6761\u914d\u7f6e <property> <name>fs.trash.interval</name> <value>1440</value> </property> (\u8be5\u503c\u53ea\u9700\u5927\u4e8e0\u5373\u53ef) \u6b64\u65f6\u5728\u5ba2\u6237\u7aef\u4f1a\u81ea\u52a8\u5c06rm\u547d\u4ee4\u8f6c\u6362\u4e3a\u4e00\u6761 hadoop fs -mv oss://bucket/a/b/c /user/<username>/.Trash/Current/a/b/c \u547d\u4ee4\uff0c\u56e0\u6b64\uff0c\u60a8\u4e0d\u9700\u8981\u611f\u77e5\u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u670d\u52a1\u7aef\u4f1a\u8d1f\u8d23\u6e05\u7406\u3002 \u5982\u679c\u60a8\u60f3\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\uff0c\u91ca\u653e\u7a7a\u95f4\uff0c\u53ef\u4ee5\u6dfb\u52a0 -skipTrash \u53c2\u6570\uff0c\u6b64\u65f6\u5c06\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\u3002","title":"\u4ecb\u7ecd"},{"location":"jindofs/jindofs_trash_quickstart/#hadoop","text":"Hive/Spark/Flink \u7b49\u7ec4\u4ef6\u5e76\u4e0d\u611f\u77e5 OSS-HDFS \u670d\u52a1\u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u4f7f\u7528 FileSystem(HCFS) \u7684 delete \u63a5\u53e3\u610f\u5473\u7740\u7acb\u5373\u5220\u9664\u3002\u56e0\u6b64\u5982\u679c\u60f3\u8981\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd\uff0c\u9700\u8981\u663e\u5f0f\u5730\u8c03\u7528 FileSystem \u7684 rename \u63a5\u53e3\uff0c\u5c06\u76ee\u6807\u6587\u4ef6\u624b\u52a8\u632a\u5230 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\uff0c\u7531 OSS-HDFS \u670d\u52a1\u7aef\u8d1f\u8d23\u5b9a\u671f\u5220\u9664\u3002\u8fd9\u4e00\u70b9\u4e0a OSS-HDFS \u91c7\u53d6\u4e86\u8ddf\u5f00\u6e90 Hadoop \u76f8\u4f3c\u7684\u7b56\u7565\u3002","title":"Hadoop \u751f\u6001\u7ec4\u4ef6\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd"},{"location":"jindofs/jindotable_storagepolicy_restore/","text":"\u4f7f\u7528 JindoTable \u5de5\u5177\u5b9e\u73b0 OSS-HDFS \u7ed3\u6784\u5316\u6570\u636e\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8 \u00a7 \u6982\u8ff0 \u00a7 JindoTable \u662f\u57fa\u4e8e JindoSDK \u7684\u7ed3\u6784\u5316\u6570\u636e\u6cbb\u7406\u5de5\u5177\uff0c\u76ee\u6807\u662f\u5728 \u201c\u8868\u201d \u6216\u8005 \u201c\u5206\u533a\u201d \u5c42\u9762\u63d0\u4f9b\u4fbf\u6377\u7684\u6570\u636e\u8bbf\u95ee\u548c\u64cd\u4f5c\u3002\u4f7f\u7528 JindoTable \u5de5\u5177\u53ef\u4ee5\u65b9\u4fbf\u5730\u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u548c\u5206\u5c42\u5b58\u50a8\u3002 \u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u4f7f\u7528 JindoTable \u5de5\u5177\u5bf9\u5b58\u50a8\u5728 OSS-HDFS \u4e0a\u7684\u7ed3\u6784\u5316\u6570\u636e\u8fdb\u884c\u51b7\u70ed\u5206\u5c42\u5b58\u50a8\u3002 \u83b7\u53d6 \u00a7 \u53ef\u4ee5\u4ece JindoData \u4e0b\u8f7d\u9875\u9762 \u83b7\u53d6 JindoTable \u5de5\u5177\uff1a 1. \u4e0b\u8f7d jindosdk-<version>-<platform>.tar.gz \uff0c\u6839\u636e\u7cfb\u7edf\u9009\u62e9\u5408\u9002\u7684 <platform> \u3002 2. \u672c\u6587\u4ecb\u7ecd\u7684\u529f\u80fd\u6700\u4f4e\u7248\u672c\u8981\u6c42\u4e3a <version> 6.1.2\u3002 \u6ce8\uff1a - JindoTable \u5de5\u5177\u5bf9 OSS-HDFS \u7684\u4e34\u65f6\u89e3\u51bb\uff08 -once \uff09\u652f\u6301\u8bbe\u7f6e\u89e3\u51bb\u53c2\u6570\uff08 -d/-restoreDays <restore days> \uff09\uff0c\u7248\u672c\u8981\u6c42\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0\u3002\u82e5\u5bf9\u8be5\u529f\u80fd\u6709\u9700\u8981\uff0c\u8bf7\u4f7f\u7528\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0 \u7684\u7248\u672c\u3002 \u57fa\u672c\u7528\u6cd5 \u00a7 \u6309\u7167\u4e0a\u9762\u7684\u6b65\u9aa4\u83b7\u53d6\u538b\u7f29\u5305\u540e\uff0c\u89e3\u538b\u7f29\uff0c\u4fbf\u53ef\u5728 bin/ \u76ee\u5f55\u4e0b\u627e\u5230\u540d\u4e3a jindotable \u7684\u811a\u672c\u3002\u6267\u884c\u811a\u672c\uff0c\u5373\u53ef\u4f7f\u7528 JindoTable \u63d0\u4f9b\u7684\u5404\u79cd\u547d\u4ee4\u3002 \u6267\u884c jindotable -usage \u53ef\u4ee5\u663e\u793a\u5de5\u5177\u652f\u6301\u7684\u5404\u79cd\u547d\u4ee4\u3002 \u4e00\u4e9b\u6ce8\u610f\u4e8b\u9879\uff1a - \u6267\u884c\u65f6\uff0c\u987b\u4fdd\u6301\u89e3\u538b\u7f29\u540e\u7684\u76ee\u5f55\u7ed3\u6784\u3002 - \u9700\u8981\u96c6\u7fa4\u4e0a\u6709 Hadoop \u548c Hive \u73af\u5883\uff0c\u76f8\u5173\u7684\u73af\u5883\u53d8\u91cf\u5df2\u7ecf\u88ab\u6b63\u786e\u914d\u7f6e\u3002 \u5206\u5c42\u5b58\u50a8\u547d\u4ee4 \u00a7 \u652f\u6301 OSS-HDFS \u7ed3\u6784\u5316\u6570\u636e\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8\u529f\u80fd\uff0c\u547d\u4ee4\u4e3a -setStoragePolicy \uff0c\u6267\u884c jindotable -help setStoragePolicy \u53ef\u4ee5\u663e\u793a\u5e2e\u52a9\u3002 \u53c2\u6570\u8bf4\u660e \u00a7 jindotable -setStoragePolicy -t <dbName.tableName> -std/-ia/-archive/-coldArchive/-once [-c \"<condition>\" | -fullTable] [-d/-restoreDays <restore days>] [-b/-before <before days>] [-p/-parallel <parallelism>] [-mr/-mapReduce] [-e/-explain] [-w/-workingDir <working directory>] [-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u76ee\u6807\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a\u6570\u636e\u5e93\u540d.\u8868\u540d\u3002\u6570\u636e\u5e93\u548c\u8868\u540d\u4e4b\u95f4\u4ee5\u534a\u89d2\u53e5\u53f7\uff08.\uff09\u5206\u9694\u3002\u8868\u53ef\u4ee5\u662f\u5206\u533a\u8868\u6216\u975e\u5206\u533a\u8868\u3002 \u662f -std/-ia/-archive/-coldArchive/-once \u76ee\u6807\u5b58\u50a8\u65b9\u5f0f\u3002\u652f\u6301\u5982\u4e0b\u7c7b\u578b\uff1a-std\uff1a\u6807\u51c6\u5b58\u50a8\uff0c-ia\uff1a\u4f4e\u9891\u5b58\u50a8\uff0c-archive\uff1a\u5f52\u6863\u5b58\u50a8\uff0c-coldArchive\uff1a\u51b7\u5f52\u6863\u5b58\u50a8\uff0c-once\uff1a\u4e34\u65f6\u89e3\u51bb\u3002 \u662f -c \"<condition>\" / -fullTable -fullTable \u548c -c \"<condition>\" \u53ea\u9700\u63d0\u4f9b\u4e00\u4e2a\uff0c\u5373\u8981\u4e48\u6307\u5b9a -c \"<condition>\" \uff0c\u8981\u4e48\u6307\u5b9a -fullTable \u3002\u6307\u5b9a -fullTable \u65f6\uff0c\u5219\u4e3a\u79fb\u52a8\u6574\u8868\uff0c\u65e2\u53ef\u4ee5\u662f\u975e\u5206\u533a\u8868\u4e5f\u53ef\u4ee5\u662f\u5206\u533a\u8868\u3002\u6307\u5b9a -c \"<condition>\" \u65f6\uff0c\u5219\u63d0\u4f9b\u4e86\u4e00\u4e2a\u8fc7\u6ee4\u6761\u4ef6\uff0c\u7528\u6765\u9009\u62e9\u5e0c\u671b\u79fb\u52a8\u7684\u5206\u533a\uff0c\u652f\u6301\u5e38\u89c1\u8fd0\u7b97\u7b26\uff0c\u4f8b\u5982\u5927\u4e8e\u53f7\uff08>\uff09\u3002\u4f8b\u5982\uff0c\u6570\u636e\u7c7b\u578b\u4e3aString\u7684\u5206\u533ads\uff0c\u5e0c\u671b\u5206\u533a\u540d\u5927\u4e8e 'd'\uff0c\u5219\u4ee3\u7801\u4e3a-c \" ds > 'd' \"\u3002 \u662f -d/-restoreDays <restore days> \u6307\u5b9a\u4e34\u65f6\u89e3\u51bb\u7684\u5929\u6570\uff0c\u4ec5\u5bf9\u4e34\u65f6\u89e3\u51bb -once \u751f\u6548\u3002 \u5426 -b/before <before days> \u53ea\u6709\u521b\u5efa\u65f6\u95f4\u8ddd\u79bb\u73b0\u5728\u8d85\u8fc7\u4e00\u5b9a\u5929\u6570\u7684\u8868\u6216\u5206\u533a\u624d\u4f1a\u88ab\u6267\u884c\u3002 \u5426 -p/-parallel <parallelism> \u547d\u4ee4\u6267\u884c\u7684\u5e76\u884c\u5ea6\u3002 \u5426 -mr/-mapReduce \u4f7f\u7528Hadoop MapReduce\u800c\u975e\u672c\u5730\u591a\u7ebf\u7a0b\u6765\u6267\u884c\u547d\u4ee4\u3002 \u5426 -e/-explain \u5982\u679c\u51fa\u73b0\u8be5\u9009\u9879\uff0c\u5219\u4e3a\u89e3\u91ca\uff08explain\uff09\u6a21\u5f0f\uff0c\u53ea\u4f1a\u663e\u793a\u5f85\u64cd\u4f5c\u7684\u5206\u533a\u5217\u8868\uff0c\u800c\u4e0d\u4f1a\u771f\u6b63\u6267\u884c\u547d\u4ee4\u3002 \u5426 -w/-workingDir <working directory> \u53ea\u5728MapReduce\u4f5c\u4e1a\u65f6\u4f7f\u7528\uff0c\u4e3aMapReduce\u4f5c\u4e1a\u7684\u5de5\u4f5c\u76ee\u5f55\u3002\u5fc5\u987b\u6709\u8bfb\u5199\u6743\u9650\uff0c\u5de5\u4f5c\u76ee\u5f55\u53ef\u4ee5\u975e\u7a7a\uff08\u4f5c\u4e1a\u6267\u884c\u8fc7\u7a0b\u4e2d\u4f1a\u521b\u5efa\u4e34\u65f6\u6587\u4ef6\uff0c\u6267\u884c\u5b8c\u6bd5\u4f1a\u6e05\u7406\u4e34\u65f6\u6587\u4ef6\uff09\u3002 \u5426 -l/-logDir <log directory> \u6307\u5b9aLog\u6587\u4ef6\u7684\u76ee\u5f55\u3002 \u5426 \u6ce8\u610f\u4e8b\u9879 \u00a7 JindoTable \u63d0\u4f9b\u7684\u4e0a\u8ff0\u529f\u80fd\u57fa\u672c\u5bf9\u5e94\u4e8e OSS-HDFS \u7ebf\u4e0a\u670d\u52a1\u7684\u5206\u5c42\u5b58\u50a8\u80fd\u529b\uff0c\u6709\u4e0b\u5217\u4e8b\u9879\u9700\u8981\u6ce8\u610f\uff1a 1. \u4ec5\u652f\u6301 OSS-HDFS \u5b58\u50a8\u7684\u8868\u6216\u8005\u5206\u533a\u3002\u5bf9\u666e\u901a OSS \u6570\u636e\u7684\u5206\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoTable \u63d0\u4f9b\u7684 archiveTable \u4e0e unarchiveTable \u547d\u4ee4\u3002 2. \u6210\u529f\u6267\u884c -std/-ia/-archive/-coldArchive/-once \u6307\u5b9a\u7684\u8f6c\u6362\u5b58\u50a8\u7c7b\u578b\u547d\u4ee4\u540e\uff0c\u5fc5\u987b\u7b49\u5f85 2 \u5929\u624d\u53ef\u518d\u6b21\u8f6c\u6362\u5b58\u50a8\u7c7b\u578b\u3002 3. \u6807\u51c6\u548c\u4f4e\u9891\u7c7b\u578b\u65e0\u6cd5\u4e34\u65f6\u89e3\u51bb\uff08 -once \uff09\u3002\u5f52\u6863\u3001\u51b7\u5f52\u6863\uff0c\u6216\u8005\u5df2\u7ecf\u5904\u4e8e\u89e3\u51bb\u72b6\u6001\u65f6\uff0c\u53ef\u4ee5\u89e3\u51bb\uff0c\u4f46\u5982\u4e0a\u6240\u8ff0\uff0c\u95f4\u9694\u987b\u8d85\u8fc7 2 \u5929\u3002 4. \u4e34\u65f6\u89e3\u51bb\u547d\u4ee4\u8fd4\u56de\u540e\uff0c\u6570\u636e\u4ecd\u4e0d\u80fd\u7acb\u5373\u53ef\u8bfb\uff0c\u9700\u8981\u7b49\u5f85\u8fdc\u7aef\u670d\u52a1\u5b8c\u6210\u6570\u636e\u7684\u89e3\u51bb\u3002\u5bf9\u4e8e\u5f52\u6863\u6570\u636e\uff0c\u901a\u5e38\u9700\u8981\u6570\u5206\u949f\uff1b\u5bf9\u4e8e\u51b7\u5f52\u6863\u6570\u636e\uff0c\u901a\u5e38\u9700\u8981\u6570\u5c0f\u65f6\u3002 5. \u4e34\u65f6\u89e3\u51bb\u6709\u671f\u9650\u9650\u5236\uff0c\u901a\u8fc7 -d/restoreDays <restore days> \u6307\u5b9a\uff0c\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4 1 \u5929\uff0c\u5230\u671f\u540e\u6570\u636e\u5c06\u56de\u5230\u4e0d\u53ef\u8bfb\u72b6\u6001\u3002\u8be5\u529f\u80fd\u8981\u6c42\u7248\u672c\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0\u3002","title":"\u4f7f\u7528 JindoTable \u5de5\u5177\u5b9e\u73b0 OSS-HDFS \u7ed3\u6784\u5316\u6570\u636e\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8"},{"location":"jindofs/jindotable_storagepolicy_restore/#jindotable-oss-hdfs","text":"","title":"\u4f7f\u7528 JindoTable \u5de5\u5177\u5b9e\u73b0 OSS-HDFS \u7ed3\u6784\u5316\u6570\u636e\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8"},{"location":"jindofs/jindotable_storagepolicy_restore/#_1","text":"JindoTable \u662f\u57fa\u4e8e JindoSDK \u7684\u7ed3\u6784\u5316\u6570\u636e\u6cbb\u7406\u5de5\u5177\uff0c\u76ee\u6807\u662f\u5728 \u201c\u8868\u201d \u6216\u8005 \u201c\u5206\u533a\u201d \u5c42\u9762\u63d0\u4f9b\u4fbf\u6377\u7684\u6570\u636e\u8bbf\u95ee\u548c\u64cd\u4f5c\u3002\u4f7f\u7528 JindoTable \u5de5\u5177\u53ef\u4ee5\u65b9\u4fbf\u5730\u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u548c\u5206\u5c42\u5b58\u50a8\u3002 \u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u4f7f\u7528 JindoTable \u5de5\u5177\u5bf9\u5b58\u50a8\u5728 OSS-HDFS \u4e0a\u7684\u7ed3\u6784\u5316\u6570\u636e\u8fdb\u884c\u51b7\u70ed\u5206\u5c42\u5b58\u50a8\u3002","title":"\u6982\u8ff0"},{"location":"jindofs/jindotable_storagepolicy_restore/#_2","text":"\u53ef\u4ee5\u4ece JindoData \u4e0b\u8f7d\u9875\u9762 \u83b7\u53d6 JindoTable \u5de5\u5177\uff1a 1. \u4e0b\u8f7d jindosdk-<version>-<platform>.tar.gz \uff0c\u6839\u636e\u7cfb\u7edf\u9009\u62e9\u5408\u9002\u7684 <platform> \u3002 2. \u672c\u6587\u4ecb\u7ecd\u7684\u529f\u80fd\u6700\u4f4e\u7248\u672c\u8981\u6c42\u4e3a <version> 6.1.2\u3002 \u6ce8\uff1a - JindoTable \u5de5\u5177\u5bf9 OSS-HDFS \u7684\u4e34\u65f6\u89e3\u51bb\uff08 -once \uff09\u652f\u6301\u8bbe\u7f6e\u89e3\u51bb\u53c2\u6570\uff08 -d/-restoreDays <restore days> \uff09\uff0c\u7248\u672c\u8981\u6c42\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0\u3002\u82e5\u5bf9\u8be5\u529f\u80fd\u6709\u9700\u8981\uff0c\u8bf7\u4f7f\u7528\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0 \u7684\u7248\u672c\u3002","title":"\u83b7\u53d6"},{"location":"jindofs/jindotable_storagepolicy_restore/#_3","text":"\u6309\u7167\u4e0a\u9762\u7684\u6b65\u9aa4\u83b7\u53d6\u538b\u7f29\u5305\u540e\uff0c\u89e3\u538b\u7f29\uff0c\u4fbf\u53ef\u5728 bin/ \u76ee\u5f55\u4e0b\u627e\u5230\u540d\u4e3a jindotable \u7684\u811a\u672c\u3002\u6267\u884c\u811a\u672c\uff0c\u5373\u53ef\u4f7f\u7528 JindoTable \u63d0\u4f9b\u7684\u5404\u79cd\u547d\u4ee4\u3002 \u6267\u884c jindotable -usage \u53ef\u4ee5\u663e\u793a\u5de5\u5177\u652f\u6301\u7684\u5404\u79cd\u547d\u4ee4\u3002 \u4e00\u4e9b\u6ce8\u610f\u4e8b\u9879\uff1a - \u6267\u884c\u65f6\uff0c\u987b\u4fdd\u6301\u89e3\u538b\u7f29\u540e\u7684\u76ee\u5f55\u7ed3\u6784\u3002 - \u9700\u8981\u96c6\u7fa4\u4e0a\u6709 Hadoop \u548c Hive \u73af\u5883\uff0c\u76f8\u5173\u7684\u73af\u5883\u53d8\u91cf\u5df2\u7ecf\u88ab\u6b63\u786e\u914d\u7f6e\u3002","title":"\u57fa\u672c\u7528\u6cd5"},{"location":"jindofs/jindotable_storagepolicy_restore/#_4","text":"\u652f\u6301 OSS-HDFS \u7ed3\u6784\u5316\u6570\u636e\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8\u529f\u80fd\uff0c\u547d\u4ee4\u4e3a -setStoragePolicy \uff0c\u6267\u884c jindotable -help setStoragePolicy \u53ef\u4ee5\u663e\u793a\u5e2e\u52a9\u3002","title":"\u5206\u5c42\u5b58\u50a8\u547d\u4ee4"},{"location":"jindofs/jindotable_storagepolicy_restore/#_5","text":"jindotable -setStoragePolicy -t <dbName.tableName> -std/-ia/-archive/-coldArchive/-once [-c \"<condition>\" | -fullTable] [-d/-restoreDays <restore days>] [-b/-before <before days>] [-p/-parallel <parallelism>] [-mr/-mapReduce] [-e/-explain] [-w/-workingDir <working directory>] [-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u76ee\u6807\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a\u6570\u636e\u5e93\u540d.\u8868\u540d\u3002\u6570\u636e\u5e93\u548c\u8868\u540d\u4e4b\u95f4\u4ee5\u534a\u89d2\u53e5\u53f7\uff08.\uff09\u5206\u9694\u3002\u8868\u53ef\u4ee5\u662f\u5206\u533a\u8868\u6216\u975e\u5206\u533a\u8868\u3002 \u662f -std/-ia/-archive/-coldArchive/-once \u76ee\u6807\u5b58\u50a8\u65b9\u5f0f\u3002\u652f\u6301\u5982\u4e0b\u7c7b\u578b\uff1a-std\uff1a\u6807\u51c6\u5b58\u50a8\uff0c-ia\uff1a\u4f4e\u9891\u5b58\u50a8\uff0c-archive\uff1a\u5f52\u6863\u5b58\u50a8\uff0c-coldArchive\uff1a\u51b7\u5f52\u6863\u5b58\u50a8\uff0c-once\uff1a\u4e34\u65f6\u89e3\u51bb\u3002 \u662f -c \"<condition>\" / -fullTable -fullTable \u548c -c \"<condition>\" \u53ea\u9700\u63d0\u4f9b\u4e00\u4e2a\uff0c\u5373\u8981\u4e48\u6307\u5b9a -c \"<condition>\" \uff0c\u8981\u4e48\u6307\u5b9a -fullTable \u3002\u6307\u5b9a -fullTable \u65f6\uff0c\u5219\u4e3a\u79fb\u52a8\u6574\u8868\uff0c\u65e2\u53ef\u4ee5\u662f\u975e\u5206\u533a\u8868\u4e5f\u53ef\u4ee5\u662f\u5206\u533a\u8868\u3002\u6307\u5b9a -c \"<condition>\" \u65f6\uff0c\u5219\u63d0\u4f9b\u4e86\u4e00\u4e2a\u8fc7\u6ee4\u6761\u4ef6\uff0c\u7528\u6765\u9009\u62e9\u5e0c\u671b\u79fb\u52a8\u7684\u5206\u533a\uff0c\u652f\u6301\u5e38\u89c1\u8fd0\u7b97\u7b26\uff0c\u4f8b\u5982\u5927\u4e8e\u53f7\uff08>\uff09\u3002\u4f8b\u5982\uff0c\u6570\u636e\u7c7b\u578b\u4e3aString\u7684\u5206\u533ads\uff0c\u5e0c\u671b\u5206\u533a\u540d\u5927\u4e8e 'd'\uff0c\u5219\u4ee3\u7801\u4e3a-c \" ds > 'd' \"\u3002 \u662f -d/-restoreDays <restore days> \u6307\u5b9a\u4e34\u65f6\u89e3\u51bb\u7684\u5929\u6570\uff0c\u4ec5\u5bf9\u4e34\u65f6\u89e3\u51bb -once \u751f\u6548\u3002 \u5426 -b/before <before days> \u53ea\u6709\u521b\u5efa\u65f6\u95f4\u8ddd\u79bb\u73b0\u5728\u8d85\u8fc7\u4e00\u5b9a\u5929\u6570\u7684\u8868\u6216\u5206\u533a\u624d\u4f1a\u88ab\u6267\u884c\u3002 \u5426 -p/-parallel <parallelism> \u547d\u4ee4\u6267\u884c\u7684\u5e76\u884c\u5ea6\u3002 \u5426 -mr/-mapReduce \u4f7f\u7528Hadoop MapReduce\u800c\u975e\u672c\u5730\u591a\u7ebf\u7a0b\u6765\u6267\u884c\u547d\u4ee4\u3002 \u5426 -e/-explain \u5982\u679c\u51fa\u73b0\u8be5\u9009\u9879\uff0c\u5219\u4e3a\u89e3\u91ca\uff08explain\uff09\u6a21\u5f0f\uff0c\u53ea\u4f1a\u663e\u793a\u5f85\u64cd\u4f5c\u7684\u5206\u533a\u5217\u8868\uff0c\u800c\u4e0d\u4f1a\u771f\u6b63\u6267\u884c\u547d\u4ee4\u3002 \u5426 -w/-workingDir <working directory> \u53ea\u5728MapReduce\u4f5c\u4e1a\u65f6\u4f7f\u7528\uff0c\u4e3aMapReduce\u4f5c\u4e1a\u7684\u5de5\u4f5c\u76ee\u5f55\u3002\u5fc5\u987b\u6709\u8bfb\u5199\u6743\u9650\uff0c\u5de5\u4f5c\u76ee\u5f55\u53ef\u4ee5\u975e\u7a7a\uff08\u4f5c\u4e1a\u6267\u884c\u8fc7\u7a0b\u4e2d\u4f1a\u521b\u5efa\u4e34\u65f6\u6587\u4ef6\uff0c\u6267\u884c\u5b8c\u6bd5\u4f1a\u6e05\u7406\u4e34\u65f6\u6587\u4ef6\uff09\u3002 \u5426 -l/-logDir <log directory> \u6307\u5b9aLog\u6587\u4ef6\u7684\u76ee\u5f55\u3002 \u5426","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"jindofs/jindotable_storagepolicy_restore/#_6","text":"JindoTable \u63d0\u4f9b\u7684\u4e0a\u8ff0\u529f\u80fd\u57fa\u672c\u5bf9\u5e94\u4e8e OSS-HDFS \u7ebf\u4e0a\u670d\u52a1\u7684\u5206\u5c42\u5b58\u50a8\u80fd\u529b\uff0c\u6709\u4e0b\u5217\u4e8b\u9879\u9700\u8981\u6ce8\u610f\uff1a 1. \u4ec5\u652f\u6301 OSS-HDFS \u5b58\u50a8\u7684\u8868\u6216\u8005\u5206\u533a\u3002\u5bf9\u666e\u901a OSS \u6570\u636e\u7684\u5206\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoTable \u63d0\u4f9b\u7684 archiveTable \u4e0e unarchiveTable \u547d\u4ee4\u3002 2. \u6210\u529f\u6267\u884c -std/-ia/-archive/-coldArchive/-once \u6307\u5b9a\u7684\u8f6c\u6362\u5b58\u50a8\u7c7b\u578b\u547d\u4ee4\u540e\uff0c\u5fc5\u987b\u7b49\u5f85 2 \u5929\u624d\u53ef\u518d\u6b21\u8f6c\u6362\u5b58\u50a8\u7c7b\u578b\u3002 3. \u6807\u51c6\u548c\u4f4e\u9891\u7c7b\u578b\u65e0\u6cd5\u4e34\u65f6\u89e3\u51bb\uff08 -once \uff09\u3002\u5f52\u6863\u3001\u51b7\u5f52\u6863\uff0c\u6216\u8005\u5df2\u7ecf\u5904\u4e8e\u89e3\u51bb\u72b6\u6001\u65f6\uff0c\u53ef\u4ee5\u89e3\u51bb\uff0c\u4f46\u5982\u4e0a\u6240\u8ff0\uff0c\u95f4\u9694\u987b\u8d85\u8fc7 2 \u5929\u3002 4. \u4e34\u65f6\u89e3\u51bb\u547d\u4ee4\u8fd4\u56de\u540e\uff0c\u6570\u636e\u4ecd\u4e0d\u80fd\u7acb\u5373\u53ef\u8bfb\uff0c\u9700\u8981\u7b49\u5f85\u8fdc\u7aef\u670d\u52a1\u5b8c\u6210\u6570\u636e\u7684\u89e3\u51bb\u3002\u5bf9\u4e8e\u5f52\u6863\u6570\u636e\uff0c\u901a\u5e38\u9700\u8981\u6570\u5206\u949f\uff1b\u5bf9\u4e8e\u51b7\u5f52\u6863\u6570\u636e\uff0c\u901a\u5e38\u9700\u8981\u6570\u5c0f\u65f6\u3002 5. \u4e34\u65f6\u89e3\u51bb\u6709\u671f\u9650\u9650\u5236\uff0c\u901a\u8fc7 -d/restoreDays <restore days> \u6307\u5b9a\uff0c\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4 1 \u5929\uff0c\u5230\u671f\u540e\u6570\u636e\u5c06\u56de\u5230\u4e0d\u53ef\u8bfb\u72b6\u6001\u3002\u8be5\u529f\u80fd\u8981\u6c42\u7248\u672c\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0\u3002","title":"\u6ce8\u610f\u4e8b\u9879"},{"location":"jindofs/issues/dls_unarchive_issue/","text":"\u95ee\u9898\u63cf\u8ff0 \u00a7 \u7528\u6237\u8c03\u7528\u89e3\u5f52\u6863\u5c06\u6570\u636e\u4ece\u5f52\u6863/\u51b7\u5f52\u6863\u6062\u590d\u6807\u51c6\u64cd\u4f5c\u540e\uff0c\u6709\u65f6\u8fd8\u662f\u4f1a\u51fa\u73b0\u89e3\u5f52\u6863\u7684\u6587\u4ef6\u4e0d\u53ef\u8bfb\u53d6\u60c5\u51b5\uff0c\u51fa\u73b0\u5982\u4e0b\u62a5\u9519 Read from oss://<DLS_BUCKET>.<DLS_ENDPOINT>/<FILE> with error message: Caused by error 30005: IO error: IO error: [RequestId]: XXXXXXXXXXXXXX [HostId]: <OSS_ENDPOINT> [ErrorMessage]: [E1010]HTTP/1.1 403 Forbidden: <?xml version=\"1.0\" encoding=\"UTF-8\"?><Error> <Code>InvalidObjectState</Code> <Message>The operation is not valid for the object's state</Message> <RequestId>XXXXXXXXXXXXXXX</RequestId> <HostId><DLS_BUCKET>.<OSS_ENDPOINT></HostId> <ObjectName>.dlsdata/xxxxxxxxxxxx</ObjectName></Error> [ErrorCode]: 1010 [RequestId]: XXXXXXXXXXXXXXX \u95ee\u9898\u6839\u56e0 \u00a7 \u51fa\u73b0\u4ee5\u4e0a\u60c5\u51b5\u5927\u6982\u7387\u7531\u4e8e\u7528\u6237\u6570\u636e\u5757\u8fc7\u5927\uff0c\u5bfc\u81f4OSS\u6570\u636e\u5757\u5728\u8f6c\u6362\u5b58\u50a8\u7c7b\u578b\u65f6\u51fa\u73b0\u6570\u636e\u7684\u62f7\u8d1d\u8d85\u65f6\uff0c\u4efb\u52a1\u89e3\u5f52\u6863\u6700\u7ec8\u51fa\u73b0\u5931\u8d25\u3002\u6c38\u4e45\u89e3\u5f52\u6863\u5b58\u5728\u89e3\u51bb\u9650\u5236\uff0c\u63a8\u8350\u7528\u6237\u4f7f\u7528\u4e34\u65f6\u89e3\u51bb\u529f\u80fd\uff0c\u8be6\u7ec6\u7ec6\u8282\u53ef\u4ee5\u53c2\u8003 \u6587\u6863 \u3002 \u4fee\u590d\u6d41\u7a0b \u00a7 \u51fa\u73b0\u8fd9\u79cd\u60c5\u51b5\u7528\u6237\u5982\u4f55\u8fdb\u884c\u6570\u636e\u4fee\u590d\uff0c\u9700\u8981\u4f9d\u8d56JindoFsck\u547d\u4ee4\u6765\u4fee\u590d\uff0cJindoFsck \u547d\u4ee4 \u4e0b\u8f7d\u94fe\u63a5 \uff0c JindoFsck\u547d\u4ee4\u4f7f\u7528\u5982\u4e0b [root@master-1-1(192.168.0.182) ~]# ./jindofsck jindofsck [-help] [-version] [-restoreFileBlocks -dlsUri <path> [-restoreDays <restoreDays>]] [-restoreBlock -dlsUri <path> -block <block> [-restoreDays <restoreDays>]] [-restoreBlocks -dlsUri <path> [-days <days>] [-restoreDays <restoreDays>]] JindoFsck \u73b0\u5728\u652f\u6301\u4e09\u79cd\u7c7b\u578b\u6570\u636e\u7684\u4fee\u590d\uff0c\u5206\u522b\u5bf9\u5e94\u89e3\u51bb\u5355\u4e2a\u6570\u636e\u5757\uff0c\u89e3\u51bb\u5355\u4e2a\u6587\u4ef6\uff0c\u89e3\u51bb\u7279\u5b9a\u65f6\u95f4\u5185\u6c38\u4e45\u89e3\u51bb\u5931\u8d25\u7684\u6570\u636e\u5757\u3002\u7528\u6237\u53ef\u4ee5\u6839\u636e\u9700\u8981\u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\u53bb\u4e34\u65f6\u89e3\u51bb\u6570\u636e\u3002 \u89e3\u51bb\u5355\u4e2a\u6570\u636e\u5757 \u00a7 ./jindofsck -restoreBlock -dlsUri oss://<DLS_BUCKET>.<DLS_ENDPOINT>/ -block .dlsdata/hosts24 -restoreDays \u53c2\u6570\u8bf4\u660e | \u53c2\u6570 | \u63cf\u8ff0 | \u662f\u5426\u5fc5\u9009 | | :--- | :--- | :--- | | -dlsUri | OSS-HDFS Bucket\u5730\u5740\u3002 | \u662f| | -block | \u9700\u8981\u6062\u590d\u7684block\u7684\u5730\u5740\uff0c\u91c7\u7528\u76f8\u5bf9\u8def\u5f84\uff0c\u4e0d\u80fdoss:// \u6216\u8005 / \u5f00\u5934\uff0c \u683c\u5f0f .dlsdata/xxxxxx | \u662f| | -restoreDays | \u6307\u5b9a\u4e34\u65f6\u89e3\u51bb\u5929\u6570\uff0c\u5bf9\u4e8e\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 7 \u5929\uff0c\u51b7\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 365 \u5929\uff0c \u9ed8\u8ba4\u89e3\u51bb\u5f52\u68637\u5929\uff0c\u51b7\u5f52\u686330\u5929\u3002 | \u5426 | \u89e3\u51bb\u5355\u4e2a\u6587\u4ef6 \u00a7 ./jindofsck -restoreFileBlocks -dlsUri oss://<DLS_BUCKET>.<DLS_ENDPOINT>/<PATH> -restoreDays 2 \u53c2\u6570\u8bf4\u660e | \u53c2\u6570 | \u63cf\u8ff0 | \u662f\u5426\u5fc5\u9009 | | :--- | :--- | :--- | | -dlsUri | OSS-HDFS Bucket\u5730\u5740\u3002 | \u662f| | -restoreDays | \u6307\u5b9a\u4e34\u65f6\u89e3\u51bb\u5929\u6570\uff0c\u5bf9\u4e8e\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 7 \u5929\uff0c\u51b7\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 365 \u5929\uff0c \u9ed8\u8ba4\u89e3\u51bb\u5f52\u68637\u5929\uff0c\u51b7\u5f52\u686330\u5929\u3002 | \u5426 | \u89e3\u51bb\u6307\u5b9a\u65f6\u95f4\u6bb5\u5931\u8d25\u7684\u6570\u636e \u00a7 ./jindofsck -restoreBlocks -dlsUri oss://<DLS_BUCKET>.<DLS_ENDPOINT>/<PATH> -days 2 -restoreDays 2 \u53c2\u6570\u8bf4\u660e | \u53c2\u6570 | \u63cf\u8ff0 | \u662f\u5426\u5fc5\u9009 | | :--- | :--- | :--- | | -dlsUri | OSS-HDFS Bucket\u5730\u5740\u3002 | \u662f| | -days | \u8868\u793a\u6062\u590d\u51e0\u5929\u5185\u6c38\u4e45\u89e3\u5f52\u6863\u5931\u8d25\u7684\u6570\u636e\uff0c\u9ed8\u8ba4\u503c\u4e3a 2\uff0c \u8868\u793a\u6062\u590d\u4e24\u5929\u5185\u6c38\u4e45\u89e3\u5f52\u6863\u5931\u8d25\u7684\u6570\u636e\u3002 | \u5426| | -restoreDays | \u6307\u5b9a\u4e34\u65f6\u89e3\u51bb\u5929\u6570\uff0c\u5bf9\u4e8e\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 7 \u5929\uff0c\u51b7\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 365 \u5929\uff0c \u9ed8\u8ba4\u89e3\u51bb\u5f52\u68637\u5929\uff0c\u51b7\u5f52\u686330\u5929\u3002 | \u5426 |","title":"Dls unarchive issue"},{"location":"jindofs/issues/dls_unarchive_issue/#_1","text":"\u7528\u6237\u8c03\u7528\u89e3\u5f52\u6863\u5c06\u6570\u636e\u4ece\u5f52\u6863/\u51b7\u5f52\u6863\u6062\u590d\u6807\u51c6\u64cd\u4f5c\u540e\uff0c\u6709\u65f6\u8fd8\u662f\u4f1a\u51fa\u73b0\u89e3\u5f52\u6863\u7684\u6587\u4ef6\u4e0d\u53ef\u8bfb\u53d6\u60c5\u51b5\uff0c\u51fa\u73b0\u5982\u4e0b\u62a5\u9519 Read from oss://<DLS_BUCKET>.<DLS_ENDPOINT>/<FILE> with error message: Caused by error 30005: IO error: IO error: [RequestId]: XXXXXXXXXXXXXX [HostId]: <OSS_ENDPOINT> [ErrorMessage]: [E1010]HTTP/1.1 403 Forbidden: <?xml version=\"1.0\" encoding=\"UTF-8\"?><Error> <Code>InvalidObjectState</Code> <Message>The operation is not valid for the object's state</Message> <RequestId>XXXXXXXXXXXXXXX</RequestId> <HostId><DLS_BUCKET>.<OSS_ENDPOINT></HostId> <ObjectName>.dlsdata/xxxxxxxxxxxx</ObjectName></Error> [ErrorCode]: 1010 [RequestId]: XXXXXXXXXXXXXXX","title":"\u95ee\u9898\u63cf\u8ff0"},{"location":"jindofs/issues/dls_unarchive_issue/#_2","text":"\u51fa\u73b0\u4ee5\u4e0a\u60c5\u51b5\u5927\u6982\u7387\u7531\u4e8e\u7528\u6237\u6570\u636e\u5757\u8fc7\u5927\uff0c\u5bfc\u81f4OSS\u6570\u636e\u5757\u5728\u8f6c\u6362\u5b58\u50a8\u7c7b\u578b\u65f6\u51fa\u73b0\u6570\u636e\u7684\u62f7\u8d1d\u8d85\u65f6\uff0c\u4efb\u52a1\u89e3\u5f52\u6863\u6700\u7ec8\u51fa\u73b0\u5931\u8d25\u3002\u6c38\u4e45\u89e3\u5f52\u6863\u5b58\u5728\u89e3\u51bb\u9650\u5236\uff0c\u63a8\u8350\u7528\u6237\u4f7f\u7528\u4e34\u65f6\u89e3\u51bb\u529f\u80fd\uff0c\u8be6\u7ec6\u7ec6\u8282\u53ef\u4ee5\u53c2\u8003 \u6587\u6863 \u3002","title":"\u95ee\u9898\u6839\u56e0"},{"location":"jindofs/issues/dls_unarchive_issue/#_3","text":"\u51fa\u73b0\u8fd9\u79cd\u60c5\u51b5\u7528\u6237\u5982\u4f55\u8fdb\u884c\u6570\u636e\u4fee\u590d\uff0c\u9700\u8981\u4f9d\u8d56JindoFsck\u547d\u4ee4\u6765\u4fee\u590d\uff0cJindoFsck \u547d\u4ee4 \u4e0b\u8f7d\u94fe\u63a5 \uff0c JindoFsck\u547d\u4ee4\u4f7f\u7528\u5982\u4e0b [root@master-1-1(192.168.0.182) ~]# ./jindofsck jindofsck [-help] [-version] [-restoreFileBlocks -dlsUri <path> [-restoreDays <restoreDays>]] [-restoreBlock -dlsUri <path> -block <block> [-restoreDays <restoreDays>]] [-restoreBlocks -dlsUri <path> [-days <days>] [-restoreDays <restoreDays>]] JindoFsck \u73b0\u5728\u652f\u6301\u4e09\u79cd\u7c7b\u578b\u6570\u636e\u7684\u4fee\u590d\uff0c\u5206\u522b\u5bf9\u5e94\u89e3\u51bb\u5355\u4e2a\u6570\u636e\u5757\uff0c\u89e3\u51bb\u5355\u4e2a\u6587\u4ef6\uff0c\u89e3\u51bb\u7279\u5b9a\u65f6\u95f4\u5185\u6c38\u4e45\u89e3\u51bb\u5931\u8d25\u7684\u6570\u636e\u5757\u3002\u7528\u6237\u53ef\u4ee5\u6839\u636e\u9700\u8981\u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\u53bb\u4e34\u65f6\u89e3\u51bb\u6570\u636e\u3002","title":"\u4fee\u590d\u6d41\u7a0b"},{"location":"jindofs/issues/dls_unarchive_issue/#_4","text":"./jindofsck -restoreBlock -dlsUri oss://<DLS_BUCKET>.<DLS_ENDPOINT>/ -block .dlsdata/hosts24 -restoreDays \u53c2\u6570\u8bf4\u660e | \u53c2\u6570 | \u63cf\u8ff0 | \u662f\u5426\u5fc5\u9009 | | :--- | :--- | :--- | | -dlsUri | OSS-HDFS Bucket\u5730\u5740\u3002 | \u662f| | -block | \u9700\u8981\u6062\u590d\u7684block\u7684\u5730\u5740\uff0c\u91c7\u7528\u76f8\u5bf9\u8def\u5f84\uff0c\u4e0d\u80fdoss:// \u6216\u8005 / \u5f00\u5934\uff0c \u683c\u5f0f .dlsdata/xxxxxx | \u662f| | -restoreDays | \u6307\u5b9a\u4e34\u65f6\u89e3\u51bb\u5929\u6570\uff0c\u5bf9\u4e8e\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 7 \u5929\uff0c\u51b7\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 365 \u5929\uff0c \u9ed8\u8ba4\u89e3\u51bb\u5f52\u68637\u5929\uff0c\u51b7\u5f52\u686330\u5929\u3002 | \u5426 |","title":"\u89e3\u51bb\u5355\u4e2a\u6570\u636e\u5757"},{"location":"jindofs/issues/dls_unarchive_issue/#_5","text":"./jindofsck -restoreFileBlocks -dlsUri oss://<DLS_BUCKET>.<DLS_ENDPOINT>/<PATH> -restoreDays 2 \u53c2\u6570\u8bf4\u660e | \u53c2\u6570 | \u63cf\u8ff0 | \u662f\u5426\u5fc5\u9009 | | :--- | :--- | :--- | | -dlsUri | OSS-HDFS Bucket\u5730\u5740\u3002 | \u662f| | -restoreDays | \u6307\u5b9a\u4e34\u65f6\u89e3\u51bb\u5929\u6570\uff0c\u5bf9\u4e8e\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 7 \u5929\uff0c\u51b7\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 365 \u5929\uff0c \u9ed8\u8ba4\u89e3\u51bb\u5f52\u68637\u5929\uff0c\u51b7\u5f52\u686330\u5929\u3002 | \u5426 |","title":"\u89e3\u51bb\u5355\u4e2a\u6587\u4ef6"},{"location":"jindofs/issues/dls_unarchive_issue/#_6","text":"./jindofsck -restoreBlocks -dlsUri oss://<DLS_BUCKET>.<DLS_ENDPOINT>/<PATH> -days 2 -restoreDays 2 \u53c2\u6570\u8bf4\u660e | \u53c2\u6570 | \u63cf\u8ff0 | \u662f\u5426\u5fc5\u9009 | | :--- | :--- | :--- | | -dlsUri | OSS-HDFS Bucket\u5730\u5740\u3002 | \u662f| | -days | \u8868\u793a\u6062\u590d\u51e0\u5929\u5185\u6c38\u4e45\u89e3\u5f52\u6863\u5931\u8d25\u7684\u6570\u636e\uff0c\u9ed8\u8ba4\u503c\u4e3a 2\uff0c \u8868\u793a\u6062\u590d\u4e24\u5929\u5185\u6c38\u4e45\u89e3\u5f52\u6863\u5931\u8d25\u7684\u6570\u636e\u3002 | \u5426| | -restoreDays | \u6307\u5b9a\u4e34\u65f6\u89e3\u51bb\u5929\u6570\uff0c\u5bf9\u4e8e\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 7 \u5929\uff0c\u51b7\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 365 \u5929\uff0c \u9ed8\u8ba4\u89e3\u51bb\u5f52\u68637\u5929\uff0c\u51b7\u5f52\u686330\u5929\u3002 | \u5426 |","title":"\u89e3\u51bb\u6307\u5b9a\u65f6\u95f4\u6bb5\u5931\u8d25\u7684\u6570\u636e"},{"location":"jindofuse/jindofuse_quickstart/","text":"JindoFuse \u4f7f\u7528\u6307\u5357 \u00a7 \u6982\u8ff0 \u00a7 OSS/OSS-HDFS \u53ef\u4ee5\u901a\u8fc7 JindoFuse \u63d0\u4f9b POSIX \u652f\u6301\uff0c\u5c06 OSS/OSS-HDFS \u4e0a\u7684\u6587\u4ef6\u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c OSS/OSS-HDFS \u4e0a\u7684\u6587\u4ef6\u3002 \u90e8\u7f72\u5b89\u88c5 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.y.z.tar.gz\uff08x.y.z\u8868\u7248\u672c\u53f7\uff09\uff0c \u4e0b\u8f7d\u9875\u9762 \u3002 \u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u53c2\u89c1 \u6587\u6863\u94fe\u63a5 \u6302\u8f7d JindoFuse \u00a7 \u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002 \u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a mkdir -p <mountpoint> \u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a jindo-fuse <mount_point> -ouri=[<osspath>] \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 <oss_path> \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 <mount_point> \u3002 <mount_point> \u9700\u66ff\u6362\u4e3a\u4e00\u4e2a\u672c\u5730\u8def\u5f84\u3002 <oss_path> \u9700\u66ff\u6362\u4e3a\u5f85\u6620\u5c04\u7684 OSS/OSS-HDFS \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a Bucket \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002\u5982 oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/subdir/ \u6302\u8f7d OSS \u548c OSS-HDFS \u8def\u5f84\u7684\u65b9\u5f0f\u57fa\u672c\u76f8\u540c\uff0c\u4ec5\u8def\u5f84\u4e2d\u7684 endpoint \u7565\u6709\u4e0d\u540c\u3002\u6839\u8def\u5f84\u793a\u4f8b\u53ef\u53c2\u89c1\uff1a \u300aOSS/OSS-HDFS \u5feb\u901f\u5165\u95e8\u300b \u786e\u8ba4\u6302\u8f7d\u6210\u529f\uff0cjindo-fuse \u8fdb\u7a0b\u5b58\u5728\uff0c\u4e14\u542f\u52a8\u53c2\u6570\u4e0e\u9884\u671f\u4e00\u81f4 ps -ef | grep jindo-fuse \u8bbf\u95ee JindoFuse \u00a7 \u5982\u679c\u5c06 JindoFS \u670d\u52a1\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1 ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/ \u5378\u8f7d JindoFuse \u00a7 \u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point> \u81ea\u52a8\u5378\u8f7d JindoFuse \u00a7 \u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u7279\u6027\u652f\u6301 \u00a7 \u76ee\u524d JindoFuse \u5df2\u7ecf\u652f\u6301\u4ee5\u4e0b POSIX API\uff1a \u7279\u6027 \u8bf4\u660e OSS OSS-HDFS getattr() \u67e5\u8be2\u6587\u4ef6\u5c5e\u6027\uff0c\u7c7b\u4f3c ls \u2705 \u2705 mkdir() \u521b\u5efa\u76ee\u5f55\uff0c\u7c7b\u4f3c mkdir \u2705 \u2705 rmdir() \u5220\u9664\u76ee\u5f55\uff0c\u7c7b\u4f3c rm -rf \u2705 \u2705 unlink() \u5220\u9664\u6587\u4ef6\uff0c\u7c7b\u4f3c unlink \u2705 \u2705 rename() \u91cd\u547d\u540d\uff0c\u7c7b\u4f3c mv \u2705 \u2705 read() \u987a\u5e8f\u8bfb\u53d6 \u2705 \u2705 pread() \u968f\u673a\u8bfb \u2705 \u2705 write() \u987a\u5e8f\u5199 \u2705 \u2705 pwrite() \u968f\u673a\u5199 \u274c \u2705 flush() \u5237\u65b0\u5185\u5b58\u5230\u5185\u6838\u7f13\u51b2\u533a \u4ec5\u652f\u6301\u4ee5\u6700\u8ffd\u52a0\u5199\u65b9\u5f0f\u6253\u5f00\u7684\u6587\u4ef6 \u2705 fsync() \u5237\u65b0\u5185\u5b58\u5230\u78c1\u76d8 \u4ec5\u652f\u6301\u4ee5\u6700\u8ffd\u52a0\u5199\u65b9\u5f0f\u6253\u5f00\u7684\u6587\u4ef6 \u2705 release() \u5173\u95ed\u6587\u4ef6 \u2705 \u2705 readdir() \u8bfb\u53d6\u76ee\u5f55 \u2705 \u2705 create() \u521b\u5efa\u6587\u4ef6 \u2705 \u2705 open() O_APPEND \u901a\u8fc7\u8ffd\u52a0\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 \u652f\u6301\uff0c\u4f46\u6709 \u4f7f\u7528\u9650\u5236 \u2705 open() O_TRUNC \u901a\u8fc7\u8986\u76d6\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 \u2705 \u2705 ftruncate() \u5bf9\u6253\u5f00\u7684\u6587\u4ef6\u8fdb\u884c\u622a\u65ad \u274c \u2705 truncate() \u5bf9\u672a\u6253\u5f00\u7684\u6587\u4ef6\u8fdb\u884c\u622a\u65ad\uff0c\u7c7b\u4f3c truncate -s \u274c \u2705 lseek() \u6307\u5b9a\u6253\u5f00\u6587\u4ef6\u4e2d\u7684\u8bfb\u5199\u4f4d\u7f6e\u3002 \u274c \u2705 chmod() \u4fee\u6539\u6587\u4ef6\u6743\u9650\uff0c\u7c7b\u4f3c chmod \u274c \u2705 access() \u67e5\u8be2\u6587\u4ef6\u6743\u9650 \u2705 \u2705 utimes() \u4fee\u6539\u6587\u4ef6\u7684\u5b58\u53d6\u65f6\u95f4\u548c\u66f4\u6539\u65f6\u95f4 \u274c \u2705 setxattr() \u4fee\u6539\u6587\u4ef6 xattr \u5c5e\u6027 \u274c \u2705 getxattr() \u83b7\u53d6\u6587\u4ef6 xattr \u5c5e\u6027 \u274c \u2705 listxattr() \u5217\u4e3e\u6587\u4ef6 xattr \u5c5e\u6027 \u274c \u2705 removexattr() \u5220\u9664\u6587\u4ef6 xattr \u5c5e\u6027 \u274c \u2705 lock() \u652f\u6301 posix \u9501\uff0c\u7c7b\u4f3c fcntl \u274c \u2705 fallocate() \u4e3a\u6587\u4ef6\u9884\u5206\u914d\u7269\u7406\u7a7a\u95f4 \u274c \u2705 symlink() \u521b\u5efa\u8f6f\u8fde\u63a5 \u274c \u76ee\u524d\u4ec5\u652f\u6301 OSS-HDFS \u5185\u90e8\u4f7f\u7528\uff0c\u4e14\u4e0d\u652f\u6301\u7f13\u5b58\u52a0\u901f readlink() \u8bfb\u53d6\u8f6f\u8fde\u63a5 \u274c \u76ee\u524d\u4ec5\u652f\u6301 OSS-HDFS \u5185\u90e8\u4f7f\u7528\uff0c\u4e14\u4e0d\u652f\u6301\u7f13\u5b58\u52a0\u901f \u9ad8\u9636\u4f7f\u7528 \u00a7 \u6302\u8f7d\u9009\u9879 \u00a7 \u53c2\u6570\u540d\u79f0 \u5fc5\u9009 \u7248\u672c \u53c2\u6570\u8bf4\u660e \u4f7f\u7528\u8303\u4f8b uri \u2713 4.3.0+ \u914d\u7f6e\u9700\u8981\u6620\u5c04\u7684 oss \u8def\u5f84\u3002\u8def\u5f84\u53ef\u4ee5\u662f\u6839\u76ee\u5f55\uff0c\u4e5f\u53ef\u4ee5\u662f\u5b50\u76ee\u5f55\u3002\u4f8b\u5982\uff1aoss://examplebucket/ \u6216 oss://examplebucket/subdir\u3002 -ouri=oss://examplebucket/ f 4.3.0+ \u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u9ed8\u8ba4\u4f7f\u7528\u5b88\u62a4\u8fdb\u7a0b\u65b9\u5f0f\u540e\u53f0\u542f\u52a8\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -f d 4.3.0+ \u4f7f\u7528 Debug \u6a21\u5f0f\uff0c\u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -d auto_unmount 4.3.0+ fuse\u8fdb\u7a0b\u9000\u51fa\u540e\u81ea\u52a8umount\u6302\u8f7d\u8282\u70b9\u3002 -oauto_unmount ro 4.3.0+ \u53ea\u8bfb\u6302\u8f7d\uff0c\u542f\u7528\u53c2\u6570\u540e\u4e0d\u5141\u8bb8\u5199\u64cd\u4f5c\u3002 -oro direct_io 4.3.0+ \u5f00\u542f\u540e\uff0c\u8bfb\u5199\u6587\u4ef6\u53ef\u4ee5\u7ed5\u8fc7page cache\u3002 -odirect_io kernel_cache 4.3.0+ \u5f00\u542f\u540e\uff0c\u5229\u7528\u5185\u6838\u7f13\u5b58\u4f18\u5316\u8bfb\u6027\u80fd\u3002 -okernel_cache auto_cache 4.3.0+ \u9ed8\u8ba4\u5f00\u542f\uff0c\u4e0ekernel_cache \u4e8c\u9009\u4e00\uff0c\u4e0ekernel_cache\u4e0d\u540c\u7684\u662f\uff0c\u5982\u679c\u6587\u4ef6\u5927\u5c0f\u6216\u4fee\u6539\u65f6\u95f4\u53d1\u751f\u53d8\u5316\uff0c\u7f13\u5b58\u5c31\u4f1a\u5931\u6548\u3002 entry_timeout 4.3.0+ \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oentry_timeout=60 attr_timeout 4.3.0+ \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oattr_timeout=60 negative_timeout 4.3.0+ \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u5931\u8d25\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -onegative_timeout=0 max_idle_threads 4.3.0+ \u9ed8\u8ba4\u503c\uff0c10\u3002\u5904\u7406\u5185\u6838\u56de\u8c03\u7684\u7a7a\u95f2\u7ebf\u7a0b\u6c60\u3002 -omax_idle_threads=10 xengine 4.3.0+ \u6253\u5f00\u7f13\u5b58 -oxengine pread 4.5.1+ \u9ed8\u8ba4\u4f7f\u7528\u987a\u5e8f\u8bfb\u3002\u6253\u5f00\u540e\uff0c\u4f7f\u7528\u968f\u673a\u8bfb\u4ee3\u66ff\u987a\u5e8f\u8bfb\uff0c\u9002\u7528\u4e8e\u968f\u673a\u8bfb\u8fdc\u591a\u4e8e\u987a\u5e8f\u8bfb\u7684\u573a\u666f\u3002 -opread no_symlink 4.5.1+ \u914d\u7f6e\u540e\uff0c\u5173\u95edsymlink\u529f\u80fd\u3002 -ono_symlink no_writeback 4.5.1+ \u914d\u7f6e\u540e\uff0c\u5173\u95edwriteback\u529f\u80fd\u3002 -ono_writeback no_flock 4.5.1+ \u914d\u7f6e\u540e\uff0c\u5173\u95edflock\u529f\u80fd\u3002 -ono_flock no_xattr 4.5.1+ \u914d\u7f6e\u540e\uff0c\u5173\u95edxttar\u529f\u80fd\u3002 -ono_xattr \u914d\u7f6e\u9009\u9879 \u00a7 \u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u8bf4\u660e logger.dir /tmp/bigboot-log \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF logger.verbose 0 \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint \u8bbf\u95ee JindoFS \u670d\u52a1\u7684\u5730\u5740\uff0c\u5982oss-cn-xxx.aliyuncs.com fs.oss.accessKeyId \u8bbf\u95ee JindoFS \u670d\u52a1\u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret \u8bbf\u95ee JindoFS \u670d\u52a1\u9700\u8981\u7684 accessKeySecret \u66f4\u591a\u53c2\u6570\u53ef\u89c1 \u300a\u5ba2\u6237\u7aef\u5e38\u7528\u914d\u7f6e\u300b \u3002 \u652f\u6301\u5c06 jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<oss_path>] -ofs.oss.endpoint=[<your_endpoint>] -ofs.oss.accessKeyId=[<your_key_id>] -ofs.oss.accessKeySecret=[<your_key_secret>] \u5e38\u89c1\u95ee\u9898 \u00a7 Input/Output error \u00a7 \u4e0d\u50cf\u4f7f\u7528 JindoSDK \u8c03\u7528 API \u53ef\u4ee5\u83b7\u53d6\u66f4\u4e3a\u5177\u4f53\u7684 ErrorMsg\uff0cJindoFuse \u53ea\u80fd\u663e\u793a\u64cd\u4f5c\u7cfb\u7edf\u9884\u8bbe\u7684\u9519\u8bef\u4fe1\u606f\uff0c\u6bd4\u5982\u4ee5\u4e0b\u9519\u8bef\u5c31\u975e\u5e38\u5e38\u89c1\uff1a $ ls /mnt/oss/ ls: /mnt/oss/: Input/output error \u5982\u679c\u9700\u8981\u5b9a\u4f4d\u5177\u4f53\u7684\u9519\u8bef\u539f\u56e0\uff0c\u53ef\u4ee5\u6839\u636e JindoSDK \u914d\u7f6e\u4e2d\u7684 logger.dir\uff0c\u5728\u6307\u5b9a\u8def\u5f84\u4e0b\u7684 jindosdk.log \u6587\u4ef6\u4e2d\uff0c\u5bfb\u627e\u5177\u4f53\u7684\u9519\u8bef\u3002 \u4e0b\u9762\u5c55\u793a\u7684\u8fd9\u4e2a\u9519\u8bef\u5c31\u6765\u81ea jindosdk.log \uff0c\u6839\u636e\u62a5\u9519\u4fe1\u606f\u53ef\u89c1\uff0c\u8fd9\u4e2a\u9519\u8bef\u662f\u5e38\u89c1\u7684\u9274\u6743\u95ee\u9898\u3002 EMMDD HH:mm:ss jindofs_connectivity.cpp:13] Please check your Endpoint/Bucket/RoleArn. Failed test connectivity, operation: mkdir, errMsg: [RequestId]: 618B8183343EA53531C62B74 [HostId]: oss-cn-shanghai-internal.aliyuncs.com [ErrorMessage]: [E1010]HTTP/1.1 403 Forbidden ...","title":"JindoFuse \u5feb\u901f\u5165\u95e8"},{"location":"jindofuse/jindofuse_quickstart/#jindofuse","text":"","title":"JindoFuse \u4f7f\u7528\u6307\u5357"},{"location":"jindofuse/jindofuse_quickstart/#_1","text":"OSS/OSS-HDFS \u53ef\u4ee5\u901a\u8fc7 JindoFuse \u63d0\u4f9b POSIX \u652f\u6301\uff0c\u5c06 OSS/OSS-HDFS \u4e0a\u7684\u6587\u4ef6\u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c OSS/OSS-HDFS \u4e0a\u7684\u6587\u4ef6\u3002","title":"\u6982\u8ff0"},{"location":"jindofuse/jindofuse_quickstart/#_2","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.y.z.tar.gz\uff08x.y.z\u8868\u7248\u672c\u53f7\uff09\uff0c \u4e0b\u8f7d\u9875\u9762 \u3002 \u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u53c2\u89c1 \u6587\u6863\u94fe\u63a5","title":"\u90e8\u7f72\u5b89\u88c5"},{"location":"jindofuse/jindofuse_quickstart/#jindofuse_1","text":"\u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002 \u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a mkdir -p <mountpoint> \u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a jindo-fuse <mount_point> -ouri=[<osspath>] \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 <oss_path> \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 <mount_point> \u3002 <mount_point> \u9700\u66ff\u6362\u4e3a\u4e00\u4e2a\u672c\u5730\u8def\u5f84\u3002 <oss_path> \u9700\u66ff\u6362\u4e3a\u5f85\u6620\u5c04\u7684 OSS/OSS-HDFS \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a Bucket \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002\u5982 oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/subdir/ \u6302\u8f7d OSS \u548c OSS-HDFS \u8def\u5f84\u7684\u65b9\u5f0f\u57fa\u672c\u76f8\u540c\uff0c\u4ec5\u8def\u5f84\u4e2d\u7684 endpoint \u7565\u6709\u4e0d\u540c\u3002\u6839\u8def\u5f84\u793a\u4f8b\u53ef\u53c2\u89c1\uff1a \u300aOSS/OSS-HDFS \u5feb\u901f\u5165\u95e8\u300b \u786e\u8ba4\u6302\u8f7d\u6210\u529f\uff0cjindo-fuse \u8fdb\u7a0b\u5b58\u5728\uff0c\u4e14\u542f\u52a8\u53c2\u6570\u4e0e\u9884\u671f\u4e00\u81f4 ps -ef | grep jindo-fuse","title":"\u6302\u8f7d JindoFuse"},{"location":"jindofuse/jindofuse_quickstart/#jindofuse_2","text":"\u5982\u679c\u5c06 JindoFS \u670d\u52a1\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1 ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/","title":"\u8bbf\u95ee JindoFuse"},{"location":"jindofuse/jindofuse_quickstart/#jindofuse_3","text":"\u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point>","title":"\u5378\u8f7d JindoFuse"},{"location":"jindofuse/jindofuse_quickstart/#jindofuse_4","text":"\u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002","title":"\u81ea\u52a8\u5378\u8f7d JindoFuse"},{"location":"jindofuse/jindofuse_quickstart/#_3","text":"\u76ee\u524d JindoFuse \u5df2\u7ecf\u652f\u6301\u4ee5\u4e0b POSIX API\uff1a \u7279\u6027 \u8bf4\u660e OSS OSS-HDFS getattr() \u67e5\u8be2\u6587\u4ef6\u5c5e\u6027\uff0c\u7c7b\u4f3c ls \u2705 \u2705 mkdir() \u521b\u5efa\u76ee\u5f55\uff0c\u7c7b\u4f3c mkdir \u2705 \u2705 rmdir() \u5220\u9664\u76ee\u5f55\uff0c\u7c7b\u4f3c rm -rf \u2705 \u2705 unlink() \u5220\u9664\u6587\u4ef6\uff0c\u7c7b\u4f3c unlink \u2705 \u2705 rename() \u91cd\u547d\u540d\uff0c\u7c7b\u4f3c mv \u2705 \u2705 read() \u987a\u5e8f\u8bfb\u53d6 \u2705 \u2705 pread() \u968f\u673a\u8bfb \u2705 \u2705 write() \u987a\u5e8f\u5199 \u2705 \u2705 pwrite() \u968f\u673a\u5199 \u274c \u2705 flush() \u5237\u65b0\u5185\u5b58\u5230\u5185\u6838\u7f13\u51b2\u533a \u4ec5\u652f\u6301\u4ee5\u6700\u8ffd\u52a0\u5199\u65b9\u5f0f\u6253\u5f00\u7684\u6587\u4ef6 \u2705 fsync() \u5237\u65b0\u5185\u5b58\u5230\u78c1\u76d8 \u4ec5\u652f\u6301\u4ee5\u6700\u8ffd\u52a0\u5199\u65b9\u5f0f\u6253\u5f00\u7684\u6587\u4ef6 \u2705 release() \u5173\u95ed\u6587\u4ef6 \u2705 \u2705 readdir() \u8bfb\u53d6\u76ee\u5f55 \u2705 \u2705 create() \u521b\u5efa\u6587\u4ef6 \u2705 \u2705 open() O_APPEND \u901a\u8fc7\u8ffd\u52a0\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 \u652f\u6301\uff0c\u4f46\u6709 \u4f7f\u7528\u9650\u5236 \u2705 open() O_TRUNC \u901a\u8fc7\u8986\u76d6\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 \u2705 \u2705 ftruncate() \u5bf9\u6253\u5f00\u7684\u6587\u4ef6\u8fdb\u884c\u622a\u65ad \u274c \u2705 truncate() \u5bf9\u672a\u6253\u5f00\u7684\u6587\u4ef6\u8fdb\u884c\u622a\u65ad\uff0c\u7c7b\u4f3c truncate -s \u274c \u2705 lseek() \u6307\u5b9a\u6253\u5f00\u6587\u4ef6\u4e2d\u7684\u8bfb\u5199\u4f4d\u7f6e\u3002 \u274c \u2705 chmod() \u4fee\u6539\u6587\u4ef6\u6743\u9650\uff0c\u7c7b\u4f3c chmod \u274c \u2705 access() \u67e5\u8be2\u6587\u4ef6\u6743\u9650 \u2705 \u2705 utimes() \u4fee\u6539\u6587\u4ef6\u7684\u5b58\u53d6\u65f6\u95f4\u548c\u66f4\u6539\u65f6\u95f4 \u274c \u2705 setxattr() \u4fee\u6539\u6587\u4ef6 xattr \u5c5e\u6027 \u274c \u2705 getxattr() \u83b7\u53d6\u6587\u4ef6 xattr \u5c5e\u6027 \u274c \u2705 listxattr() \u5217\u4e3e\u6587\u4ef6 xattr \u5c5e\u6027 \u274c \u2705 removexattr() \u5220\u9664\u6587\u4ef6 xattr \u5c5e\u6027 \u274c \u2705 lock() \u652f\u6301 posix \u9501\uff0c\u7c7b\u4f3c fcntl \u274c \u2705 fallocate() \u4e3a\u6587\u4ef6\u9884\u5206\u914d\u7269\u7406\u7a7a\u95f4 \u274c \u2705 symlink() \u521b\u5efa\u8f6f\u8fde\u63a5 \u274c \u76ee\u524d\u4ec5\u652f\u6301 OSS-HDFS \u5185\u90e8\u4f7f\u7528\uff0c\u4e14\u4e0d\u652f\u6301\u7f13\u5b58\u52a0\u901f readlink() \u8bfb\u53d6\u8f6f\u8fde\u63a5 \u274c \u76ee\u524d\u4ec5\u652f\u6301 OSS-HDFS \u5185\u90e8\u4f7f\u7528\uff0c\u4e14\u4e0d\u652f\u6301\u7f13\u5b58\u52a0\u901f","title":"\u7279\u6027\u652f\u6301"},{"location":"jindofuse/jindofuse_quickstart/#_4","text":"","title":"\u9ad8\u9636\u4f7f\u7528"},{"location":"jindofuse/jindofuse_quickstart/#_5","text":"\u53c2\u6570\u540d\u79f0 \u5fc5\u9009 \u7248\u672c \u53c2\u6570\u8bf4\u660e \u4f7f\u7528\u8303\u4f8b uri \u2713 4.3.0+ \u914d\u7f6e\u9700\u8981\u6620\u5c04\u7684 oss \u8def\u5f84\u3002\u8def\u5f84\u53ef\u4ee5\u662f\u6839\u76ee\u5f55\uff0c\u4e5f\u53ef\u4ee5\u662f\u5b50\u76ee\u5f55\u3002\u4f8b\u5982\uff1aoss://examplebucket/ \u6216 oss://examplebucket/subdir\u3002 -ouri=oss://examplebucket/ f 4.3.0+ \u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u9ed8\u8ba4\u4f7f\u7528\u5b88\u62a4\u8fdb\u7a0b\u65b9\u5f0f\u540e\u53f0\u542f\u52a8\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -f d 4.3.0+ \u4f7f\u7528 Debug \u6a21\u5f0f\uff0c\u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -d auto_unmount 4.3.0+ fuse\u8fdb\u7a0b\u9000\u51fa\u540e\u81ea\u52a8umount\u6302\u8f7d\u8282\u70b9\u3002 -oauto_unmount ro 4.3.0+ \u53ea\u8bfb\u6302\u8f7d\uff0c\u542f\u7528\u53c2\u6570\u540e\u4e0d\u5141\u8bb8\u5199\u64cd\u4f5c\u3002 -oro direct_io 4.3.0+ \u5f00\u542f\u540e\uff0c\u8bfb\u5199\u6587\u4ef6\u53ef\u4ee5\u7ed5\u8fc7page cache\u3002 -odirect_io kernel_cache 4.3.0+ \u5f00\u542f\u540e\uff0c\u5229\u7528\u5185\u6838\u7f13\u5b58\u4f18\u5316\u8bfb\u6027\u80fd\u3002 -okernel_cache auto_cache 4.3.0+ \u9ed8\u8ba4\u5f00\u542f\uff0c\u4e0ekernel_cache \u4e8c\u9009\u4e00\uff0c\u4e0ekernel_cache\u4e0d\u540c\u7684\u662f\uff0c\u5982\u679c\u6587\u4ef6\u5927\u5c0f\u6216\u4fee\u6539\u65f6\u95f4\u53d1\u751f\u53d8\u5316\uff0c\u7f13\u5b58\u5c31\u4f1a\u5931\u6548\u3002 entry_timeout 4.3.0+ \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oentry_timeout=60 attr_timeout 4.3.0+ \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oattr_timeout=60 negative_timeout 4.3.0+ \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u5931\u8d25\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -onegative_timeout=0 max_idle_threads 4.3.0+ \u9ed8\u8ba4\u503c\uff0c10\u3002\u5904\u7406\u5185\u6838\u56de\u8c03\u7684\u7a7a\u95f2\u7ebf\u7a0b\u6c60\u3002 -omax_idle_threads=10 xengine 4.3.0+ \u6253\u5f00\u7f13\u5b58 -oxengine pread 4.5.1+ \u9ed8\u8ba4\u4f7f\u7528\u987a\u5e8f\u8bfb\u3002\u6253\u5f00\u540e\uff0c\u4f7f\u7528\u968f\u673a\u8bfb\u4ee3\u66ff\u987a\u5e8f\u8bfb\uff0c\u9002\u7528\u4e8e\u968f\u673a\u8bfb\u8fdc\u591a\u4e8e\u987a\u5e8f\u8bfb\u7684\u573a\u666f\u3002 -opread no_symlink 4.5.1+ \u914d\u7f6e\u540e\uff0c\u5173\u95edsymlink\u529f\u80fd\u3002 -ono_symlink no_writeback 4.5.1+ \u914d\u7f6e\u540e\uff0c\u5173\u95edwriteback\u529f\u80fd\u3002 -ono_writeback no_flock 4.5.1+ \u914d\u7f6e\u540e\uff0c\u5173\u95edflock\u529f\u80fd\u3002 -ono_flock no_xattr 4.5.1+ \u914d\u7f6e\u540e\uff0c\u5173\u95edxttar\u529f\u80fd\u3002 -ono_xattr","title":"\u6302\u8f7d\u9009\u9879"},{"location":"jindofuse/jindofuse_quickstart/#_6","text":"\u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u8bf4\u660e logger.dir /tmp/bigboot-log \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF logger.verbose 0 \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint \u8bbf\u95ee JindoFS \u670d\u52a1\u7684\u5730\u5740\uff0c\u5982oss-cn-xxx.aliyuncs.com fs.oss.accessKeyId \u8bbf\u95ee JindoFS \u670d\u52a1\u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret \u8bbf\u95ee JindoFS \u670d\u52a1\u9700\u8981\u7684 accessKeySecret \u66f4\u591a\u53c2\u6570\u53ef\u89c1 \u300a\u5ba2\u6237\u7aef\u5e38\u7528\u914d\u7f6e\u300b \u3002 \u652f\u6301\u5c06 jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<oss_path>] -ofs.oss.endpoint=[<your_endpoint>] -ofs.oss.accessKeyId=[<your_key_id>] -ofs.oss.accessKeySecret=[<your_key_secret>]","title":"\u914d\u7f6e\u9009\u9879"},{"location":"jindofuse/jindofuse_quickstart/#_7","text":"","title":"\u5e38\u89c1\u95ee\u9898"},{"location":"jindofuse/jindofuse_quickstart/#inputoutput-error","text":"\u4e0d\u50cf\u4f7f\u7528 JindoSDK \u8c03\u7528 API \u53ef\u4ee5\u83b7\u53d6\u66f4\u4e3a\u5177\u4f53\u7684 ErrorMsg\uff0cJindoFuse \u53ea\u80fd\u663e\u793a\u64cd\u4f5c\u7cfb\u7edf\u9884\u8bbe\u7684\u9519\u8bef\u4fe1\u606f\uff0c\u6bd4\u5982\u4ee5\u4e0b\u9519\u8bef\u5c31\u975e\u5e38\u5e38\u89c1\uff1a $ ls /mnt/oss/ ls: /mnt/oss/: Input/output error \u5982\u679c\u9700\u8981\u5b9a\u4f4d\u5177\u4f53\u7684\u9519\u8bef\u539f\u56e0\uff0c\u53ef\u4ee5\u6839\u636e JindoSDK \u914d\u7f6e\u4e2d\u7684 logger.dir\uff0c\u5728\u6307\u5b9a\u8def\u5f84\u4e0b\u7684 jindosdk.log \u6587\u4ef6\u4e2d\uff0c\u5bfb\u627e\u5177\u4f53\u7684\u9519\u8bef\u3002 \u4e0b\u9762\u5c55\u793a\u7684\u8fd9\u4e2a\u9519\u8bef\u5c31\u6765\u81ea jindosdk.log \uff0c\u6839\u636e\u62a5\u9519\u4fe1\u606f\u53ef\u89c1\uff0c\u8fd9\u4e2a\u9519\u8bef\u662f\u5e38\u89c1\u7684\u9274\u6743\u95ee\u9898\u3002 EMMDD HH:mm:ss jindofs_connectivity.cpp:13] Please check your Endpoint/Bucket/RoleArn. Failed test connectivity, operation: mkdir, errMsg: [RequestId]: 618B8183343EA53531C62B74 [HostId]: oss-cn-shanghai-internal.aliyuncs.com [ErrorMessage]: [E1010]HTTP/1.1 403 Forbidden ...","title":"Input/Output error"},{"location":"jindosdk/jindosdk_cli_options/","text":"Jindo CLI \u4f7f\u7528\u6307\u5357 \u00a7 \u7528\u6cd5\u6c47\u603b \u00a7 Usage: jindo fs [-archive [-i] [-c] <path>] [-cat <src> ...] [-count [-h] <path> ...] [-cp [-f] <src> ... <dst>] [-checksum <src> ...] [-chgrp [-R] GROUP PATH...] [-chmod [-R] <MODE[,MODE]... | OCTALMODE> PATH...] [-chown [-R] [OWNER][:[GROUP]] PATH...] [-copyFromLocal [-f] <localsrc> ... <dst>] [-copyToLocal [-f] <src> ... <localdst>] [-createSnapshot <snapshotDir> [<snapshotName>]] [-checkStoragePolicy -path <path>] [-deleteSnapshot <snapshotDir> <snapshotName>] [-du [-s] <path> ...] [-decommission -hosts <hosts>] [-formatCache] [-get [-f] <src> ... <localdst>] [-getfacl [-R] <path>] [-getfattr [-R] {-n name | -d} <path>] [-getStoragePolicy -path <path>] [-ls [-R] <path>] [-listPolicies] [-load [-meta] [-data] [-s] [-m] [-R] [-replica value] <path>] [-mkdir <path>] [-mv <src> ... <dst>] [-moveFromLocal <localsrc> ... <dst>] [-moveToLocal <src> <localdst>] [-metaDiff <path>] [-put [-f] <localsrc> ... <dst>] [-rm [-f] [-r|-R] <src> ...] [-rmdir <src> ...] [-test -[defsz] <path>] [-renameSnapshot <snapshotDir> <oldName> <newName>] [-restore [-days value] <path>] [-stat [format] <path> ...] [-setStoragePolicy -path <path> -policy <policy>] [-setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>]] [-setfattr {-n name [-v value] | -x name} <path>] [-sync [-update] [-thread thread_num] <localsrc> <dst> [-touchz <path> ...] [-truncate [-w] <length> <path> ...] [-unarchive [-i] [-a] <path>] [-uncache <path>] [-unsetStoragePolicy -path <path>] FS \u5b50\u547d\u4ee4 \u00a7 archive \u00a7 jindo fs -archive oss://<bucket>/<dir> \u5c06 OSS \u4e0a\u6587\u4ef6\u8bbe\u7f6e\u4e3a\u5f52\u6863\u7c7b\u578b\uff0c\u53ef\u9009\u53c2\u6570\uff1a \u53c2\u6570 Description -i ia \u4f4e\u9891 -c coldArchive \u51b7\u5f52\u6863 jindo fs -archive -i oss://<bucket>/<dir> jindo fs -archive -c oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS cat \u00a7 jindo fs -cat oss://<bucket>/<dir> \u663e\u793a\u6587\u672c\u5185\u5bb9\uff0c\u65e0\u53ef\u9009\u53c2\u6570 \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS count \u00a7 \u663e\u793a\u6587\u4ef6\u5927\u5c0f\u4ee5\u53ca\u6587\u4ef6\u6570\u91cf\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -h | \u663e\u793a\u6587\u4ef6\u5927\u5c0f\u5355\u4f4d| jindo fs -count -h oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS cp \u00a7 \u590d\u5236\u6587\u4ef6\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u8986\u76d6| jindo fs -cp -f oss://<bucket>/<sourcedir> oss://<bucket>/<targetdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS checksum \u00a7 \u8ba1\u7b97\u6587\u4ef6\u7684CRC32 CheckSum\u5927\u5c0f jindo fs -checksum oss://<bucket>/file \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS chgrp \u00a7 \u6539\u53d8\u6587\u4ef6\u7684\u6240\u5c5e\u7fa4\u7ec4 jindo fs -chgrp <groupname> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS chmod \u00a7 \u6539\u53d8\u6587\u4ef6\u7684\u8bbf\u95ee\u6743\u9650 jindo fs -chmod <mode> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS chown \u00a7 \u6539\u53d8\u6587\u4ef6\u7684\u6240\u6709\u4eba jindo fs -chown <username> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS copyFromLocal \u00a7 \u4ece\u672c\u5730\u590d\u5236\u6587\u4ef6\u5230oss\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u8986\u76d6| jindo fs -copyFromLocal -f <localdir> oss://<bucket>/<targetdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS copyToLocal \u00a7 \u4eceoss\u590d\u5236\u6587\u4ef6\u5230\u672c\u5730\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u8986\u76d6| jindo fs -copyToLocal -f oss://<bucket>/<targetdir> <localdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS createSnapshot \u00a7 \u521b\u5efa\u5feb\u7167\uff0c\u4f7f\u7528\u524d\u5fc5\u987b\u5148\u5141\u8bb8\u5feb\u7167\u529f\u80fd jindo fs -createSnapShot <snapshotDir> [<snapshotName>] \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS checkStoragePolicy \u00a7 \u663e\u793a\u6587\u4ef6\u7684\u5b58\u50a8\u7b56\u7565 jindo fs -checkStoragePolicy -path oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS deleteSnapshot \u00a7 \u5220\u9664\u5feb\u7167 jindo fs -deleteSnapshot oss://<bucket>/<dir> <snapshotName> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS du \u00a7 \u663e\u793a\u76ee\u5f55\u4e2d\u6240\u6709\u6587\u4ef6\u7684\u5927\u5c0f \u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -s | \u6c42\u76ee\u6807\u6587\u4ef6\u5939\u7684\u603b\u548c| | -h | \u6807\u51c6\u5355\u4f4d\u663e\u793a| jindo fs -du oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS get \u00a7 \u4e0b\u8f7d\u6587\u4ef6\u5230\u672c\u5730\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u8986\u76d6| jindo fs -get -f oss://<bucket>/<dir> <localdst> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS getfacl \u00a7 \u663e\u793a\u6587\u4ef6\u6216\u76ee\u5f55\u7684\u8bbf\u95ee\u63a7\u5236\u5217\u8868\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -R | \u9012\u5f52\u663e\u793a| jindo fs -getfacl -R oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS getfattr \u00a7 \u663e\u793a\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u7684\u6269\u5c55\u5c5e\u6027\u540d\u79f0\u548c\u503c\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -R | \u9012\u5f52\u663e\u793a| | -n | \u6309\u540d\u79f0\u663e\u793a| | -d | \u663e\u793a\u6240\u6709| jindo fs -getfattr [-R] {-n user.myAttr | -d} oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS getStoragePolicy \u00a7 \u663e\u793a\u5b58\u50a8\u7b56\u7565 jindo fs -getStoragePolicy -path oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS ls \u00a7 \u5217\u51fa\u76ee\u5f55\u4e0b\u6587\u4ef6\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -R | \u9012\u5f52\u663e\u793a| jindo fs -ls [-R] oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS listPolicies \u00a7 \u663e\u793a\u652f\u6301\u7684\u5b58\u50a8\u7b56\u7565 jindo fs -listPolicies \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS load \u00a7 \u6570\u636e\u7f13\u5b58\u547d\u4ee4\u53ef\u4ee5\u5907\u4efd\u5bf9\u5e94\u8def\u5f84\u7684\u6570\u636e\u81f3\u672c\u96c6\u7fa4\u7684\u78c1\u76d8\uff0c\u4ee5\u4fbf\u4e8e\u540e\u7eed\u53ef\u4ee5\u8bfb\u53d6\u672c\u5730\u6570\u636e\uff0c\u65e0\u9700\u8bfb\u53d6OSS\u7b49\u540e\u7aef\u4e0a\u7684\u6570\u636e\uff0c\u53ef\u9009\u53c2\u6570\u6709\u3002 | \u53c2\u6570 | \u8bf4\u660e | | --- | --- | | -s | \u8868\u793a\u7f13\u5b58\u8fc7\u7a0b\u540c\u6b65\u6267\u884c\uff0c\u6253\u5370\u8fdb\u5ea6\u548c\u6267\u884c\u4fe1\u606f\uff0c\u63a8\u8350\u5f00\u542f | | -replica | \u7f13\u5b58\u526f\u672c\u6570\u91cf\uff0c\u9ed8\u8ba4\u7f13\u5b581\u4e2a\u526f\u672c | | -R | \u9012\u5f52\u7f13\u5b58\u6587\u4ef6\uff0c\u5f53 path \u662f\u6587\u4ef6\u5939\u65f6\u5f00\u542f | jindo fs -load [-meta] [-data] [-s] [-m] [-R] [-replica value] oss://<bucket>/<dir> mkdir \u00a7 \u521b\u5efa\u6587\u4ef6\u5939 jindo fs -mkdir oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS mv \u00a7 \u79fb\u52a8\u6587\u4ef6 jindo fs -mv oss://<bucket>/<srcdir> oss://<bucket>/<dstdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS moveFromLocal \u00a7 \u5c06\u6587\u4ef6\u4ece\u672c\u5730\u79fb\u52a8\u5230\u8fdc\u7a0b\u8def\u5f84\u4e0a\uff0c\uff0c\u79fb\u52a8\u5b8c\u6210\u540e\u4f1a\u5220\u9664\u672c\u5730\u6587\u4ef6 jindo fs -moveFromLocal <localsrc> oss://<bucket>/<dstdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS moveToLocal \u00a7 \u5c06\u6587\u4ef6\u4ece\u8fdc\u7a0b\u8def\u5f84\u4e0a\u79fb\u52a8\u5230\u672c\u5730\uff0c\u79fb\u52a8\u5b8c\u6210\u540e\u4f1aOSS\u4e0a\u6587\u4ef6 jindo fs -moveToLocal oss://<bucket>/<srcdir> <localdst> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS metaDiff \u00a7 \u4f7f\u7528\u7f13\u5b58\u52a0\u901f\u7cfb\u7edf\u540e\uff0c\u663e\u793a\u672c\u5730\u5143\u6570\u636e\u548c\u8fdc\u7aef\u7684\u540c\u6b65\u60c5\u51b5 jindo fs -metaDiff oss://<bucket>/<srcdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS put \u00a7 \u5c06\u672c\u5730\u6587\u4ef6\u590d\u5236\u5230\u8fdc\u7a0b\u76ee\u5f55\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u8986\u76d6| jindo fs -put -f <localsrc> oss://<bucket>/<targetdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS rm \u00a7 \u5220\u9664\u8fdc\u7a0b\u8def\u5f84\u4e0a\u7684\u6587\u4ef6\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u5220\u9664| | -r/-rmr/-R | \u9012\u5f52\u5220\u9664| jindo fs -rm -f oss://<bucket>/<dir> jindo fs -rm -R oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS rmdir \u00a7 \u5220\u9664\u6587\u4ef6\u5939 jindo fs -rmdir oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS test \u00a7 \u57fa\u672c\u529f\u80fd\u68c0\u6d4b\uff1a | \u53c2\u6570 | Description | | --- | --- | | -d | \u662f\u5426\u662f\u6587\u4ef6\u5939| | -e | \u662f\u5426\u5b58\u5728\u8def\u5f84| | -f | \u662f\u5426\u662f\u6587\u4ef6| | -s | \u662f\u5426\u4e3a\u7a7a\u6587\u4ef6\u5939| | -z | \u6587\u4ef6\u957f\u5ea6\u662f\u5426\u4e3a0| jindo fs -test -d oss://<bucket>/<dir>/file \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS renameSnapshot \u00a7 \u4e3a\u5feb\u7167\u91cd\u547d\u540d jindo fs -renameSnapshot oss://<bucket>/<dir> <oldName> <newName> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS restore \u00a7 \u5c06\u6587\u4ef6\u4ece\u5f52\u6863\u72b6\u6001\u6062\u590d\uff0c\u51b7\u5f52\u6863\u6587\u4ef6\u548c\u5f52\u6863\u6587\u4ef6\u5fc5\u987b\u6062\u590d\u540e\u624d\u80fdunarchive\uff0c\u5fc5\u586b\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -days | \u89e3\u51bb\u7684\u65f6\u957f| jindo fs -restore -days value oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS stat \u00a7 \u663e\u793a\u6587\u4ef6\u72b6\u6001 jindo fs -stat oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS setStoragePolicy \u00a7 \u8bbe\u7f6e\u5b58\u50a8\u7b56\u7565 jindo fs -setStoragePolicy -path oss://<bucket>/<dir> -policy <policy> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS setfacl \u00a7 \u8bbe\u7f6e\u6587\u4ef6\u8bbf\u95ee\u7b56\u7565\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -b | \u5220\u9664\u6240\u6709\u8bbf\u95ee\u7b56\u7565| | -k | \u5220\u9664\u9ed8\u8ba4\u8bbf\u95ee\u7b56\u7565| | -m | \u6309\u540d\u79f0\u4fee\u6539\u8bbf\u95ee\u7b56\u7565| | -x | \u6309\u540d\u79f0\u5220\u9664\u8bbf\u95ee\u7b56\u7565| | -set | \u8bbe\u5b9a\u8bbf\u95ee\u7b56\u7565| jindo fs -setfacl -b oss://<bucket>/<dir> jindo fs -setfacl -k oss://<bucket>/<dir> jindo fs -setfacl -m user:<username>:<acl> oss://<bucket>/<dir> jindo fs -setfacl -x user:<username> oss://<bucket>/<dir> jindo fs -setfacl -set user::<acl> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS setfattr \u00a7 \u8bbe\u7f6e\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u7684\u6269\u5c55\u5c5e\u6027\u540d\u79f0\u548c\u503c\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -n | \u6309\u540d\u79f0\u8bbe\u7f6e| | -v | \u8bbe\u7f6e\u5c5e\u6027\u503c| | -x | \u6309\u540d\u79f0\u5220\u9664| jindo fs -setfattr {-n name [-v value] | -x name} oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS sync \u00a7 \u5c06\u672c\u5730\u8def\u5f84\u3001hdfs\u8def\u5f84\u3001oss\u8def\u5f84\u6216oss-hdfs\u8def\u5f84\u5230\u8fdc\u7a0b\u8def\u5f84\uff0c\u4f7f\u7528update\u53c2\u6570\u5219\u4f1a\u8df3\u8fc7\u76ee\u7684\u8def\u5f84\u4e2d\u4fee\u6539\u65f6\u95f4\u4e0d\u665a\u4e8e\u6e90\u8def\u5f84\u4e2d\u540c\u540d\u6587\u4ef6\u7684\u6587\u4ef6\u3002\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -update | \u65ad\u70b9\u7eed\u4f20 | jindo fs -sync [-update] -thread thread_num <localsrc> oss://<bucket>/<dir> jindo fs -sync [-update] -thread thread_num oss://<bucket>/<dir> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS touchz \u00a7 \u751f\u6210\u4e00\u4e2a\u5927\u5c0f\u4e3a0\u7684\u6587\u4ef6 jindo fs -touchz oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS truncate \u00a7 \u5c06\u6587\u4ef6\u88c1\u526a\u5230\u6307\u5b9a\u5927\u5c0f jindo fs -truncate <length> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS unarchive \u00a7 \u5c06\u6587\u4ef6\u6062\u590d\u4e3a\u6807\u51c6\u6587\u4ef6\uff0cAR\u548cCOLD\u7c7b\u578b\u6587\u4ef6\u9700\u8981\u5148restore jindo fs -unarchive oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS uncache \u00a7 \u5220\u9664\u7f13\u5b58\u7cfb\u7edf\u7684\u7f13\u5b58\u6587\u4ef6 jindo fs -uncache oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS unsetStoragePolicy \u00a7 \u53d6\u6d88\u5b58\u50a8\u7b56\u7565 jindo fs -unsetStoragePolicy -path oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"JindoCli \u53c2\u6570"},{"location":"jindosdk/jindosdk_cli_options/#jindo-cli","text":"","title":"Jindo CLI \u4f7f\u7528\u6307\u5357"},{"location":"jindosdk/jindosdk_cli_options/#_1","text":"Usage: jindo fs [-archive [-i] [-c] <path>] [-cat <src> ...] [-count [-h] <path> ...] [-cp [-f] <src> ... <dst>] [-checksum <src> ...] [-chgrp [-R] GROUP PATH...] [-chmod [-R] <MODE[,MODE]... | OCTALMODE> PATH...] [-chown [-R] [OWNER][:[GROUP]] PATH...] [-copyFromLocal [-f] <localsrc> ... <dst>] [-copyToLocal [-f] <src> ... <localdst>] [-createSnapshot <snapshotDir> [<snapshotName>]] [-checkStoragePolicy -path <path>] [-deleteSnapshot <snapshotDir> <snapshotName>] [-du [-s] <path> ...] [-decommission -hosts <hosts>] [-formatCache] [-get [-f] <src> ... <localdst>] [-getfacl [-R] <path>] [-getfattr [-R] {-n name | -d} <path>] [-getStoragePolicy -path <path>] [-ls [-R] <path>] [-listPolicies] [-load [-meta] [-data] [-s] [-m] [-R] [-replica value] <path>] [-mkdir <path>] [-mv <src> ... <dst>] [-moveFromLocal <localsrc> ... <dst>] [-moveToLocal <src> <localdst>] [-metaDiff <path>] [-put [-f] <localsrc> ... <dst>] [-rm [-f] [-r|-R] <src> ...] [-rmdir <src> ...] [-test -[defsz] <path>] [-renameSnapshot <snapshotDir> <oldName> <newName>] [-restore [-days value] <path>] [-stat [format] <path> ...] [-setStoragePolicy -path <path> -policy <policy>] [-setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>]] [-setfattr {-n name [-v value] | -x name} <path>] [-sync [-update] [-thread thread_num] <localsrc> <dst> [-touchz <path> ...] [-truncate [-w] <length> <path> ...] [-unarchive [-i] [-a] <path>] [-uncache <path>] [-unsetStoragePolicy -path <path>]","title":"\u7528\u6cd5\u6c47\u603b"},{"location":"jindosdk/jindosdk_cli_options/#fs","text":"","title":"FS \u5b50\u547d\u4ee4"},{"location":"jindosdk/jindosdk_cli_options/#archive","text":"jindo fs -archive oss://<bucket>/<dir> \u5c06 OSS \u4e0a\u6587\u4ef6\u8bbe\u7f6e\u4e3a\u5f52\u6863\u7c7b\u578b\uff0c\u53ef\u9009\u53c2\u6570\uff1a \u53c2\u6570 Description -i ia \u4f4e\u9891 -c coldArchive \u51b7\u5f52\u6863 jindo fs -archive -i oss://<bucket>/<dir> jindo fs -archive -c oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS","title":"archive"},{"location":"jindosdk/jindosdk_cli_options/#cat","text":"jindo fs -cat oss://<bucket>/<dir> \u663e\u793a\u6587\u672c\u5185\u5bb9\uff0c\u65e0\u53ef\u9009\u53c2\u6570 \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"cat"},{"location":"jindosdk/jindosdk_cli_options/#count","text":"\u663e\u793a\u6587\u4ef6\u5927\u5c0f\u4ee5\u53ca\u6587\u4ef6\u6570\u91cf\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -h | \u663e\u793a\u6587\u4ef6\u5927\u5c0f\u5355\u4f4d| jindo fs -count -h oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"count"},{"location":"jindosdk/jindosdk_cli_options/#cp","text":"\u590d\u5236\u6587\u4ef6\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u8986\u76d6| jindo fs -cp -f oss://<bucket>/<sourcedir> oss://<bucket>/<targetdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"cp"},{"location":"jindosdk/jindosdk_cli_options/#checksum","text":"\u8ba1\u7b97\u6587\u4ef6\u7684CRC32 CheckSum\u5927\u5c0f jindo fs -checksum oss://<bucket>/file \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"checksum"},{"location":"jindosdk/jindosdk_cli_options/#chgrp","text":"\u6539\u53d8\u6587\u4ef6\u7684\u6240\u5c5e\u7fa4\u7ec4 jindo fs -chgrp <groupname> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"chgrp"},{"location":"jindosdk/jindosdk_cli_options/#chmod","text":"\u6539\u53d8\u6587\u4ef6\u7684\u8bbf\u95ee\u6743\u9650 jindo fs -chmod <mode> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"chmod"},{"location":"jindosdk/jindosdk_cli_options/#chown","text":"\u6539\u53d8\u6587\u4ef6\u7684\u6240\u6709\u4eba jindo fs -chown <username> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"chown"},{"location":"jindosdk/jindosdk_cli_options/#copyfromlocal","text":"\u4ece\u672c\u5730\u590d\u5236\u6587\u4ef6\u5230oss\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u8986\u76d6| jindo fs -copyFromLocal -f <localdir> oss://<bucket>/<targetdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"copyFromLocal"},{"location":"jindosdk/jindosdk_cli_options/#copytolocal","text":"\u4eceoss\u590d\u5236\u6587\u4ef6\u5230\u672c\u5730\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u8986\u76d6| jindo fs -copyToLocal -f oss://<bucket>/<targetdir> <localdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"copyToLocal"},{"location":"jindosdk/jindosdk_cli_options/#createsnapshot","text":"\u521b\u5efa\u5feb\u7167\uff0c\u4f7f\u7528\u524d\u5fc5\u987b\u5148\u5141\u8bb8\u5feb\u7167\u529f\u80fd jindo fs -createSnapShot <snapshotDir> [<snapshotName>] \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"createSnapshot"},{"location":"jindosdk/jindosdk_cli_options/#checkstoragepolicy","text":"\u663e\u793a\u6587\u4ef6\u7684\u5b58\u50a8\u7b56\u7565 jindo fs -checkStoragePolicy -path oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"checkStoragePolicy"},{"location":"jindosdk/jindosdk_cli_options/#deletesnapshot","text":"\u5220\u9664\u5feb\u7167 jindo fs -deleteSnapshot oss://<bucket>/<dir> <snapshotName> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"deleteSnapshot"},{"location":"jindosdk/jindosdk_cli_options/#du","text":"\u663e\u793a\u76ee\u5f55\u4e2d\u6240\u6709\u6587\u4ef6\u7684\u5927\u5c0f \u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -s | \u6c42\u76ee\u6807\u6587\u4ef6\u5939\u7684\u603b\u548c| | -h | \u6807\u51c6\u5355\u4f4d\u663e\u793a| jindo fs -du oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"du"},{"location":"jindosdk/jindosdk_cli_options/#get","text":"\u4e0b\u8f7d\u6587\u4ef6\u5230\u672c\u5730\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u8986\u76d6| jindo fs -get -f oss://<bucket>/<dir> <localdst> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"get"},{"location":"jindosdk/jindosdk_cli_options/#getfacl","text":"\u663e\u793a\u6587\u4ef6\u6216\u76ee\u5f55\u7684\u8bbf\u95ee\u63a7\u5236\u5217\u8868\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -R | \u9012\u5f52\u663e\u793a| jindo fs -getfacl -R oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"getfacl"},{"location":"jindosdk/jindosdk_cli_options/#getfattr","text":"\u663e\u793a\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u7684\u6269\u5c55\u5c5e\u6027\u540d\u79f0\u548c\u503c\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -R | \u9012\u5f52\u663e\u793a| | -n | \u6309\u540d\u79f0\u663e\u793a| | -d | \u663e\u793a\u6240\u6709| jindo fs -getfattr [-R] {-n user.myAttr | -d} oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"getfattr"},{"location":"jindosdk/jindosdk_cli_options/#getstoragepolicy","text":"\u663e\u793a\u5b58\u50a8\u7b56\u7565 jindo fs -getStoragePolicy -path oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"getStoragePolicy"},{"location":"jindosdk/jindosdk_cli_options/#ls","text":"\u5217\u51fa\u76ee\u5f55\u4e0b\u6587\u4ef6\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -R | \u9012\u5f52\u663e\u793a| jindo fs -ls [-R] oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"ls"},{"location":"jindosdk/jindosdk_cli_options/#listpolicies","text":"\u663e\u793a\u652f\u6301\u7684\u5b58\u50a8\u7b56\u7565 jindo fs -listPolicies \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"listPolicies"},{"location":"jindosdk/jindosdk_cli_options/#load","text":"\u6570\u636e\u7f13\u5b58\u547d\u4ee4\u53ef\u4ee5\u5907\u4efd\u5bf9\u5e94\u8def\u5f84\u7684\u6570\u636e\u81f3\u672c\u96c6\u7fa4\u7684\u78c1\u76d8\uff0c\u4ee5\u4fbf\u4e8e\u540e\u7eed\u53ef\u4ee5\u8bfb\u53d6\u672c\u5730\u6570\u636e\uff0c\u65e0\u9700\u8bfb\u53d6OSS\u7b49\u540e\u7aef\u4e0a\u7684\u6570\u636e\uff0c\u53ef\u9009\u53c2\u6570\u6709\u3002 | \u53c2\u6570 | \u8bf4\u660e | | --- | --- | | -s | \u8868\u793a\u7f13\u5b58\u8fc7\u7a0b\u540c\u6b65\u6267\u884c\uff0c\u6253\u5370\u8fdb\u5ea6\u548c\u6267\u884c\u4fe1\u606f\uff0c\u63a8\u8350\u5f00\u542f | | -replica | \u7f13\u5b58\u526f\u672c\u6570\u91cf\uff0c\u9ed8\u8ba4\u7f13\u5b581\u4e2a\u526f\u672c | | -R | \u9012\u5f52\u7f13\u5b58\u6587\u4ef6\uff0c\u5f53 path \u662f\u6587\u4ef6\u5939\u65f6\u5f00\u542f | jindo fs -load [-meta] [-data] [-s] [-m] [-R] [-replica value] oss://<bucket>/<dir>","title":"load"},{"location":"jindosdk/jindosdk_cli_options/#mkdir","text":"\u521b\u5efa\u6587\u4ef6\u5939 jindo fs -mkdir oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"mkdir"},{"location":"jindosdk/jindosdk_cli_options/#mv","text":"\u79fb\u52a8\u6587\u4ef6 jindo fs -mv oss://<bucket>/<srcdir> oss://<bucket>/<dstdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"mv"},{"location":"jindosdk/jindosdk_cli_options/#movefromlocal","text":"\u5c06\u6587\u4ef6\u4ece\u672c\u5730\u79fb\u52a8\u5230\u8fdc\u7a0b\u8def\u5f84\u4e0a\uff0c\uff0c\u79fb\u52a8\u5b8c\u6210\u540e\u4f1a\u5220\u9664\u672c\u5730\u6587\u4ef6 jindo fs -moveFromLocal <localsrc> oss://<bucket>/<dstdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"moveFromLocal"},{"location":"jindosdk/jindosdk_cli_options/#movetolocal","text":"\u5c06\u6587\u4ef6\u4ece\u8fdc\u7a0b\u8def\u5f84\u4e0a\u79fb\u52a8\u5230\u672c\u5730\uff0c\u79fb\u52a8\u5b8c\u6210\u540e\u4f1aOSS\u4e0a\u6587\u4ef6 jindo fs -moveToLocal oss://<bucket>/<srcdir> <localdst> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"moveToLocal"},{"location":"jindosdk/jindosdk_cli_options/#metadiff","text":"\u4f7f\u7528\u7f13\u5b58\u52a0\u901f\u7cfb\u7edf\u540e\uff0c\u663e\u793a\u672c\u5730\u5143\u6570\u636e\u548c\u8fdc\u7aef\u7684\u540c\u6b65\u60c5\u51b5 jindo fs -metaDiff oss://<bucket>/<srcdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"metaDiff"},{"location":"jindosdk/jindosdk_cli_options/#put","text":"\u5c06\u672c\u5730\u6587\u4ef6\u590d\u5236\u5230\u8fdc\u7a0b\u76ee\u5f55\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u8986\u76d6| jindo fs -put -f <localsrc> oss://<bucket>/<targetdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"put"},{"location":"jindosdk/jindosdk_cli_options/#rm","text":"\u5220\u9664\u8fdc\u7a0b\u8def\u5f84\u4e0a\u7684\u6587\u4ef6\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u5220\u9664| | -r/-rmr/-R | \u9012\u5f52\u5220\u9664| jindo fs -rm -f oss://<bucket>/<dir> jindo fs -rm -R oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"rm"},{"location":"jindosdk/jindosdk_cli_options/#rmdir","text":"\u5220\u9664\u6587\u4ef6\u5939 jindo fs -rmdir oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"rmdir"},{"location":"jindosdk/jindosdk_cli_options/#test","text":"\u57fa\u672c\u529f\u80fd\u68c0\u6d4b\uff1a | \u53c2\u6570 | Description | | --- | --- | | -d | \u662f\u5426\u662f\u6587\u4ef6\u5939| | -e | \u662f\u5426\u5b58\u5728\u8def\u5f84| | -f | \u662f\u5426\u662f\u6587\u4ef6| | -s | \u662f\u5426\u4e3a\u7a7a\u6587\u4ef6\u5939| | -z | \u6587\u4ef6\u957f\u5ea6\u662f\u5426\u4e3a0| jindo fs -test -d oss://<bucket>/<dir>/file \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"test"},{"location":"jindosdk/jindosdk_cli_options/#renamesnapshot","text":"\u4e3a\u5feb\u7167\u91cd\u547d\u540d jindo fs -renameSnapshot oss://<bucket>/<dir> <oldName> <newName> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"renameSnapshot"},{"location":"jindosdk/jindosdk_cli_options/#restore","text":"\u5c06\u6587\u4ef6\u4ece\u5f52\u6863\u72b6\u6001\u6062\u590d\uff0c\u51b7\u5f52\u6863\u6587\u4ef6\u548c\u5f52\u6863\u6587\u4ef6\u5fc5\u987b\u6062\u590d\u540e\u624d\u80fdunarchive\uff0c\u5fc5\u586b\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -days | \u89e3\u51bb\u7684\u65f6\u957f| jindo fs -restore -days value oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"restore"},{"location":"jindosdk/jindosdk_cli_options/#stat","text":"\u663e\u793a\u6587\u4ef6\u72b6\u6001 jindo fs -stat oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"stat"},{"location":"jindosdk/jindosdk_cli_options/#setstoragepolicy","text":"\u8bbe\u7f6e\u5b58\u50a8\u7b56\u7565 jindo fs -setStoragePolicy -path oss://<bucket>/<dir> -policy <policy> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"setStoragePolicy"},{"location":"jindosdk/jindosdk_cli_options/#setfacl","text":"\u8bbe\u7f6e\u6587\u4ef6\u8bbf\u95ee\u7b56\u7565\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -b | \u5220\u9664\u6240\u6709\u8bbf\u95ee\u7b56\u7565| | -k | \u5220\u9664\u9ed8\u8ba4\u8bbf\u95ee\u7b56\u7565| | -m | \u6309\u540d\u79f0\u4fee\u6539\u8bbf\u95ee\u7b56\u7565| | -x | \u6309\u540d\u79f0\u5220\u9664\u8bbf\u95ee\u7b56\u7565| | -set | \u8bbe\u5b9a\u8bbf\u95ee\u7b56\u7565| jindo fs -setfacl -b oss://<bucket>/<dir> jindo fs -setfacl -k oss://<bucket>/<dir> jindo fs -setfacl -m user:<username>:<acl> oss://<bucket>/<dir> jindo fs -setfacl -x user:<username> oss://<bucket>/<dir> jindo fs -setfacl -set user::<acl> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"setfacl"},{"location":"jindosdk/jindosdk_cli_options/#setfattr","text":"\u8bbe\u7f6e\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u7684\u6269\u5c55\u5c5e\u6027\u540d\u79f0\u548c\u503c\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -n | \u6309\u540d\u79f0\u8bbe\u7f6e| | -v | \u8bbe\u7f6e\u5c5e\u6027\u503c| | -x | \u6309\u540d\u79f0\u5220\u9664| jindo fs -setfattr {-n name [-v value] | -x name} oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"setfattr"},{"location":"jindosdk/jindosdk_cli_options/#sync","text":"\u5c06\u672c\u5730\u8def\u5f84\u3001hdfs\u8def\u5f84\u3001oss\u8def\u5f84\u6216oss-hdfs\u8def\u5f84\u5230\u8fdc\u7a0b\u8def\u5f84\uff0c\u4f7f\u7528update\u53c2\u6570\u5219\u4f1a\u8df3\u8fc7\u76ee\u7684\u8def\u5f84\u4e2d\u4fee\u6539\u65f6\u95f4\u4e0d\u665a\u4e8e\u6e90\u8def\u5f84\u4e2d\u540c\u540d\u6587\u4ef6\u7684\u6587\u4ef6\u3002\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -update | \u65ad\u70b9\u7eed\u4f20 | jindo fs -sync [-update] -thread thread_num <localsrc> oss://<bucket>/<dir> jindo fs -sync [-update] -thread thread_num oss://<bucket>/<dir> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"sync"},{"location":"jindosdk/jindosdk_cli_options/#touchz","text":"\u751f\u6210\u4e00\u4e2a\u5927\u5c0f\u4e3a0\u7684\u6587\u4ef6 jindo fs -touchz oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"touchz"},{"location":"jindosdk/jindosdk_cli_options/#truncate","text":"\u5c06\u6587\u4ef6\u88c1\u526a\u5230\u6307\u5b9a\u5927\u5c0f jindo fs -truncate <length> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"truncate"},{"location":"jindosdk/jindosdk_cli_options/#unarchive","text":"\u5c06\u6587\u4ef6\u6062\u590d\u4e3a\u6807\u51c6\u6587\u4ef6\uff0cAR\u548cCOLD\u7c7b\u578b\u6587\u4ef6\u9700\u8981\u5148restore jindo fs -unarchive oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"unarchive"},{"location":"jindosdk/jindosdk_cli_options/#uncache","text":"\u5220\u9664\u7f13\u5b58\u7cfb\u7edf\u7684\u7f13\u5b58\u6587\u4ef6 jindo fs -uncache oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"uncache"},{"location":"jindosdk/jindosdk_cli_options/#unsetstoragepolicy","text":"\u53d6\u6d88\u5b58\u50a8\u7b56\u7565 jindo fs -unsetStoragePolicy -path oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"unsetStoragePolicy"},{"location":"jindosdk/jindosdk_configuration/","text":"\u5ba2\u6237\u7aef\u5e38\u7528\u914d\u7f6e \u00a7 OSS/OSS-HDFS \u00a7 \u65e5\u5fd7\u914d\u7f6e\u9879 \u00a7 \u914d\u7f6e\u9879 \u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e \u7248\u672c logger.dir \u5b57\u7b26\u4e32 /tmp/bigboot-log \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa\uff0clogger.consolelogger \u548c logger.jnilogger \u5747\u4e3a false \u65f6\u751f\u6548\u3002 4.3.0+ logger.sync \u5e03\u5c14\u503c false \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa 4.3.0+ logger.consolelogger \u5e03\u5c14\u503c false \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef 4.3.0+ logger.jnilogger \u5e03\u5c14\u503c false \u6253\u5370\u65e5\u5fd7\u5230log4j 4.3.0+ logger.level \u6574\u578b 2 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 4.3.0+ logger.verbose \u6574\u578b 0 \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa 4.3.0+ logger.cleaner.enable \u5e03\u5c14\u503c false \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 4.3.0+ Java SDK \u8bbe\u7f6e log4j.properties \u00a7 log4j.logger.com.aliyun.jindodata=INFO log4j.logger.com.aliyun.jindodata.common.FsStats=INFO I/O \u76f8\u5173\u914d\u7f6e\u9879 \u00a7 \u914d\u7f6e\u9879 \u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e \u7248\u672c fs.oss.tmp.data.dirs \u5b57\u7b26\u4e32 /tmp/ \u5ba2\u6237\u7aef\u5199\u5165\u65f6\u7684\u4e34\u65f6\u6587\u4ef6\u76ee\u5f55\uff0c\u53ef\u914d\u7f6e\u591a\u4e2a\uff08\u9017\u53f7\u9694\u5f00\uff09\uff0c\u4f1a\u8f6e\u6d41\u5199\u5165\uff0c\u591a\u7528\u6237\u73af\u5883\u9700\u914d\u7f6e\u53ef\u8bfb\u5199\u6743\u9650 4.3.0+ fs.oss.tmp.data.cleaner.enable \u5e03\u5c14\u503c true \u4e34\u65f6\u6587\u4ef6\u81ea\u6e05\u7406\u670d\u52a1 4.3.0+ fs.oss.retry.count \u6574\u578b 5 \u8bbf\u95ee OSS/OSS-HDFS \u5931\u8d25\u91cd\u8bd5\u6b21\u6570 4.3.0+ fs.oss.retry.interval.millisecond \u6574\u578b 500 \u8bbf\u95ee OSS/OSS-HDFS \u5931\u8d25\u91cd\u8bd5\u95f4\u9694\uff08\u6beb\u79d2\uff09 4.3.0+ fs.oss.timeout.millisecond \u6574\u578b 30000 \u8bf7\u6c42 OSS/OSS-HDFS \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 4.3.0+ fs.oss.connection.timeout.millisecond \u6574\u578b 3000 \u8fde\u63a5 OSS/OSS-HDFS \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 4.3.0+ fs.oss.max.connections.per.host \u6574\u578b 100 \u8fde\u63a5 OSS/OSS-HDFS \u7684\u8fde\u63a5\u6c60\u5bf9\u6bcf\u4e2ahost\u7684\u6700\u5927\u8fde\u63a5\u6570\uff08\u8d85\u8fc7\u9608\u503c\u5916\u7684\u8fde\u63a5\u4f1a\u4f7f\u7528\u77ed\u8fde\u63a5\uff09 4.3.0+ fs.oss.upload.thread.concurrency \u6574\u578b MAX(cpu\u6838\u6570,16) \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u5e76\u53d1\u4e0a\u4f20\u7ebf\u7a0b\u6570 4.3.0+(deprecated) fs.oss.upload.queue.size \u6574\u578b MAX(cpu\u6838\u6570,16) \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u961f\u5217\u5927\u5c0f 4.3.0+(deprecated) fs.oss.upload.max.pending.tasks.per.stream \u6574\u578b 10 \u5355\u4e2a\u6587\u4ef6 OSS/OSS-HDFS \u6700\u5927\u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u6570 4.3.0+(deprecated) fs.oss.download.thread.concurrency \u6574\u578b MAX(cpu\u6838\u6570,16) \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u6700\u5927\u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u6570 4.3.0+(deprecated) fs.oss.read.readahead.max.buffer.count \u6574\u578b 48 \u6700\u5927\u540c\u65f6\u9884\u8bfb OSS/OSS-HDFS \u7684 buffer \u4e2a\u6570 4.3.0+(deprecated) fs.oss.read.buffer.size \u6574\u578b 1048576 OSS/OSS-HDFS \u8bfb\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 4.3.0+ fs.oss.write.buffer.size \u6574\u578b 1048576 OSS/OSS-HDFS \u5199\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 4.3.0+ fs.oss.flush.interval.millisecond \u6574\u578b -1 OSS/OSS-HDFS \u5237\u65b0\u7f13\u51b2\u533a\u95f4\u9694\uff08\u6beb\u79d2\uff09\uff0c\u5c0f\u4e8e 0 \u65f6\u4e0d\u751f\u6548 4.3.0+ fs.oss.blocklet.size.mb \u6574\u578b 8 OSS \u5206\u5757\u4e0a\u4f20\u65f6\u7684\u5757\u5927\u5c0f\uff08MB\uff09\u7531\u4e8e\u5206\u5757\u6570\u91cf\u6700\u591a\u4e3a10000\u5757\uff0c\u56e0\u6b64\u9ed8\u8ba4\u5199\u5165\u6587\u4ef6\u4e0d\u80fd\u8d85\u8fc780GB\u3002\u5982\u679c\u6709\u4e2a\u522b\u6587\u4ef6\u8d85\u8fc780G\uff0c\u5efa\u8bae\u6839\u636e\u6587\u4ef6\u5927\u5c0f\u5355\u72ec\u8c03\u5927\u672c\u914d\u7f6e\uff0c\u5e76\u540c\u65f6\u8c03\u5927\u8bf7\u6c42 oss \u7684\u8d85\u65f6\u65f6\u95f4\u3002\u5982\u6587\u4ef6\u5927\u5c0f\u672a\u77e5\uff0c\u6216\u8005\u8fdc\u8fdc\u8d85\u8fc780G\uff08\u5982\u8d85\u8fc7160G\uff09\uff0c\u5efa\u8bae\u8003\u8651\u4f7f\u7528 OSS-HDFS\uff08\u65e0\u6587\u4ef6\u5927\u5c0f\u9650\u5236\uff09 4.5.2+ fs.oss.checksum.crc64.enable \u5e03\u5c14\u503c true \u6587\u4ef6\u7ea7\u522b crc64 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u76ee\u524d\u5bf9\u5199OSS-HDFS\u6027\u80fd\u6709\u8f83\u5927\u5f71\u54cd\uff0c\u6027\u80fd\u4f18\u5148\u573a\u666f\u53ef\u4ee5\u8003\u8651\u5173\u95ed\u3002 4.6.0+ fs.oss.checksum.md5.enable \u5e03\u5c14\u503c false \u8bf7\u6c42\u7ea7\u522b md5 \u5b8c\u6574\u6027\u6821\u9a8c 4.6.0+ fs.oss.read.readahead.prefetcher.version \u5b57\u7b26\u4e32 default \u53ef\u9009\u503c\uff1a legacy \u539f\u9884\u8bfb\u7b97\u6cd5, default \u65b0\u9884\u8bfb\u7b97\u6cd5\u3002\u65b0\u9884\u8bfb\u7b97\u6cd5\u53ef\u80fd\u4f7f\u7528\u66f4\u591a\u5185\u5b58\u3002\u82e5\u914d\u7f6e\u4e86\u65b0\u9884\u8bfb\u7b97\u6cd5\u540e\u53d1\u751f\u6027\u80fd\u4e0b\u964d\uff0c\u53ef\u80fd\u662f\u7531\u4e8e\u5185\u5b58\u6c60\u5bb9\u91cf\u4e0d\u8db3\u5bfc\u81f4\u9884\u8bfb\u7684\u5757\u5728\u88ab\u8bbf\u95ee\u5230\u4e4b\u524d\u5c31\u88ab\u63d0\u524d\u9010\u51fa\u3002\u4e3a\u4e86\u907f\u514d\u8be5\u60c5\u51b5\u53d1\u751f\uff0c\u53ef\u4ee5\u8003\u8651\u7f29\u51cf\u6700\u5927\u9884\u8bfb\u957f\u5ea6\uff0c\u6216\u5141\u8bb8\u9884\u8bfb\u4f7f\u7528\u66f4\u591a\u5185\u5b58\u3002 6.2.0+ fs.oss.read.readahead.prefetch.size.max \u6574\u578b 268435456 \u9884\u8bfb\u6700\u5927\u957f\u5ea6\uff08\u5355\u4f4d\uff1abyte\uff09 6.2.0+ fs.oss.signer.version \u6574\u578b 0 \u7b7e\u540d\u7b97\u6cd5\u7248\u672c\uff0c\u53ef\u9009\u503c[0\uff0c1\uff0c4]\u3002\u63a8\u8350\u4f7f\u75280\uff0c\u8868\u793a\u9ed8\u8ba4\u7b97\u6cd5\u7248\u672c\uff0c6.3.0\u7248\u672c\u4e4b\u540e\u9ed8\u8ba4\u4f7f\u7528 V4 \u7b7e\u540d\u7b97\u6cd5\u63d0\u9ad8\u8bbf\u95ee OSS/OSS-HDFS\uff0c\u82e5\u7279\u6b8a\u573a\u666f\u5e0c\u671b\u6307\u5b9a\u7b7e\u540d\u7b97\u6cd5\u7248\u672c\uff0c\u8bf7\u6307\u5b9a 1 \u6216 4\u3002 6.3.0+ fs.oss.io.timeout.millisecond \u6574\u578b 90000 \u8bf7\u6c42 OSS/OSS-HDFS \u8bfb\u5199\u64cd\u4f5c\u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 6.6.0+ nextarch fs.oss.async.executor.number \u6574\u578b cpu\u6838\u6570 \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u5f02\u6b65io\u7ebf\u7a0b\u6570 6.6.0+ nextarch fs.oss.upload.async.concurrency \u6574\u578b MAX(cpu\u6838\u6570,16) \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u5f02\u6b65\u4e0a\u4f20\u5e76\u53d1\u6570 6.6.0+ nextarch fs.oss.download.async.concurrency \u6574\u578b MAX(cpu\u6838\u6570,16) \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u5f02\u6b65\u4e0b\u8f7d\u5e76\u53d1\u6570 6.6.0+ nextarch fs.oss.array.block.enable \u5e03\u5c14\u503c false \u9ed8\u8ba4\u5173\u95ed\u3002\u6253\u5f00\u540e\u3002\u5ba2\u6237\u7aef\u5199\u5165\u4f18\u5148\u4f7f\u7528\u5185\u5b58 buffer\uff0c\u5355\u4e2abuffer\u5927\u5c0f\u4e0e fs.oss.blocklet.size.mb \u5bf9\u9f50\uff0c\u82e5\u5185\u5b58\u4e0d\u8db3\uff0c\u5219\u5199\u5165\u78c1\u76d8\u3002\u4e0d\u5efa\u8bae\u5728\u5199\u5927\u6587\u4ef6\uff08>8M\uff09\u573a\u666f\u4f7f\u7528\uff0c\u53ef\u80fd\u4f1a\u9020\u6210\u5185\u5b58\u4e0d\u8db3\u3002 6.7.0+ nextarch \u5185\u5b58\u76f8\u5173\u914d\u7f6e\u9879 \u00a7 \u914d\u7f6e\u9879 \u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e \u7248\u672c fs.oss.memory.buffer.size.max.mb \u6574\u578b 6124 \u5185\u5b58\u6c60\u603b\u5bb9\u91cf\uff08\u5355\u4f4d\uff1aMB\uff09 4.3.0+(deprecated) fs.oss.memory.buffer.size.watermark \u6d6e\u70b9\u578b 0.3 \u5185\u5b58\u6c60\u7528\u4e8e\u9884\u8bfb\u7684\u5bb9\u91cf\u6bd4\u4f8b 4.3.0+(deprecated) fs.jdo.memory.pool.size.max.mb \u6574\u578b 6124 \u5185\u5b58\u6c60\u603b\u5bb9\u91cf\uff08\u5355\u4f4d\uff1aMB\uff09 6.7.0+ nextarch fs.jdo.memory.io.buffer.size.max.ratio \u6d6e\u70b9\u578b 0.8 \u5185\u5b58\u6c60\u6700\u5927\u7528\u4e8eIO buffer\u7684\u5bb9\u91cf\u6bd4\u4f8b 6.7.0+ nextarch Metrics \u76f8\u5173\u914d\u7f6e\u9879 \u00a7 \u914d\u7f6e\u9879 \u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e \u7248\u672c fs.jdo.metrics.level \u6574\u578b 1 mertrics \u6536\u96c6\u7b49\u7ea7\uff0c0\u662f\u4e0d\u6536\u96c6\uff0c1\u662f\u9759\u6001metrics\uff0c2\u662f\u5305\u542b bucket \u4fe1\u606f\u7684\u52a8\u6001 metrics\uff0c3\u5305\u542b\u6240\u6709 metrics 6.6.0+ nextarch fs.jdo.metrics.file.enable \u5e03\u5c14\u503c false \u63a7\u5236\u662f\u5426\u8f93\u51fa metrics \u5230\u6587\u4ef6 6.6.0+ nextarch fs.jdo.metrics.file.dir \u5b57\u7b26\u4e32 /tmp/metrics \u8f93\u51fa metrics \u5230\u6587\u4ef6\u7684\u6240\u5728\u76ee\u5f55 6.6.0+ nextarch fs.jdo.metrics.file.pid.append.enable \u5e03\u5c14\u503c false \u63a7\u5236\u8f93\u51fa metrics \u5230\u6587\u4ef6\u7684\u65b9\u5f0f\uff0ctrue \u4ee3\u8868\u4ee5 pid \u4f5c\u4e3a\u5b50\u76ee\u5f55\uff0c\u76ee\u5f55\u5c42\u7ea7\u4e3a(file_dir)/pid/jindosdk_(timestamp).metrics 6.7.0+ nextarch fs.jdo.metrics.file.number \u6574\u578b 16 \u63a7\u5236\u5355\u4e2a\u76ee\u5f55\u4e0b\u6700\u5927\u6587\u4ef6\u7684\u4e2a\u6570 6.6.0+ nextarch fs.jdo.metrics.interval.sec \u6574\u578b 15 \u8f93\u51fa metrics \u5230\u6587\u4ef6\u7684\u95f4\u9694\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 6.6.0+ nextarch fs.jdo.metrics.file.clean.enable \u5e03\u5c14\u503c false \u63a7\u5236\u662f\u5426\u81ea\u52a8\u6e05\u7406\u8f93\u51fa metrics \u7684\u6587\u4ef6, true \u4ee3\u8868\u6253\u5f00\u6e05\u7406\uff0c\u9ed8\u8ba424h\u6e05\u7406\u4e00\u6b21 6.6.0+ nextarch Credential Provider \u76f8\u5173\u914d\u7f6e\uff0c\u8be6\u89c1 \u300a\u914d\u7f6e OSS/OSS-HDFS Credential Provider\u300b","title":"\u914d\u7f6e"},{"location":"jindosdk/jindosdk_configuration/#_1","text":"","title":"\u5ba2\u6237\u7aef\u5e38\u7528\u914d\u7f6e"},{"location":"jindosdk/jindosdk_configuration/#ossoss-hdfs","text":"","title":"OSS/OSS-HDFS"},{"location":"jindosdk/jindosdk_configuration/#_2","text":"\u914d\u7f6e\u9879 \u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e \u7248\u672c logger.dir \u5b57\u7b26\u4e32 /tmp/bigboot-log \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa\uff0clogger.consolelogger \u548c logger.jnilogger \u5747\u4e3a false \u65f6\u751f\u6548\u3002 4.3.0+ logger.sync \u5e03\u5c14\u503c false \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa 4.3.0+ logger.consolelogger \u5e03\u5c14\u503c false \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef 4.3.0+ logger.jnilogger \u5e03\u5c14\u503c false \u6253\u5370\u65e5\u5fd7\u5230log4j 4.3.0+ logger.level \u6574\u578b 2 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 4.3.0+ logger.verbose \u6574\u578b 0 \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa 4.3.0+ logger.cleaner.enable \u5e03\u5c14\u503c false \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 4.3.0+","title":"\u65e5\u5fd7\u914d\u7f6e\u9879"},{"location":"jindosdk/jindosdk_configuration/#java-sdk-log4jproperties","text":"log4j.logger.com.aliyun.jindodata=INFO log4j.logger.com.aliyun.jindodata.common.FsStats=INFO","title":"Java SDK \u8bbe\u7f6e log4j.properties"},{"location":"jindosdk/jindosdk_configuration/#io","text":"\u914d\u7f6e\u9879 \u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e \u7248\u672c fs.oss.tmp.data.dirs \u5b57\u7b26\u4e32 /tmp/ \u5ba2\u6237\u7aef\u5199\u5165\u65f6\u7684\u4e34\u65f6\u6587\u4ef6\u76ee\u5f55\uff0c\u53ef\u914d\u7f6e\u591a\u4e2a\uff08\u9017\u53f7\u9694\u5f00\uff09\uff0c\u4f1a\u8f6e\u6d41\u5199\u5165\uff0c\u591a\u7528\u6237\u73af\u5883\u9700\u914d\u7f6e\u53ef\u8bfb\u5199\u6743\u9650 4.3.0+ fs.oss.tmp.data.cleaner.enable \u5e03\u5c14\u503c true \u4e34\u65f6\u6587\u4ef6\u81ea\u6e05\u7406\u670d\u52a1 4.3.0+ fs.oss.retry.count \u6574\u578b 5 \u8bbf\u95ee OSS/OSS-HDFS \u5931\u8d25\u91cd\u8bd5\u6b21\u6570 4.3.0+ fs.oss.retry.interval.millisecond \u6574\u578b 500 \u8bbf\u95ee OSS/OSS-HDFS \u5931\u8d25\u91cd\u8bd5\u95f4\u9694\uff08\u6beb\u79d2\uff09 4.3.0+ fs.oss.timeout.millisecond \u6574\u578b 30000 \u8bf7\u6c42 OSS/OSS-HDFS \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 4.3.0+ fs.oss.connection.timeout.millisecond \u6574\u578b 3000 \u8fde\u63a5 OSS/OSS-HDFS \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 4.3.0+ fs.oss.max.connections.per.host \u6574\u578b 100 \u8fde\u63a5 OSS/OSS-HDFS \u7684\u8fde\u63a5\u6c60\u5bf9\u6bcf\u4e2ahost\u7684\u6700\u5927\u8fde\u63a5\u6570\uff08\u8d85\u8fc7\u9608\u503c\u5916\u7684\u8fde\u63a5\u4f1a\u4f7f\u7528\u77ed\u8fde\u63a5\uff09 4.3.0+ fs.oss.upload.thread.concurrency \u6574\u578b MAX(cpu\u6838\u6570,16) \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u5e76\u53d1\u4e0a\u4f20\u7ebf\u7a0b\u6570 4.3.0+(deprecated) fs.oss.upload.queue.size \u6574\u578b MAX(cpu\u6838\u6570,16) \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u961f\u5217\u5927\u5c0f 4.3.0+(deprecated) fs.oss.upload.max.pending.tasks.per.stream \u6574\u578b 10 \u5355\u4e2a\u6587\u4ef6 OSS/OSS-HDFS \u6700\u5927\u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u6570 4.3.0+(deprecated) fs.oss.download.thread.concurrency \u6574\u578b MAX(cpu\u6838\u6570,16) \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u6700\u5927\u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u6570 4.3.0+(deprecated) fs.oss.read.readahead.max.buffer.count \u6574\u578b 48 \u6700\u5927\u540c\u65f6\u9884\u8bfb OSS/OSS-HDFS \u7684 buffer \u4e2a\u6570 4.3.0+(deprecated) fs.oss.read.buffer.size \u6574\u578b 1048576 OSS/OSS-HDFS \u8bfb\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 4.3.0+ fs.oss.write.buffer.size \u6574\u578b 1048576 OSS/OSS-HDFS \u5199\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 4.3.0+ fs.oss.flush.interval.millisecond \u6574\u578b -1 OSS/OSS-HDFS \u5237\u65b0\u7f13\u51b2\u533a\u95f4\u9694\uff08\u6beb\u79d2\uff09\uff0c\u5c0f\u4e8e 0 \u65f6\u4e0d\u751f\u6548 4.3.0+ fs.oss.blocklet.size.mb \u6574\u578b 8 OSS \u5206\u5757\u4e0a\u4f20\u65f6\u7684\u5757\u5927\u5c0f\uff08MB\uff09\u7531\u4e8e\u5206\u5757\u6570\u91cf\u6700\u591a\u4e3a10000\u5757\uff0c\u56e0\u6b64\u9ed8\u8ba4\u5199\u5165\u6587\u4ef6\u4e0d\u80fd\u8d85\u8fc780GB\u3002\u5982\u679c\u6709\u4e2a\u522b\u6587\u4ef6\u8d85\u8fc780G\uff0c\u5efa\u8bae\u6839\u636e\u6587\u4ef6\u5927\u5c0f\u5355\u72ec\u8c03\u5927\u672c\u914d\u7f6e\uff0c\u5e76\u540c\u65f6\u8c03\u5927\u8bf7\u6c42 oss \u7684\u8d85\u65f6\u65f6\u95f4\u3002\u5982\u6587\u4ef6\u5927\u5c0f\u672a\u77e5\uff0c\u6216\u8005\u8fdc\u8fdc\u8d85\u8fc780G\uff08\u5982\u8d85\u8fc7160G\uff09\uff0c\u5efa\u8bae\u8003\u8651\u4f7f\u7528 OSS-HDFS\uff08\u65e0\u6587\u4ef6\u5927\u5c0f\u9650\u5236\uff09 4.5.2+ fs.oss.checksum.crc64.enable \u5e03\u5c14\u503c true \u6587\u4ef6\u7ea7\u522b crc64 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u76ee\u524d\u5bf9\u5199OSS-HDFS\u6027\u80fd\u6709\u8f83\u5927\u5f71\u54cd\uff0c\u6027\u80fd\u4f18\u5148\u573a\u666f\u53ef\u4ee5\u8003\u8651\u5173\u95ed\u3002 4.6.0+ fs.oss.checksum.md5.enable \u5e03\u5c14\u503c false \u8bf7\u6c42\u7ea7\u522b md5 \u5b8c\u6574\u6027\u6821\u9a8c 4.6.0+ fs.oss.read.readahead.prefetcher.version \u5b57\u7b26\u4e32 default \u53ef\u9009\u503c\uff1a legacy \u539f\u9884\u8bfb\u7b97\u6cd5, default \u65b0\u9884\u8bfb\u7b97\u6cd5\u3002\u65b0\u9884\u8bfb\u7b97\u6cd5\u53ef\u80fd\u4f7f\u7528\u66f4\u591a\u5185\u5b58\u3002\u82e5\u914d\u7f6e\u4e86\u65b0\u9884\u8bfb\u7b97\u6cd5\u540e\u53d1\u751f\u6027\u80fd\u4e0b\u964d\uff0c\u53ef\u80fd\u662f\u7531\u4e8e\u5185\u5b58\u6c60\u5bb9\u91cf\u4e0d\u8db3\u5bfc\u81f4\u9884\u8bfb\u7684\u5757\u5728\u88ab\u8bbf\u95ee\u5230\u4e4b\u524d\u5c31\u88ab\u63d0\u524d\u9010\u51fa\u3002\u4e3a\u4e86\u907f\u514d\u8be5\u60c5\u51b5\u53d1\u751f\uff0c\u53ef\u4ee5\u8003\u8651\u7f29\u51cf\u6700\u5927\u9884\u8bfb\u957f\u5ea6\uff0c\u6216\u5141\u8bb8\u9884\u8bfb\u4f7f\u7528\u66f4\u591a\u5185\u5b58\u3002 6.2.0+ fs.oss.read.readahead.prefetch.size.max \u6574\u578b 268435456 \u9884\u8bfb\u6700\u5927\u957f\u5ea6\uff08\u5355\u4f4d\uff1abyte\uff09 6.2.0+ fs.oss.signer.version \u6574\u578b 0 \u7b7e\u540d\u7b97\u6cd5\u7248\u672c\uff0c\u53ef\u9009\u503c[0\uff0c1\uff0c4]\u3002\u63a8\u8350\u4f7f\u75280\uff0c\u8868\u793a\u9ed8\u8ba4\u7b97\u6cd5\u7248\u672c\uff0c6.3.0\u7248\u672c\u4e4b\u540e\u9ed8\u8ba4\u4f7f\u7528 V4 \u7b7e\u540d\u7b97\u6cd5\u63d0\u9ad8\u8bbf\u95ee OSS/OSS-HDFS\uff0c\u82e5\u7279\u6b8a\u573a\u666f\u5e0c\u671b\u6307\u5b9a\u7b7e\u540d\u7b97\u6cd5\u7248\u672c\uff0c\u8bf7\u6307\u5b9a 1 \u6216 4\u3002 6.3.0+ fs.oss.io.timeout.millisecond \u6574\u578b 90000 \u8bf7\u6c42 OSS/OSS-HDFS \u8bfb\u5199\u64cd\u4f5c\u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 6.6.0+ nextarch fs.oss.async.executor.number \u6574\u578b cpu\u6838\u6570 \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u5f02\u6b65io\u7ebf\u7a0b\u6570 6.6.0+ nextarch fs.oss.upload.async.concurrency \u6574\u578b MAX(cpu\u6838\u6570,16) \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u5f02\u6b65\u4e0a\u4f20\u5e76\u53d1\u6570 6.6.0+ nextarch fs.oss.download.async.concurrency \u6574\u578b MAX(cpu\u6838\u6570,16) \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u5f02\u6b65\u4e0b\u8f7d\u5e76\u53d1\u6570 6.6.0+ nextarch fs.oss.array.block.enable \u5e03\u5c14\u503c false \u9ed8\u8ba4\u5173\u95ed\u3002\u6253\u5f00\u540e\u3002\u5ba2\u6237\u7aef\u5199\u5165\u4f18\u5148\u4f7f\u7528\u5185\u5b58 buffer\uff0c\u5355\u4e2abuffer\u5927\u5c0f\u4e0e fs.oss.blocklet.size.mb \u5bf9\u9f50\uff0c\u82e5\u5185\u5b58\u4e0d\u8db3\uff0c\u5219\u5199\u5165\u78c1\u76d8\u3002\u4e0d\u5efa\u8bae\u5728\u5199\u5927\u6587\u4ef6\uff08>8M\uff09\u573a\u666f\u4f7f\u7528\uff0c\u53ef\u80fd\u4f1a\u9020\u6210\u5185\u5b58\u4e0d\u8db3\u3002 6.7.0+ nextarch","title":"I/O \u76f8\u5173\u914d\u7f6e\u9879"},{"location":"jindosdk/jindosdk_configuration/#_3","text":"\u914d\u7f6e\u9879 \u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e \u7248\u672c fs.oss.memory.buffer.size.max.mb \u6574\u578b 6124 \u5185\u5b58\u6c60\u603b\u5bb9\u91cf\uff08\u5355\u4f4d\uff1aMB\uff09 4.3.0+(deprecated) fs.oss.memory.buffer.size.watermark \u6d6e\u70b9\u578b 0.3 \u5185\u5b58\u6c60\u7528\u4e8e\u9884\u8bfb\u7684\u5bb9\u91cf\u6bd4\u4f8b 4.3.0+(deprecated) fs.jdo.memory.pool.size.max.mb \u6574\u578b 6124 \u5185\u5b58\u6c60\u603b\u5bb9\u91cf\uff08\u5355\u4f4d\uff1aMB\uff09 6.7.0+ nextarch fs.jdo.memory.io.buffer.size.max.ratio \u6d6e\u70b9\u578b 0.8 \u5185\u5b58\u6c60\u6700\u5927\u7528\u4e8eIO buffer\u7684\u5bb9\u91cf\u6bd4\u4f8b 6.7.0+ nextarch","title":"\u5185\u5b58\u76f8\u5173\u914d\u7f6e\u9879"},{"location":"jindosdk/jindosdk_configuration/#metrics","text":"\u914d\u7f6e\u9879 \u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e \u7248\u672c fs.jdo.metrics.level \u6574\u578b 1 mertrics \u6536\u96c6\u7b49\u7ea7\uff0c0\u662f\u4e0d\u6536\u96c6\uff0c1\u662f\u9759\u6001metrics\uff0c2\u662f\u5305\u542b bucket \u4fe1\u606f\u7684\u52a8\u6001 metrics\uff0c3\u5305\u542b\u6240\u6709 metrics 6.6.0+ nextarch fs.jdo.metrics.file.enable \u5e03\u5c14\u503c false \u63a7\u5236\u662f\u5426\u8f93\u51fa metrics \u5230\u6587\u4ef6 6.6.0+ nextarch fs.jdo.metrics.file.dir \u5b57\u7b26\u4e32 /tmp/metrics \u8f93\u51fa metrics \u5230\u6587\u4ef6\u7684\u6240\u5728\u76ee\u5f55 6.6.0+ nextarch fs.jdo.metrics.file.pid.append.enable \u5e03\u5c14\u503c false \u63a7\u5236\u8f93\u51fa metrics \u5230\u6587\u4ef6\u7684\u65b9\u5f0f\uff0ctrue \u4ee3\u8868\u4ee5 pid \u4f5c\u4e3a\u5b50\u76ee\u5f55\uff0c\u76ee\u5f55\u5c42\u7ea7\u4e3a(file_dir)/pid/jindosdk_(timestamp).metrics 6.7.0+ nextarch fs.jdo.metrics.file.number \u6574\u578b 16 \u63a7\u5236\u5355\u4e2a\u76ee\u5f55\u4e0b\u6700\u5927\u6587\u4ef6\u7684\u4e2a\u6570 6.6.0+ nextarch fs.jdo.metrics.interval.sec \u6574\u578b 15 \u8f93\u51fa metrics \u5230\u6587\u4ef6\u7684\u95f4\u9694\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 6.6.0+ nextarch fs.jdo.metrics.file.clean.enable \u5e03\u5c14\u503c false \u63a7\u5236\u662f\u5426\u81ea\u52a8\u6e05\u7406\u8f93\u51fa metrics \u7684\u6587\u4ef6, true \u4ee3\u8868\u6253\u5f00\u6e05\u7406\uff0c\u9ed8\u8ba424h\u6e05\u7406\u4e00\u6b21 6.6.0+ nextarch Credential Provider \u76f8\u5173\u914d\u7f6e\uff0c\u8be6\u89c1 \u300a\u914d\u7f6e OSS/OSS-HDFS Credential Provider\u300b","title":"Metrics \u76f8\u5173\u914d\u7f6e\u9879"},{"location":"jindosdk/jindosdk_credential_provider/","text":"\u914d\u7f6e OSS/OSS-HDFS Credential Provider \u00a7 \u57fa\u672c\u914d\u7f6e\u65b9\u5f0f \u00a7 \u60a8\u53ef\u4ee5\u5c06 OSS/OSS-HDFS \u7684 Access Key ID \u3001 Access Secret \u3001 Endpoint \u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u3002EMR \u96c6\u7fa4\u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \u3002 \u914d\u7f6e\u9879\u5982\u4e0b\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> \u4f7f\u7528 Hadoop Credential Providers \u5b58\u50a8 AccessKey \u4fe1\u606f \u00a7 \u4e0a\u9762\u7684 fs.oss.accessKeyId \u548c fs.oss.accessKeySecret \u5c06\u4f1a\u660e\u6587\u914d\u7f6e\u5728 core-site.xml \u4e2d\uff0c\u53ef\u4ee5\u5c06\u5176\u4ee5\u52a0\u5bc6\u5bf9\u65b9\u5f0f\u5b58\u50a8\u81f3 Hadoop Credential Providers\u6587\u4ef6\u4e2d\u3002 \u4f7f\u7528Hadoop\u63d0\u4f9b\u7684\u547d\u4ee4\uff0c\u5b58\u50a8AccessKey\u548cSecurityToken\u4fe1\u606f\u81f3Credential\u6587\u4ef6\u4e2d\u3002\u547d\u4ee4\u683c\u5f0f\u5982\u4e0b\u3002 hadoop credential <subcommand> [options] \u4f8b\u5982\uff0c\u5b58\u50a8AccessKey\u548cToken\u4fe1\u606f\u81f3JECKS\u6587\u4ef6\u4e2d\uff0c\u9664\u4e86\u4f7f\u7528\u6587\u4ef6\u6743\u9650\u4fdd\u62a4\u8be5\u6587\u4ef6\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u6307\u5b9a\u5bc6\u7801\u52a0\u5bc6\u5b58\u50a8\u4fe1\u606f\uff0c\u5982\u679c\u4e0d\u6307\u5b9a\u5bc6\u7801\u5219\u4f7f\u7528\u9ed8\u8ba4\u5b57\u7b26\u4e32\u52a0\u5bc6\u3002 hadoop credential create fs.oss.accessKeyId -value AAA -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.accessKeySecret -value BBB -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.securityToken -value CCC -provider jceks://file/root/oss.jceks \u751f\u6210Credential\u6587\u4ef6\u540e\uff0c\u60a8\u9700\u8981\u914d\u7f6e\u4e0b\u9762\u7684\u53c2\u6570\u6765\u6307\u5b9aProvider\u7684\u7c7b\u578b\u548c\u4f4d\u7f6e\u3002 <configuration> <property> <name>fs.oss.security.credential.provider.path</name> <value>jceks://file/root/oss.jceks</value> <description>\u914d\u7f6e\u5b58\u50a8AK\u7684Credential\u6587\u4ef6\u3002\u4f8b\u5982\uff0cjceks://file/${user.home}/oss.jceks\u4e3aHOME\u4e0b\u7684oss.jceks\u6587\u4ef6</description> </property> </configuration> \u4f7f\u7528 JindoSDK OSS/OSS-HDFS Credential Provider \u00a7 \u9ed8\u8ba4\u4f1a\u914d\u7f6e SimpleCredentialsProvider\uff0cEnvironmentVariableCredentialsProvider\uff0cCommonCredentialsProvider \u8fd9\u4e09\u4e2aCredential Provider, \u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002 <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.JindoCredentialsProvider\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u60a8\u53ef\u4ee5\u6839\u636e\u60c5\u51b5\uff0c\u9009\u62e9\u4e0d\u540c\u7684Credential Provider\uff0c\u652f\u6301\u5982\u4e0bProvider\uff1a 1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS/OSS-HDFS \u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS/OSS-HDFS\u7684AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS/OSS-HDFS\u7684AccessKey Secret</value> </property> <property> <name>fs.oss.securityToken</name> <value>OSS/OSS-HDFS\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration> 2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS/OSS-HDFS \u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS/OSS-HDFS \u7684AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS/OSS-HDFS \u7684AccessKey Secret</value> </property> </configuration> 3. EnvironmentVariableCredentialsProvider \u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6 AK\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS/OSS-HDFS \u7684 AccessKey Id OSS_ACCESS_KEY_SECRET OSS/OSS-HDFS \u7684 AccessKey Secret OSS_SECURITY_TOKEN OSS/OSS-HDFS \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002 4. CommonCredentialsProvider \u4e3a\u901a\u7528\u914d\u7f6e\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS/OSS-HDFS \u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS/OSS-HDFS \u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS/OSS-HDFS \u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration> 5. CustomCredentialsProvider \u5bf9\u63a5\u5b9a\u5236\u7684\u514d\u5bc6\u670d\u52a1\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CustomCredentialsProvider</value> </property> </configuration> \u914d\u7f6e\u514d\u5bc6\u670d\u52a1\u5730\u5740 <configuration> <property> <name>aliyun.oss.provider.url</name> <value>\u514d\u5bc6\u670d\u52a1\u5730\u5740</value> </property> </configuration> aliyun.oss.provider.url \u652f\u6301 http(s) \u534f\u8bae \u548c secrets \u534f\u8bae\uff1a a. http(s) \u534f\u8bae http(s) \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a http://localhost:1234/sts , http \u514d\u5bc6\u534f\u8bae\u8981\u6c42\u8fd4\u56de\u7ed3\u679c\u4e3a Json \u683c\u5f0f\uff0c\u5982\u679c\u60a8\u9700\u8981\u5bf9\u63a5\u60a8\u7684 http \u514d\u5bc6\u670d\u52a1\uff0c\u514d\u5bc6\u670d\u52a1\u7684\u8bbe\u8ba1\u53c2\u8003 \u300a\u901a\u8fc7API\u4f7f\u7528\u5b9e\u4f8bRAM\u89d2\u8272#\u4e34\u65f6\u6388\u6743Token\u300b { \"AccessKeyId\" : \"XXXXXXXXX\", \"AccessKeySecret\" : \"XXXXXXXXX\", \"Expiration\" : \"2020-11-01T05:20:01Z\", \"SecurityToken\" : \"XXXXXXXXX\", \"LastUpdated\" : \"2020-10-31T23:20:01Z\", \"Code\" : \"Success\" } b. Secrets \u534f\u8bae Secrets \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a secrets:///local_path_prefix \uff0c\u5e38\u89c1\u4f7f\u7528\u4e8e k8s \u573a\u666f \uff0c \u5176\u4e2d local_path_prefix \u4e3a\u8def\u5f84\u524d\u7f00\uff0c\u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOssAccessKeyId /secret/JindoOssAccessKeySecret /secret/JindoOssSecurityToken \u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss/ \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOss/AccessKeyId /secret/JindoOss/AccessKeySecret /secret/JindoOss/SecurityToken JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 OSS/OSS-HDFS bucket \u914d\u7f6e\u4e0d\u540c\u7684 Credential Provider \u00a7 \u8be6\u60c5\u53c2\u8003 \u6309 bucket \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u3002","title":"CredentialProvider \u5feb\u901f\u5165\u95e8"},{"location":"jindosdk/jindosdk_credential_provider/#ossoss-hdfs-credential-provider","text":"","title":"\u914d\u7f6e OSS/OSS-HDFS Credential Provider"},{"location":"jindosdk/jindosdk_credential_provider/#_1","text":"\u60a8\u53ef\u4ee5\u5c06 OSS/OSS-HDFS \u7684 Access Key ID \u3001 Access Secret \u3001 Endpoint \u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u3002EMR \u96c6\u7fa4\u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \u3002 \u914d\u7f6e\u9879\u5982\u4e0b\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration>","title":"\u57fa\u672c\u914d\u7f6e\u65b9\u5f0f"},{"location":"jindosdk/jindosdk_credential_provider/#hadoop-credential-providers-accesskey","text":"\u4e0a\u9762\u7684 fs.oss.accessKeyId \u548c fs.oss.accessKeySecret \u5c06\u4f1a\u660e\u6587\u914d\u7f6e\u5728 core-site.xml \u4e2d\uff0c\u53ef\u4ee5\u5c06\u5176\u4ee5\u52a0\u5bc6\u5bf9\u65b9\u5f0f\u5b58\u50a8\u81f3 Hadoop Credential Providers\u6587\u4ef6\u4e2d\u3002 \u4f7f\u7528Hadoop\u63d0\u4f9b\u7684\u547d\u4ee4\uff0c\u5b58\u50a8AccessKey\u548cSecurityToken\u4fe1\u606f\u81f3Credential\u6587\u4ef6\u4e2d\u3002\u547d\u4ee4\u683c\u5f0f\u5982\u4e0b\u3002 hadoop credential <subcommand> [options] \u4f8b\u5982\uff0c\u5b58\u50a8AccessKey\u548cToken\u4fe1\u606f\u81f3JECKS\u6587\u4ef6\u4e2d\uff0c\u9664\u4e86\u4f7f\u7528\u6587\u4ef6\u6743\u9650\u4fdd\u62a4\u8be5\u6587\u4ef6\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u6307\u5b9a\u5bc6\u7801\u52a0\u5bc6\u5b58\u50a8\u4fe1\u606f\uff0c\u5982\u679c\u4e0d\u6307\u5b9a\u5bc6\u7801\u5219\u4f7f\u7528\u9ed8\u8ba4\u5b57\u7b26\u4e32\u52a0\u5bc6\u3002 hadoop credential create fs.oss.accessKeyId -value AAA -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.accessKeySecret -value BBB -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.securityToken -value CCC -provider jceks://file/root/oss.jceks \u751f\u6210Credential\u6587\u4ef6\u540e\uff0c\u60a8\u9700\u8981\u914d\u7f6e\u4e0b\u9762\u7684\u53c2\u6570\u6765\u6307\u5b9aProvider\u7684\u7c7b\u578b\u548c\u4f4d\u7f6e\u3002 <configuration> <property> <name>fs.oss.security.credential.provider.path</name> <value>jceks://file/root/oss.jceks</value> <description>\u914d\u7f6e\u5b58\u50a8AK\u7684Credential\u6587\u4ef6\u3002\u4f8b\u5982\uff0cjceks://file/${user.home}/oss.jceks\u4e3aHOME\u4e0b\u7684oss.jceks\u6587\u4ef6</description> </property> </configuration>","title":"\u4f7f\u7528 Hadoop Credential Providers \u5b58\u50a8 AccessKey \u4fe1\u606f"},{"location":"jindosdk/jindosdk_credential_provider/#jindosdk-ossoss-hdfs-credential-provider","text":"\u9ed8\u8ba4\u4f1a\u914d\u7f6e SimpleCredentialsProvider\uff0cEnvironmentVariableCredentialsProvider\uff0cCommonCredentialsProvider \u8fd9\u4e09\u4e2aCredential Provider, \u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002 <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.JindoCredentialsProvider\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u60a8\u53ef\u4ee5\u6839\u636e\u60c5\u51b5\uff0c\u9009\u62e9\u4e0d\u540c\u7684Credential Provider\uff0c\u652f\u6301\u5982\u4e0bProvider\uff1a","title":"\u4f7f\u7528 JindoSDK OSS/OSS-HDFS Credential Provider"},{"location":"jindosdk/jindosdk_credential_provider/#1-temporarycredentialsprovider-accesskey-securitytoken-ossoss-hdfs","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS/OSS-HDFS\u7684AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS/OSS-HDFS\u7684AccessKey Secret</value> </property> <property> <name>fs.oss.securityToken</name> <value>OSS/OSS-HDFS\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration>","title":"1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS/OSS-HDFS \u7684\u60c5\u51b5\u3002"},{"location":"jindosdk/jindosdk_credential_provider/#2-simplecredentialsprovider-accesskey-ossoss-hdfs","text":"\u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS/OSS-HDFS \u7684AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS/OSS-HDFS \u7684AccessKey Secret</value> </property> </configuration>","title":"2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS/OSS-HDFS \u7684\u60c5\u51b5\u3002"},{"location":"jindosdk/jindosdk_credential_provider/#3-environmentvariablecredentialsprovider-ak","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS/OSS-HDFS \u7684 AccessKey Id OSS_ACCESS_KEY_SECRET OSS/OSS-HDFS \u7684 AccessKey Secret OSS_SECURITY_TOKEN OSS/OSS-HDFS \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002","title":"3. EnvironmentVariableCredentialsProvider \u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6 AK\u3002"},{"location":"jindosdk/jindosdk_credential_provider/#4-commoncredentialsprovider","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS/OSS-HDFS \u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS/OSS-HDFS \u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS/OSS-HDFS \u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration>","title":"4. CommonCredentialsProvider \u4e3a\u901a\u7528\u914d\u7f6e\u3002"},{"location":"jindosdk/jindosdk_credential_provider/#5-customcredentialsprovider","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CustomCredentialsProvider</value> </property> </configuration> \u914d\u7f6e\u514d\u5bc6\u670d\u52a1\u5730\u5740 <configuration> <property> <name>aliyun.oss.provider.url</name> <value>\u514d\u5bc6\u670d\u52a1\u5730\u5740</value> </property> </configuration> aliyun.oss.provider.url \u652f\u6301 http(s) \u534f\u8bae \u548c secrets \u534f\u8bae\uff1a a. http(s) \u534f\u8bae http(s) \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a http://localhost:1234/sts , http \u514d\u5bc6\u534f\u8bae\u8981\u6c42\u8fd4\u56de\u7ed3\u679c\u4e3a Json \u683c\u5f0f\uff0c\u5982\u679c\u60a8\u9700\u8981\u5bf9\u63a5\u60a8\u7684 http \u514d\u5bc6\u670d\u52a1\uff0c\u514d\u5bc6\u670d\u52a1\u7684\u8bbe\u8ba1\u53c2\u8003 \u300a\u901a\u8fc7API\u4f7f\u7528\u5b9e\u4f8bRAM\u89d2\u8272#\u4e34\u65f6\u6388\u6743Token\u300b { \"AccessKeyId\" : \"XXXXXXXXX\", \"AccessKeySecret\" : \"XXXXXXXXX\", \"Expiration\" : \"2020-11-01T05:20:01Z\", \"SecurityToken\" : \"XXXXXXXXX\", \"LastUpdated\" : \"2020-10-31T23:20:01Z\", \"Code\" : \"Success\" } b. Secrets \u534f\u8bae Secrets \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a secrets:///local_path_prefix \uff0c\u5e38\u89c1\u4f7f\u7528\u4e8e k8s \u573a\u666f \uff0c \u5176\u4e2d local_path_prefix \u4e3a\u8def\u5f84\u524d\u7f00\uff0c\u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOssAccessKeyId /secret/JindoOssAccessKeySecret /secret/JindoOssSecurityToken \u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss/ \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOss/AccessKeyId /secret/JindoOss/AccessKeySecret /secret/JindoOss/SecurityToken","title":"5. CustomCredentialsProvider \u5bf9\u63a5\u5b9a\u5236\u7684\u514d\u5bc6\u670d\u52a1\u3002"},{"location":"jindosdk/jindosdk_credential_provider/#jindosdk-ossoss-hdfs-bucket-credential-provider","text":"\u8be6\u60c5\u53c2\u8003 \u6309 bucket \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u3002","title":"JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 OSS/OSS-HDFS bucket \u914d\u7f6e\u4e0d\u540c\u7684 Credential Provider"},{"location":"jindosdk/jindosdk_credential_provider_bucket/","text":"\u6309 bucket \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u00a7 \u6309\u7167 bucket \u914d\u7f6e Credential Provider \u00a7 <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleAliyunCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.AliyunCredentialsProvider\u7684\u5b9e\u73b0\u7c7b\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u8bf4\u660e XXX \u4e3a OSS/OSS-HDFS bucket \u540d\u79f0\u3002 Credential Provider \u7c7b\u578b \u00a7 1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS/OSS-HDFS \u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS/OSS-HDFS bucket\u7684AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS/OSS-HDFS bucket\u7684AccessKey Secret</value> </property> <property> <name>fs.oss.bucket.XXX.securityToken</name> <value>OSS/OSS-HDFS bucket\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration> 2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS/OSS-HDFS \u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS/OSS-HDFS bucket\u7684AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS/OSS-HDFS bucket\u7684AccessKey Secret</value> </property> </configuration> 3. EnvironmentVariableCredentialsProvider \u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6 AccessKey\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS/OSS-HDFS bucket\u7684AccessKey Id OSS_ACCESS_KEY_SECRET OSS/OSS-HDFS bucket\u7684AccessKey Secret OSS_SECURITY_TOKEN OSS/OSS-HDFS bucket\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548Token\u65f6\u9700\u8981\u3002 4. CommonCredentialsProvider\u4e3a\u901a\u7528\u914d\u7f6e\u3002 \u00a7 \u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS/OSS-HDFS bucket \u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS/OSS-HDFS bucket \u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS/OSS-HDFS bucket \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration>","title":"Bucket \u7ea7\u522b\u914d\u7f6e CredentialProvider"},{"location":"jindosdk/jindosdk_credential_provider_bucket/#bucket-ossoss-hdfs-credential-provider","text":"","title":"\u6309 bucket \u914d\u7f6e OSS/OSS-HDFS Credential Provider"},{"location":"jindosdk/jindosdk_credential_provider_bucket/#bucket-credential-provider","text":"<configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleAliyunCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.AliyunCredentialsProvider\u7684\u5b9e\u73b0\u7c7b\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u8bf4\u660e XXX \u4e3a OSS/OSS-HDFS bucket \u540d\u79f0\u3002","title":"\u6309\u7167 bucket \u914d\u7f6e Credential Provider"},{"location":"jindosdk/jindosdk_credential_provider_bucket/#credential-provider","text":"","title":"Credential Provider \u7c7b\u578b"},{"location":"jindosdk/jindosdk_credential_provider_bucket/#1-temporarycredentialsprovider-accesskey-securitytoken-ossoss-hdfs","text":"\u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS/OSS-HDFS bucket\u7684AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS/OSS-HDFS bucket\u7684AccessKey Secret</value> </property> <property> <name>fs.oss.bucket.XXX.securityToken</name> <value>OSS/OSS-HDFS bucket\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration>","title":"1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS/OSS-HDFS \u7684\u60c5\u51b5\u3002"},{"location":"jindosdk/jindosdk_credential_provider_bucket/#2-simplecredentialsprovider-accesskey-ossoss-hdfs","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS/OSS-HDFS bucket\u7684AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS/OSS-HDFS bucket\u7684AccessKey Secret</value> </property> </configuration>","title":"2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS/OSS-HDFS \u7684\u60c5\u51b5\u3002"},{"location":"jindosdk/jindosdk_credential_provider_bucket/#3-environmentvariablecredentialsprovider-accesskey","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS/OSS-HDFS bucket\u7684AccessKey Id OSS_ACCESS_KEY_SECRET OSS/OSS-HDFS bucket\u7684AccessKey Secret OSS_SECURITY_TOKEN OSS/OSS-HDFS bucket\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548Token\u65f6\u9700\u8981\u3002","title":"3. EnvironmentVariableCredentialsProvider \u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6 AccessKey\u3002"},{"location":"jindosdk/jindosdk_credential_provider_bucket/#4-commoncredentialsprovider","text":"\u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS/OSS-HDFS bucket \u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS/OSS-HDFS bucket \u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS/OSS-HDFS bucket \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration>","title":"4. CommonCredentialsProvider\u4e3a\u901a\u7528\u914d\u7f6e\u3002"},{"location":"jindosdk/jindosdk_credential_provider_faq/","text":"\u8bbf\u95ee OSS/OSS-HDFS \u65f6 AK \u76f8\u5173\u5e38\u89c1\u95ee\u9898 \u00a7 1. EMR \u96c6\u7fa4\u5185\u5982\u4f55\u514d\u5bc6\u8bbf\u95ee OSS/OSS-HDFS \u00a7 JindoSDK \u4f1a\u4f7f\u7528\u521b\u5efa\u96c6\u7fa4\u65f6\u7ed1\u5b9a\u7684 ECS \u5e94\u7528\u89d2\u8272\uff08\u9ed8\u8ba4\u4e3a AliyunECSInstanceForEMRRole\uff09\u83b7\u53d6 Security Token \u8bbf\u95ee OSS/OSS-HDFS\u3002 \u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \u4e2d\u66f4\u65b0\u4e0b\u5217\u53c2\u6570\uff1a \u53c2\u6570 \u503c fs.oss.credentials.provider com.aliyun.jindodata.oss.auth.EcsStsCredentialsProvider 2. \u5982\u4f55\u68c0\u67e5 EMR \u96c6\u7fa4\u5185\u514d\u5bc6\u670d\u52a1\u662f\u5426\u53ef\u7528\uff1f \u00a7 \u65b9\u6cd5\u4e00\uff1a \u00a7 \u6267\u884c\u4e0b\u9762\u547d\u4ee4\uff1a curl http://100.100.100.200/latest/meta-data/Ram/Security-credentials/AliyunECSInstanceForEMRRole \u5982\u679c\u8fd4\u56de\u4e0b\u9762\u683c\u5f0f\u7684\u5185\u5bb9\uff0c\u5219\u6b63\u5e38\u3002 \"AccessKeyId\" : \"STS.NUreXXXXXX\", \"AccessKeySecret\" : \"BsmbnDoXXXXXXXX\", \"Expiration\" : \"2022-11-22T11:27:39Z\", \"SecurityToken\" : \"CAISlwJ1q6FXXXXXXX\", \"LastUpdated\" : \"2022-11-22T05:27:39Z\", \"Code\" : \"Success\" \u65b9\u6cd5\u4e8c\uff1a \u00a7 \u914d\u7f6e\u4e3a ECS \u514d\u5bc6\uff1a \u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \u4e2d\u66f4\u65b0\u4e0b\u5217\u53c2\u6570\uff1a \u53c2\u6570 \u503c fs.oss.credentials.provider com.aliyun.jindodata.oss.auth.EcsStsCredentialsProvider \u4f7f\u7528 HDFS shell \u8bbf\u95ee OSS/OSS-HDFS bucket, \u82e5\u80fd\u6b63\u5e38\u8bbf\u95ee\uff0c\u5219\u786e\u8ba4\u514d\u5bc6\u6b63\u5e38\u3002 3. \u5e38\u9a7b\u670d\u52a1\u6bcf 5-6 \u4e2a\u5c0f\u65f6\u8bbf\u95ee OSS/OSS-HDFS \u5931\u8d25 \u00a7 \u67d0\u4e9b\u7248\u672c\u6709\u5df2\u77e5\u514d\u5bc6\u95ee\u9898\uff0c\u8bf7\u67e5\u770b \u5df2\u77e5\u95ee\u9898 \u65b9\u6cd5\u4e00\uff1a\u4f7f\u7528\u56fa\u5b9a AK \u00a7 \u914d\u7f6e\u4f7f\u7528 SimpleCredentialsProvider , \u8bf7\u53c2\u8003 \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u65b9\u6cd5\u4e8c\uff1a\u5347\u7ea7 JindoSDK \u4e3a\u6700\u65b0\u7248\u672c \u00a7 \u65e7\u7248\u63a7\u5236\u53f0\u5347\u7ea7\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b \u65b0\u7248\u63a7\u5236\u53f0\u5347\u7ea7\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b E-MapReduce EMR-5.5.0/EMR-3.39.0 \u4ee5\u524d\u7248\u672c\u7684\u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b 4. \u8bbf\u95ee OSS/OSS-HDFS \u8def\u5f84\u4e2d\u5305\u542b AK \u4fe1\u606f\u62a5\u9519 \u00a7 \u62a5\u9519\u4fe1\u606f\u5982\u4e0b\uff1a The Filesystem URI contains login details. This authentication mechanism is no longer supported. \u4e3a\u4e86\u9632\u6b62 AK \u6cc4\u9732\u7b49\u5b89\u5168\u95ee\u9898\uff0cJindoSDK 4.0.0 \u5f00\u59cb\u4e0d\u652f\u6301\u8def\u5f84\u4e2d\u643a\u5e26 AK(AccessKeyId,AccessKeySecret\u7b49\uff09\u4fe1\u606f\u3002 \u65b9\u6cd5\u4e00\uff1a \u00a7 \u8bbf\u95ee\u8def\u5f84\u4e2d\u53bb\u9664 AK \u4fe1\u606f\uff1b \u65b9\u6cd5\u4e8c\uff1a \u00a7 \u5982\u679c\u4e00\u5b9a\u8981\u8fd9\u6837\u4f7f\u7528\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u53c2\u6570\u5f00\u542f\u3002\u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \u4e2d\u65b0\u589e\u4e0b\u5217\u53c2\u6570\uff1a \u53c2\u6570 \u503c fs.oss.uri-with-secrets.enable true","title":"\u8bbf\u95ee OSS/OSS-HDFS \u65f6 AK \u76f8\u5173\u5e38\u89c1\u95ee\u9898"},{"location":"jindosdk/jindosdk_credential_provider_faq/#ossoss-hdfs-ak","text":"","title":"\u8bbf\u95ee OSS/OSS-HDFS \u65f6 AK \u76f8\u5173\u5e38\u89c1\u95ee\u9898"},{"location":"jindosdk/jindosdk_credential_provider_faq/#1-emr-ossoss-hdfs","text":"JindoSDK \u4f1a\u4f7f\u7528\u521b\u5efa\u96c6\u7fa4\u65f6\u7ed1\u5b9a\u7684 ECS \u5e94\u7528\u89d2\u8272\uff08\u9ed8\u8ba4\u4e3a AliyunECSInstanceForEMRRole\uff09\u83b7\u53d6 Security Token \u8bbf\u95ee OSS/OSS-HDFS\u3002 \u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \u4e2d\u66f4\u65b0\u4e0b\u5217\u53c2\u6570\uff1a \u53c2\u6570 \u503c fs.oss.credentials.provider com.aliyun.jindodata.oss.auth.EcsStsCredentialsProvider","title":"1. EMR \u96c6\u7fa4\u5185\u5982\u4f55\u514d\u5bc6\u8bbf\u95ee OSS/OSS-HDFS"},{"location":"jindosdk/jindosdk_credential_provider_faq/#2-emr","text":"","title":"2. \u5982\u4f55\u68c0\u67e5 EMR \u96c6\u7fa4\u5185\u514d\u5bc6\u670d\u52a1\u662f\u5426\u53ef\u7528\uff1f"},{"location":"jindosdk/jindosdk_credential_provider_faq/#_1","text":"\u6267\u884c\u4e0b\u9762\u547d\u4ee4\uff1a curl http://100.100.100.200/latest/meta-data/Ram/Security-credentials/AliyunECSInstanceForEMRRole \u5982\u679c\u8fd4\u56de\u4e0b\u9762\u683c\u5f0f\u7684\u5185\u5bb9\uff0c\u5219\u6b63\u5e38\u3002 \"AccessKeyId\" : \"STS.NUreXXXXXX\", \"AccessKeySecret\" : \"BsmbnDoXXXXXXXX\", \"Expiration\" : \"2022-11-22T11:27:39Z\", \"SecurityToken\" : \"CAISlwJ1q6FXXXXXXX\", \"LastUpdated\" : \"2022-11-22T05:27:39Z\", \"Code\" : \"Success\"","title":"\u65b9\u6cd5\u4e00\uff1a"},{"location":"jindosdk/jindosdk_credential_provider_faq/#_2","text":"\u914d\u7f6e\u4e3a ECS \u514d\u5bc6\uff1a \u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \u4e2d\u66f4\u65b0\u4e0b\u5217\u53c2\u6570\uff1a \u53c2\u6570 \u503c fs.oss.credentials.provider com.aliyun.jindodata.oss.auth.EcsStsCredentialsProvider \u4f7f\u7528 HDFS shell \u8bbf\u95ee OSS/OSS-HDFS bucket, \u82e5\u80fd\u6b63\u5e38\u8bbf\u95ee\uff0c\u5219\u786e\u8ba4\u514d\u5bc6\u6b63\u5e38\u3002","title":"\u65b9\u6cd5\u4e8c\uff1a"},{"location":"jindosdk/jindosdk_credential_provider_faq/#3-5-6-ossoss-hdfs","text":"\u67d0\u4e9b\u7248\u672c\u6709\u5df2\u77e5\u514d\u5bc6\u95ee\u9898\uff0c\u8bf7\u67e5\u770b \u5df2\u77e5\u95ee\u9898","title":"3. \u5e38\u9a7b\u670d\u52a1\u6bcf 5-6 \u4e2a\u5c0f\u65f6\u8bbf\u95ee OSS/OSS-HDFS \u5931\u8d25"},{"location":"jindosdk/jindosdk_credential_provider_faq/#ak","text":"\u914d\u7f6e\u4f7f\u7528 SimpleCredentialsProvider , \u8bf7\u53c2\u8003 \u914d\u7f6e OSS/OSS-HDFS Credential Provider","title":"\u65b9\u6cd5\u4e00\uff1a\u4f7f\u7528\u56fa\u5b9a AK"},{"location":"jindosdk/jindosdk_credential_provider_faq/#jindosdk","text":"\u65e7\u7248\u63a7\u5236\u53f0\u5347\u7ea7\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b \u65b0\u7248\u63a7\u5236\u53f0\u5347\u7ea7\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b E-MapReduce EMR-5.5.0/EMR-3.39.0 \u4ee5\u524d\u7248\u672c\u7684\u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b","title":"\u65b9\u6cd5\u4e8c\uff1a\u5347\u7ea7 JindoSDK \u4e3a\u6700\u65b0\u7248\u672c"},{"location":"jindosdk/jindosdk_credential_provider_faq/#4-ossoss-hdfs-ak","text":"\u62a5\u9519\u4fe1\u606f\u5982\u4e0b\uff1a The Filesystem URI contains login details. This authentication mechanism is no longer supported. \u4e3a\u4e86\u9632\u6b62 AK \u6cc4\u9732\u7b49\u5b89\u5168\u95ee\u9898\uff0cJindoSDK 4.0.0 \u5f00\u59cb\u4e0d\u652f\u6301\u8def\u5f84\u4e2d\u643a\u5e26 AK(AccessKeyId,AccessKeySecret\u7b49\uff09\u4fe1\u606f\u3002","title":"4. \u8bbf\u95ee OSS/OSS-HDFS \u8def\u5f84\u4e2d\u5305\u542b AK \u4fe1\u606f\u62a5\u9519"},{"location":"jindosdk/jindosdk_credential_provider_faq/#_3","text":"\u8bbf\u95ee\u8def\u5f84\u4e2d\u53bb\u9664 AK \u4fe1\u606f\uff1b","title":"\u65b9\u6cd5\u4e00\uff1a"},{"location":"jindosdk/jindosdk_credential_provider_faq/#_4","text":"\u5982\u679c\u4e00\u5b9a\u8981\u8fd9\u6837\u4f7f\u7528\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u53c2\u6570\u5f00\u542f\u3002\u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \u4e2d\u65b0\u589e\u4e0b\u5217\u53c2\u6570\uff1a \u53c2\u6570 \u503c fs.oss.uri-with-secrets.enable true","title":"\u65b9\u6cd5\u4e8c\uff1a"},{"location":"jindosdk/jindosdk_deployment_ai/","text":"AI \u573a\u666f\u90e8\u7f72 JindoSDK \u00a7 \u5728\u4f7f\u7528 JindoFuse\uff0cJindo CLI\uff0cPyJindo \u7b49\u975e Hadoop \u751f\u6001\u7ec4\u4ef6\u65f6\uff0c\u4f1a\u8bbf\u95ee\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6240\u5728\u7684\u76ee\u5f55\u8bfb\u53d6\u914d\u7f6e\u6587\u4ef6\u3002 \u90e8\u7f72 JindoSDK \u00a7 \u4e0b\u8f7d\u5e76\u89e3\u538b JindoSDK TAR \u5305\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u4e0b\u8f7d6.7.3\u7248\u672cJindoSDK TAR\u5305\u3002\u4ee5\u5927\u591a\u6570linux x86\u73af\u5883\u4e3a\u4f8b\u3002\u5176\u4ed6\u5e73\u53f0\u90e8\u7f72\u53c2\u89c1 \u300a\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK\u300b wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u89e3\u538b JindoSDK TAR \u5305\u3002 tar zxvf jindosdk-6.7.3-linux.tar.gz \u914d\u7f6e\u73af\u5883\u53d8\u91cf\u3002 \u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u89e3\u538b\u5728 /usr/lib/jindosdk-6.7.3-linux \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_HOME=/usr/lib/jindosdk-6.7.3-linux export JINDOSDK_CONF_DIR=${JINDOSDK_HOME}/conf export PATH=${PATH}:${JINDOSDK_HOME}/bin export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${JINDOSDK_HOME}/lib/native \u91cd\u8981 \u8bf7\u5c06\u5b89\u88c5\u76ee\u5f55\u548c\u73af\u5883\u53d8\u91cf\u90e8\u7f72\u5230\u6240\u6709\u6240\u9700\u8282\u70b9\u4e0a\u3002 \u4fee\u6539\u914d\u7f6e \u00a7 \u914d\u7f6e\u6587\u4ef6\u4f4d\u4e8e JINDOSDK_CONF_DIR \u6240\u5728\u76ee\u5f55\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\u3002 cfg \u914d\u7f6e\u6587\u4ef6 \u00a7 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/jindosdk-log [jindosdk] # \u5df2\u521b\u5efa OSS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3a oss-cn-hangzhou.aliyuncs.com\u3002 # \u5df2\u521b\u5efa OSS-HDFS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3a cn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95eeOSS\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> \u514d\u5bc6\u8bbf\u95ee \u00a7 \u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7 RAM\u89d2\u8272\u6388\u6743 \u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/jindosdk-log [jindosdk] # \u5df2\u521b\u5efa OSS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3a oss-cn-hangzhou.aliyuncs.com\u3002 # \u5df2\u521b\u5efa OSS-HDFS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3a cn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE fs.oss.provider.format = JSON","title":"AI \u573a\u666f\u90e8\u7f72"},{"location":"jindosdk/jindosdk_deployment_ai/#ai-jindosdk","text":"\u5728\u4f7f\u7528 JindoFuse\uff0cJindo CLI\uff0cPyJindo \u7b49\u975e Hadoop \u751f\u6001\u7ec4\u4ef6\u65f6\uff0c\u4f1a\u8bbf\u95ee\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6240\u5728\u7684\u76ee\u5f55\u8bfb\u53d6\u914d\u7f6e\u6587\u4ef6\u3002","title":"AI \u573a\u666f\u90e8\u7f72 JindoSDK"},{"location":"jindosdk/jindosdk_deployment_ai/#jindosdk","text":"\u4e0b\u8f7d\u5e76\u89e3\u538b JindoSDK TAR \u5305\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u4e0b\u8f7d6.7.3\u7248\u672cJindoSDK TAR\u5305\u3002\u4ee5\u5927\u591a\u6570linux x86\u73af\u5883\u4e3a\u4f8b\u3002\u5176\u4ed6\u5e73\u53f0\u90e8\u7f72\u53c2\u89c1 \u300a\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK\u300b wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u89e3\u538b JindoSDK TAR \u5305\u3002 tar zxvf jindosdk-6.7.3-linux.tar.gz \u914d\u7f6e\u73af\u5883\u53d8\u91cf\u3002 \u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u89e3\u538b\u5728 /usr/lib/jindosdk-6.7.3-linux \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_HOME=/usr/lib/jindosdk-6.7.3-linux export JINDOSDK_CONF_DIR=${JINDOSDK_HOME}/conf export PATH=${PATH}:${JINDOSDK_HOME}/bin export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${JINDOSDK_HOME}/lib/native \u91cd\u8981 \u8bf7\u5c06\u5b89\u88c5\u76ee\u5f55\u548c\u73af\u5883\u53d8\u91cf\u90e8\u7f72\u5230\u6240\u6709\u6240\u9700\u8282\u70b9\u4e0a\u3002","title":"\u90e8\u7f72 JindoSDK"},{"location":"jindosdk/jindosdk_deployment_ai/#_1","text":"\u914d\u7f6e\u6587\u4ef6\u4f4d\u4e8e JINDOSDK_CONF_DIR \u6240\u5728\u76ee\u5f55\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\u3002","title":"\u4fee\u6539\u914d\u7f6e"},{"location":"jindosdk/jindosdk_deployment_ai/#cfg","text":"\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/jindosdk-log [jindosdk] # \u5df2\u521b\u5efa OSS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3a oss-cn-hangzhou.aliyuncs.com\u3002 # \u5df2\u521b\u5efa OSS-HDFS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3a cn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95eeOSS\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret>","title":"cfg \u914d\u7f6e\u6587\u4ef6"},{"location":"jindosdk/jindosdk_deployment_ai/#_2","text":"\u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7 RAM\u89d2\u8272\u6388\u6743 \u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/jindosdk-log [jindosdk] # \u5df2\u521b\u5efa OSS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3a oss-cn-hangzhou.aliyuncs.com\u3002 # \u5df2\u521b\u5efa OSS-HDFS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3a cn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE fs.oss.provider.format = JSON","title":"\u514d\u5bc6\u8bbf\u95ee"},{"location":"jindosdk/jindosdk_deployment_hadoop/","text":"\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK \u00a7 \u90e8\u7f72 JindoSDK \u00a7 \u4e0b\u8f7d\u5e76\u89e3\u538b JindoSDK TAR \u5305\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u4e0b\u8f7d6.7.3\u7248\u672cJindoSDK TAR\u5305\u3002\u4ee5\u5927\u591a\u6570linux x86\u73af\u5883\u4e3a\u4f8b\u3002\u5176\u4ed6\u5e73\u53f0\u90e8\u7f72\u53c2\u89c1 \u300a\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK\u300b wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u89e3\u538b JindoSDK TAR \u5305\u3002 tar zxvf jindosdk-6.7.3-linux.tar.gz \u914d\u7f6e\u73af\u5883\u53d8\u91cf\u3002 \u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u89e3\u538b\u5728 /usr/lib/jindosdk-6.7.3-linux \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_HOME=/usr/lib/jindosdk-6.7.3-linux export JINDOSDK_CONF_DIR=${JINDOSDK_HOME}/conf export PATH=${PATH}:${JINDOSDK_HOME}/bin export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${JINDOSDK_HOME}/lib/native export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:${JINDOSDK_HOME}/lib/* \u91cd\u8981 \u8bf7\u5c06\u5b89\u88c5\u76ee\u5f55\u548c\u73af\u5883\u53d8\u91cf\u90e8\u7f72\u5230\u6240\u6709\u6240\u9700\u8282\u70b9\u4e0a\u3002 Hadoop \u914d\u7f6e\u6587\u4ef6 \u00a7 \u914d\u7f6e OSS/OSS-HDFS \u5b9e\u73b0\u7c7b\u53caAccessKey\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u8fdb\u5165Hadoop\u7684core-site.xml\u914d\u7f6e\u6587\u4ef6\u3002 vim <HADOOP_HOME>/etc/hadoop/core-site.xml \u5c06 OSS/OSS-HDFS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230Hadoop\u7684core-site.xml\u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06 OSS/OSS-HDFS \u7684Bucket\u5bf9\u5e94\u7684accessKeyId\u3001accessKeySecret\u9884\u5148\u914d\u7f6e\u5728Hadoop\u7684core-site.xml\u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS \u670d\u52a1 Endpoint\u3002 \u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u66f4\u591a\u914d\u7f6e\u65b9\u5f0f\u3002 \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u6309 bucket \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u8bbf\u95ee OSS/OSS-HDFS \u65f6 AK \u76f8\u5173\u5e38\u89c1\u95ee\u9898","title":"Hadoop \u73af\u5883\u5b8c\u6574\u90e8\u7f72"},{"location":"jindosdk/jindosdk_deployment_hadoop/#hadoop-jindosdk","text":"","title":"\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK"},{"location":"jindosdk/jindosdk_deployment_hadoop/#jindosdk","text":"\u4e0b\u8f7d\u5e76\u89e3\u538b JindoSDK TAR \u5305\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u4e0b\u8f7d6.7.3\u7248\u672cJindoSDK TAR\u5305\u3002\u4ee5\u5927\u591a\u6570linux x86\u73af\u5883\u4e3a\u4f8b\u3002\u5176\u4ed6\u5e73\u53f0\u90e8\u7f72\u53c2\u89c1 \u300a\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK\u300b wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u89e3\u538b JindoSDK TAR \u5305\u3002 tar zxvf jindosdk-6.7.3-linux.tar.gz \u914d\u7f6e\u73af\u5883\u53d8\u91cf\u3002 \u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u89e3\u538b\u5728 /usr/lib/jindosdk-6.7.3-linux \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_HOME=/usr/lib/jindosdk-6.7.3-linux export JINDOSDK_CONF_DIR=${JINDOSDK_HOME}/conf export PATH=${PATH}:${JINDOSDK_HOME}/bin export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${JINDOSDK_HOME}/lib/native export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:${JINDOSDK_HOME}/lib/* \u91cd\u8981 \u8bf7\u5c06\u5b89\u88c5\u76ee\u5f55\u548c\u73af\u5883\u53d8\u91cf\u90e8\u7f72\u5230\u6240\u6709\u6240\u9700\u8282\u70b9\u4e0a\u3002","title":"\u90e8\u7f72 JindoSDK"},{"location":"jindosdk/jindosdk_deployment_hadoop/#hadoop","text":"\u914d\u7f6e OSS/OSS-HDFS \u5b9e\u73b0\u7c7b\u53caAccessKey\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u8fdb\u5165Hadoop\u7684core-site.xml\u914d\u7f6e\u6587\u4ef6\u3002 vim <HADOOP_HOME>/etc/hadoop/core-site.xml \u5c06 OSS/OSS-HDFS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230Hadoop\u7684core-site.xml\u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06 OSS/OSS-HDFS \u7684Bucket\u5bf9\u5e94\u7684accessKeyId\u3001accessKeySecret\u9884\u5148\u914d\u7f6e\u5728Hadoop\u7684core-site.xml\u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS \u670d\u52a1 Endpoint\u3002 \u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u66f4\u591a\u914d\u7f6e\u65b9\u5f0f\u3002 \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u6309 bucket \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u8bbf\u95ee OSS/OSS-HDFS \u65f6 AK \u76f8\u5173\u5e38\u89c1\u95ee\u9898","title":"Hadoop \u914d\u7f6e\u6587\u4ef6"},{"location":"jindosdk/jindosdk_deployment_lite_hadoop/","text":"\u5728 Hadoop \u73af\u5883\u6700\u7b80\u5b89\u88c5\u90e8\u7f72 JindoSDK \u00a7 \u90e8\u7f72 JindoSDK \u00a7 \u4e0b\u8f7d JindoSDK JAR \u5305\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u4e0b\u8f7d6.7.3\u7248\u672cJindoSDK JAR\u5305\u3002\u4ee5\u5927\u591a\u6570linux x86\u73af\u5883\u4e3a\u4f8b\u3002\u5176\u4ed6\u5e73\u53f0\u90e8\u7f72\u53c2\u89c1 \u300a\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK\u300b wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-sdk/6.7.3/jindo-sdk-6.7.3.jar wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-core/6.7.3/jindo-core-6.7.3.jar \u5b89\u88c5 JindoSDK JAR \u5305 \u5c06\u4e0b\u8f7d\u7684 jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a jindo-core-x.x.x.jar jindo-sdk-x.x.x.jar \u4ee5 jindosdk-6.7.3 \u4e3a\u4f8b: cp jindo-core-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ \u91cd\u8981 \u8bf7\u5c06\u5b89\u88c5\u76ee\u5f55\u548c\u73af\u5883\u53d8\u91cf\u90e8\u7f72\u5230\u6240\u6709\u6240\u9700\u8282\u70b9\u4e0a\u3002 Hadoop \u914d\u7f6e\u6587\u4ef6 \u00a7 \u914d\u7f6e OSS/OSS-HDFS \u5b9e\u73b0\u7c7b\u53caAccessKey\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u8fdb\u5165Hadoop\u7684core-site.xml\u914d\u7f6e\u6587\u4ef6\u3002 vim <HADOOP_HOME>/etc/hadoop/core-site.xml \u5c06 OSS/OSS-HDFS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230Hadoop\u7684core-site.xml\u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06 OSS/OSS-HDFS \u7684Bucket\u5bf9\u5e94\u7684accessKeyId\u3001accessKeySecret\u9884\u5148\u914d\u7f6e\u5728Hadoop\u7684core-site.xml\u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS \u670d\u52a1 Endpoint\u3002 \u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u66f4\u591a\u914d\u7f6e\u65b9\u5f0f\u3002 \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u6309 bucket \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u8bbf\u95ee OSS/OSS-HDFS \u65f6 AK \u76f8\u5173\u5e38\u89c1\u95ee\u9898","title":"Hadoop \u73af\u5883\u4ec5\u90e8\u7f72 Jar \u5305"},{"location":"jindosdk/jindosdk_deployment_lite_hadoop/#hadoop-jindosdk","text":"","title":"\u5728 Hadoop \u73af\u5883\u6700\u7b80\u5b89\u88c5\u90e8\u7f72 JindoSDK"},{"location":"jindosdk/jindosdk_deployment_lite_hadoop/#jindosdk","text":"\u4e0b\u8f7d JindoSDK JAR \u5305\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u4e0b\u8f7d6.7.3\u7248\u672cJindoSDK JAR\u5305\u3002\u4ee5\u5927\u591a\u6570linux x86\u73af\u5883\u4e3a\u4f8b\u3002\u5176\u4ed6\u5e73\u53f0\u90e8\u7f72\u53c2\u89c1 \u300a\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK\u300b wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-sdk/6.7.3/jindo-sdk-6.7.3.jar wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-core/6.7.3/jindo-core-6.7.3.jar \u5b89\u88c5 JindoSDK JAR \u5305 \u5c06\u4e0b\u8f7d\u7684 jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a jindo-core-x.x.x.jar jindo-sdk-x.x.x.jar \u4ee5 jindosdk-6.7.3 \u4e3a\u4f8b: cp jindo-core-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ \u91cd\u8981 \u8bf7\u5c06\u5b89\u88c5\u76ee\u5f55\u548c\u73af\u5883\u53d8\u91cf\u90e8\u7f72\u5230\u6240\u6709\u6240\u9700\u8282\u70b9\u4e0a\u3002","title":"\u90e8\u7f72 JindoSDK"},{"location":"jindosdk/jindosdk_deployment_lite_hadoop/#hadoop","text":"\u914d\u7f6e OSS/OSS-HDFS \u5b9e\u73b0\u7c7b\u53caAccessKey\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u8fdb\u5165Hadoop\u7684core-site.xml\u914d\u7f6e\u6587\u4ef6\u3002 vim <HADOOP_HOME>/etc/hadoop/core-site.xml \u5c06 OSS/OSS-HDFS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230Hadoop\u7684core-site.xml\u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06 OSS/OSS-HDFS \u7684Bucket\u5bf9\u5e94\u7684accessKeyId\u3001accessKeySecret\u9884\u5148\u914d\u7f6e\u5728Hadoop\u7684core-site.xml\u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS \u670d\u52a1 Endpoint\u3002 \u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u66f4\u591a\u914d\u7f6e\u65b9\u5f0f\u3002 \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u6309 bucket \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u8bbf\u95ee OSS/OSS-HDFS \u65f6 AK \u76f8\u5173\u5e38\u89c1\u95ee\u9898","title":"Hadoop \u914d\u7f6e\u6587\u4ef6"},{"location":"jindosdk/jindosdk_deployment_multi_platform/","text":"\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK \u00a7 JindoSDK \u591a\u5e73\u53f0\u652f\u6301\u73b0\u72b6 \u00a7 JindoSDK \u76ee\u524d\u652f\u6301\u5927\u90e8\u5206\u5e38\u89c1\u7684 Linux \u53d1\u884c\u7248\uff0c\u5e76\u652f\u6301 x86/aarch64 \u5e73\u53f0\u3002 \u652f\u6301 MacOS \u64cd\u4f5c\u7cfb\u7edf\uff0c\u6240\u6709\u7248\u672c\u5747\u652f\u6301 x86 \u5e73\u53f0\uff0c\u90e8\u5206\u7248\u672c\u652f\u6301 M1/M2 \u82af\u7247\u7684 arm \u5e73\u53f0\u3002 \u6682\u4e0d\u652f\u6301 Windows \u64cd\u4f5c\u7cfb\u7edf\u3002 \u6682\u4e0d\u652f\u6301\u672a\u517c\u5bb9 glibc \u7684 linux \u53d1\u884c\u7248\uff0c\u5982\u9ed8\u8ba4\u4f7f\u7528 musl \u7684\u53d1\u884c\u7248\u3002 \u4ea7\u51fa\u7269 \u00a7 \u4ea7\u51fa\u7269\u5305\u542b\u4e24\u7c7b\uff1a\u5b8c\u6574\u4ea7\u51fa\u7269\u3001lite\u4ea7\u51fa\u7269\u3002\u4ee5\u4e0b x.y.z \u8868\u793a\u7248\u672c\u53f7\u3002 \u5b8c\u6574\u4ea7\u51fa\u7269\u8bf4\u660e \u00a7 \u5728 Aliyun EMR \u4e2d\u9ed8\u8ba4\u5b89\u88c5\u4e86\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u5b89\u88c5\u8def\u5f84\u4e3a/opt/apps/JINDOSDK/jindosdk-current/\u3002 \u5b8c\u6574\u4ea7\u51fa\u7269\u662f\u4e00\u4e2atar\u5305\uff0c\u540d\u79f0\u4e3a jindosdk-x.y.z-<\u5e73\u53f0\u540d\u79f0>.tar.gz \u3002 \u5b8c\u6574\u4ea7\u51fa\u7269\u5305\u542b hadoop sdk\uff0cjindo-fuse, jindo-distcp, jindotable\uff0cjindo flink connector, jindocli \u7b49\u5de5\u5177\uff0c\u53ef\u4ee5\u90e8\u7f72\u4f7f\u7528\u3002\u4e5f\u5305\u542b\u4e8c\u6b21\u5f00\u53d1\u6240\u9700\u7684so\u7b49\u4f9d\u8d56\u3002 \u5982\u679c\u5e0c\u671b\u5b89\u88c5\u5b8c\u6574\u7684 JindoSDK\uff0c\u53ef\u4ee5\u90e8\u7f72\u5b89\u88c5\u8fd9\u7c7b\u4ea7\u51fa\u7269\uff0c\u4f46\u8be5\u4ea7\u51fa\u7269\u4e00\u822c\u8f83\u5927\u3002 lite \u4ea7\u51fa\u7269 \u00a7 \u5982\u679c\u5e0c\u671b\u5728 Hadoop \u73af\u5883\u4e2d\u4ec5\u4f7f\u7528 JindoSDK \u4e2d\u7684 hadoop sdk\uff0c\u53ef\u4ee5\u4f7f\u7528\u6b64\u7c7b\u4ea7\u51fa\u7269\u3002 \u4f7f\u7528Java SDK\u65f6\uff0cLinux X86\u5e73\u53f0\u9700\u8981\u4f9d\u8d56 jindo-sdk-x.y.z.jar \u548c jindo-core-x.y.z.jar \uff0c\u5176\u4ed6\u5e73\u53f0\u9700\u8981\u4f9d\u8d56 jindo-sdk-x.y.z.jar \u3001 jindo-core-x.y.z.jar \u548c jindo-core-<\u5e73\u53f0\u540d\u79f0>-x.y.z .jar\u3002 \u5b89\u88c5\u65b9\u5f0f\u7684\u8bdd\uff0c\u5373\u53ef\u4ee5\u901a\u8fc7maven\u5728pom.xml\u4e2d\u914d\u7f6e\u4f9d\u8d56\uff0c\u5982 6.7.3-oss-maven \u3002\u4e5f\u53ef\u4ee5\u901a\u8fc7oss\u5730\u5740\u76f4\u63a5\u4e0b\u8f7d\u90e8\u7f72\uff0c\u5982 6.7.3-download \u3002 \u4ea7\u7269\u7269\u5e73\u53f0\u8bf4\u660e\u8868 \u00a7 \u4ea7\u51fa\u7269\u540d\u79f0 \u8bf4\u660e jindosdk-x.y.z-linux.tar.gz \u652f\u6301\u5927\u591a\u6570 linux x86 \u53d1\u884c\u7248\u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.17+ \u7248\u672c jindosdk-x.y.z-linux-el7-aarch64.tar.gz \u652f\u6301\u5927\u591a\u6570 linux aarch64 \u53d1\u884c\u7248\u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.17+ \u7248\u672c jindosdk-x.y.z-linux-el6-x86_64.tar.gz \u652f\u6301 centos6 x86 \u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.12+ \u7248\u672c jindosdk-x.y.z-linux-ubuntu22-x86_64.tar.gz \u652f\u6301 ubuntu22 x86 \u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.34+ \u7248\u672c jindosdk-x.y.z-macos-11_0-x86_64.tar.gz \u652f\u6301 macos x86 \u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 macos 11.0 \u4ee5\u4e0a\u7248\u672c jindosdk-x.y.z-macos-11_0-aarch64.tar.gz \u652f\u6301 macos m1/m2 \u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 macos 11.0 \u4ee5\u4e0a\u7248\u672c jindo-sdk-x.y.z.jar \u652f\u6301\u6240\u6709\u5e73\u53f0\u7684 lite \u4ea7\u51fa\u7269\uff0c\u7eaf java \u5b9e\u73b0\uff0c\u4e0d\u5305\u542b native \u5b9e\u73b0 jindo-core-x.y.z.jar \u652f\u6301\u5927\u591a\u6570 linux x86 \u53d1\u884c\u7248\u7684 lite \u4ea7\u51fa\u7269\uff0c\u5305\u542b native \u5b9e\u73b0\uff0c\u517c\u5bb9 glibc 2.17+ \u7248\u672c jindo-core-linux-el7-aarch64-x.y.z.jar \u652f\u6301\u5927\u591a\u6570 linux aarch64 \u53d1\u884c\u7248\u7684 lite \u4ea7\u51fa\u7269\uff0c\u5305\u542b native \u5b9e\u73b0\uff0c\u517c\u5bb9 glibc 2.17+ \u7248\u672c jindo-core-linux-el6-x86_64-x.y.z.jar \u652f\u6301 centos6 x86 \u7684 lite \u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.12+ \u7248\u672c jindo-core-linux-ubuntu22-x86_64-x.y.z.jar \u652f\u6301 ubuntu22 x86 \u7684 lite \u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.34+ \u7248\u672c jindo-core-macos-11_0-x86_64-x.y.z.jar \u652f\u6301 macos x86 \u7684 lite \u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 macos 11.0 \u4ee5\u4e0a\u7248\u672c jindo-core-macos-11_0-aarch64-x.y.z.jar \u652f\u6301 macos m1/m2 \u7684 lite \u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 macos 11.0 \u4ee5\u4e0a\u7248\u672c ps: 6.5.0 \u7248\u672c\u5f00\u59cb tar \u5305\u4e2d\u7684 lite jar \u9ed8\u8ba4\u4f7f\u7528 nextarch classifer\uff0c\u5373 jindo-core-x.y.z-nextarch.jar\u3001jindo-core-linux-el7-aarch64-x.y.z-nextarch.jar \u3002 JindoSDK \u90e8\u7f72\u793a\u4f8b \u00a7 \u4ee5\u4e0b 6.7.3 \u7248\u672c\u4e3a\u4f8b\u3002 Linux x86 \u73af\u5883\u793a\u4f8b \u00a7 \u5b8c\u6574\u5b89\u88c5\uff0c\u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u89e3\u538b\u5728 /usr/lib/jindosdk-6.7.3-linux \u76ee\u5f55\u4e3a\u4f8b\uff1a wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz tar zxvf jindosdk-6.7.3-linux.tar.gz -C /usr/lib/ export JINDOSDK_HOME=/usr/lib/jindosdk-6.7.3-linux export JINDOSDK_CONF_DIR=${JINDOSDK_HOME}/conf export PATH=${PATH}:${JINDOSDK_HOME}/bin export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${JINDOSDK_HOME}/lib/native export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:${JINDOSDK_HOME}/lib/* lite\u5b89\u88c5\uff0c\u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u4e0b\u8f7d\u5230 /share/hadoop/hdfs/lib/ \u76ee\u5f55\u4e3a\u4f8b\uff1a wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-sdk/6.7.3/jindo-sdk-6.7.3.jar wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-core/6.7.3/jindo-core-6.7.3.jar cp jindo-core-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ Linux aarch64 \u73af\u5883\u793a\u4f8b \u00a7 \u5b8c\u6574\u5b89\u88c5\uff0c\u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u89e3\u538b\u5728 /usr/lib/jindosdk-6.7.3-linux-el7-aarch64 \u76ee\u5f55\u4e3a\u4f8b\uff1a wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/x.y.z/jindosdk-6.7.3-linux-el7-aarch64.tar.gz tar zxvf jindosdk-6.7.3-linux-el7-aarch64.tar.gz -C /usr/lib/ export JINDOSDK_HOME=/usr/lib/jindosdk-6.7.3-linux-el7-aarch64 export JINDOSDK_CONF_DIR=${JINDOSDK_HOME}/conf export PATH=${PATH}:${JINDOSDK_HOME}/bin export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${JINDOSDK_HOME}/lib/native export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:${JINDOSDK_HOME}/lib/* lite\u5b89\u88c5\uff0c\u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u4e0b\u8f7d\u5230 /share/hadoop/hdfs/lib/ \u76ee\u5f55\u4e3a\u4f8b\uff1a wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-sdk/6.7.3/jindo-sdk-6.7.3.jar wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-core-linux-el7-aarch64/6.7.3/jindo-core-linux-el7-aarch64-6.7.3.jar cp jindo-core-linux-el7-aarch64-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindo-core-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ \u9644\u5f55 \u00a7 \u5b8c\u6574\u4ea7\u51fa\u7269\u793a\u4f8b \u00a7 \u4ee5\u4e0b x.y.z \u8868\u793a\u7248\u672c\u53f7\u3002 . \u251c\u2500\u2500 bin \u2502 \u251c\u2500\u2500 fusermount3 \u2502 \u251c\u2500\u2500 jindo \u2502 \u251c\u2500\u2500 jindo-fuse \u2502 \u251c\u2500\u2500 jindobench \u2502 \u251c\u2500\u2500 jindodiag \u2502 \u251c\u2500\u2500 jindosync \u2502 \u2514\u2500\u2500 jindotable \u251c\u2500\u2500 conf \u2502 \u251c\u2500\u2500 core-site.xml.template \u2502 \u251c\u2500\u2500 jindosdk.cfg.template \u2502 \u2514\u2500\u2500 log4j.properties.template \u251c\u2500\u2500 include \u2502 \u251c\u2500\u2500 jdo_api.h \u2502 \u251c\u2500\u2500 jdo_common.h \u2502 \u251c\u2500\u2500 jdo_content_summary.h \u2502 \u251c\u2500\u2500 jdo_data_types.h \u2502 \u251c\u2500\u2500 jdo_defines.h \u2502 \u251c\u2500\u2500 jdo_error.h \u2502 \u251c\u2500\u2500 jdo_file_status.h \u2502 \u251c\u2500\u2500 jdo_list_dir_result.h \u2502 \u251c\u2500\u2500 jdo_option_keys.h \u2502 \u2514\u2500\u2500 jdo_options.h \u251c\u2500\u2500 lib \u2502 \u251c\u2500\u2500 jindo-core-x.y.z-nextarch.jar \u2502 \u251c\u2500\u2500 jindo-core-linux-el7-aarch64-x.y.z-nextarch.jar \u2502 \u251c\u2500\u2500 jindo-sdk-x.y.z-nextarch.jar \u2502 \u251c\u2500\u2500 native \u2502 \u2502 \u251c\u2500\u2500 libfuse3.so.3 \u2502 \u2502 \u251c\u2500\u2500 libjemalloc.so \u2502 \u2502 \u251c\u2500\u2500 libjindo-csdk.so \u2502 \u2502 \u251c\u2500\u2500 libjindo-tensorflow1.15.so \u2502 \u2502 \u251c\u2500\u2500 libjindo-tensorflow2.8.so \u2502 \u2502 \u251c\u2500\u2500 libjindosdk.so \u2502 \u2502 \u251c\u2500\u2500 libjindosdk_c.so -> libjindosdk_c.so.x \u2502 \u2502 \u251c\u2500\u2500 libjindosdk_c.so.x -> libjindosdk_c.so.x.y.z \u2502 \u2502 \u2514\u2500\u2500 libjindosdk_c.so.x.y.z \u2502 \u2514\u2500\u2500 site-packages \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp310-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp311-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp312-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp36-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp37-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp38-abi3-linux_x86_64.whl \u2502 \u2514\u2500\u2500 pyjindo-x.y.z-cp39-abi3-linux_x86_64.whl \u251c\u2500\u2500 plugins \u2502 \u2514\u2500\u2500 libjindo-auth-client-plugin-nextarch.so \u2502 \u2514\u2500\u2500 flink \u2502 \u251c\u2500\u2500 jindo-flink-x.y.z-full.jar \u2502 \u2514\u2500\u2500 jindo-flink-x.y.z.jar \u2502 \u251c\u2500\u2500 spark2 \u2502 \u2502 \u2514\u2500\u2500 jindo-spark2-x.y.z.jar \u2502 \u2514\u2500\u2500 spark3 \u2502 \u2514\u2500\u2500 jindo-spark3-x.y.z.jar \u251c\u2500\u2500 tools \u2502 \u251c\u2500\u2500 jindo-dependence-shaded-x.y.z.jar \u2502 \u251c\u2500\u2500 jindo-distcp-tool-x.y.z.jar \u2502 \u251c\u2500\u2500 jindo-distjob-tool-x.y.z.jar \u2502 \u251c\u2500\u2500 jindotable-hive-tool-x.y.z.jar \u2502 \u2514\u2500\u2500 jindotable-shell-x.y.z.jar \u2514\u2500\u2500 versions \u251c\u2500\u2500 JINDOSDK_VERSION \u251c\u2500\u2500 JINDOTABLE_VERSION \u2514\u2500\u2500 JINDO_CONNECTORS_VERSION 9 directories, 29 files","title":"\u591a\u5e73\u53f0\u90e8\u7f72"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#jindosdk","text":"","title":"\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#jindosdk_1","text":"JindoSDK \u76ee\u524d\u652f\u6301\u5927\u90e8\u5206\u5e38\u89c1\u7684 Linux \u53d1\u884c\u7248\uff0c\u5e76\u652f\u6301 x86/aarch64 \u5e73\u53f0\u3002 \u652f\u6301 MacOS \u64cd\u4f5c\u7cfb\u7edf\uff0c\u6240\u6709\u7248\u672c\u5747\u652f\u6301 x86 \u5e73\u53f0\uff0c\u90e8\u5206\u7248\u672c\u652f\u6301 M1/M2 \u82af\u7247\u7684 arm \u5e73\u53f0\u3002 \u6682\u4e0d\u652f\u6301 Windows \u64cd\u4f5c\u7cfb\u7edf\u3002 \u6682\u4e0d\u652f\u6301\u672a\u517c\u5bb9 glibc \u7684 linux \u53d1\u884c\u7248\uff0c\u5982\u9ed8\u8ba4\u4f7f\u7528 musl \u7684\u53d1\u884c\u7248\u3002","title":"JindoSDK \u591a\u5e73\u53f0\u652f\u6301\u73b0\u72b6"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#_1","text":"\u4ea7\u51fa\u7269\u5305\u542b\u4e24\u7c7b\uff1a\u5b8c\u6574\u4ea7\u51fa\u7269\u3001lite\u4ea7\u51fa\u7269\u3002\u4ee5\u4e0b x.y.z \u8868\u793a\u7248\u672c\u53f7\u3002","title":"\u4ea7\u51fa\u7269"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#_2","text":"\u5728 Aliyun EMR \u4e2d\u9ed8\u8ba4\u5b89\u88c5\u4e86\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u5b89\u88c5\u8def\u5f84\u4e3a/opt/apps/JINDOSDK/jindosdk-current/\u3002 \u5b8c\u6574\u4ea7\u51fa\u7269\u662f\u4e00\u4e2atar\u5305\uff0c\u540d\u79f0\u4e3a jindosdk-x.y.z-<\u5e73\u53f0\u540d\u79f0>.tar.gz \u3002 \u5b8c\u6574\u4ea7\u51fa\u7269\u5305\u542b hadoop sdk\uff0cjindo-fuse, jindo-distcp, jindotable\uff0cjindo flink connector, jindocli \u7b49\u5de5\u5177\uff0c\u53ef\u4ee5\u90e8\u7f72\u4f7f\u7528\u3002\u4e5f\u5305\u542b\u4e8c\u6b21\u5f00\u53d1\u6240\u9700\u7684so\u7b49\u4f9d\u8d56\u3002 \u5982\u679c\u5e0c\u671b\u5b89\u88c5\u5b8c\u6574\u7684 JindoSDK\uff0c\u53ef\u4ee5\u90e8\u7f72\u5b89\u88c5\u8fd9\u7c7b\u4ea7\u51fa\u7269\uff0c\u4f46\u8be5\u4ea7\u51fa\u7269\u4e00\u822c\u8f83\u5927\u3002","title":"\u5b8c\u6574\u4ea7\u51fa\u7269\u8bf4\u660e"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#lite","text":"\u5982\u679c\u5e0c\u671b\u5728 Hadoop \u73af\u5883\u4e2d\u4ec5\u4f7f\u7528 JindoSDK \u4e2d\u7684 hadoop sdk\uff0c\u53ef\u4ee5\u4f7f\u7528\u6b64\u7c7b\u4ea7\u51fa\u7269\u3002 \u4f7f\u7528Java SDK\u65f6\uff0cLinux X86\u5e73\u53f0\u9700\u8981\u4f9d\u8d56 jindo-sdk-x.y.z.jar \u548c jindo-core-x.y.z.jar \uff0c\u5176\u4ed6\u5e73\u53f0\u9700\u8981\u4f9d\u8d56 jindo-sdk-x.y.z.jar \u3001 jindo-core-x.y.z.jar \u548c jindo-core-<\u5e73\u53f0\u540d\u79f0>-x.y.z .jar\u3002 \u5b89\u88c5\u65b9\u5f0f\u7684\u8bdd\uff0c\u5373\u53ef\u4ee5\u901a\u8fc7maven\u5728pom.xml\u4e2d\u914d\u7f6e\u4f9d\u8d56\uff0c\u5982 6.7.3-oss-maven \u3002\u4e5f\u53ef\u4ee5\u901a\u8fc7oss\u5730\u5740\u76f4\u63a5\u4e0b\u8f7d\u90e8\u7f72\uff0c\u5982 6.7.3-download \u3002","title":"lite \u4ea7\u51fa\u7269"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#_3","text":"\u4ea7\u51fa\u7269\u540d\u79f0 \u8bf4\u660e jindosdk-x.y.z-linux.tar.gz \u652f\u6301\u5927\u591a\u6570 linux x86 \u53d1\u884c\u7248\u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.17+ \u7248\u672c jindosdk-x.y.z-linux-el7-aarch64.tar.gz \u652f\u6301\u5927\u591a\u6570 linux aarch64 \u53d1\u884c\u7248\u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.17+ \u7248\u672c jindosdk-x.y.z-linux-el6-x86_64.tar.gz \u652f\u6301 centos6 x86 \u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.12+ \u7248\u672c jindosdk-x.y.z-linux-ubuntu22-x86_64.tar.gz \u652f\u6301 ubuntu22 x86 \u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.34+ \u7248\u672c jindosdk-x.y.z-macos-11_0-x86_64.tar.gz \u652f\u6301 macos x86 \u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 macos 11.0 \u4ee5\u4e0a\u7248\u672c jindosdk-x.y.z-macos-11_0-aarch64.tar.gz \u652f\u6301 macos m1/m2 \u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 macos 11.0 \u4ee5\u4e0a\u7248\u672c jindo-sdk-x.y.z.jar \u652f\u6301\u6240\u6709\u5e73\u53f0\u7684 lite \u4ea7\u51fa\u7269\uff0c\u7eaf java \u5b9e\u73b0\uff0c\u4e0d\u5305\u542b native \u5b9e\u73b0 jindo-core-x.y.z.jar \u652f\u6301\u5927\u591a\u6570 linux x86 \u53d1\u884c\u7248\u7684 lite \u4ea7\u51fa\u7269\uff0c\u5305\u542b native \u5b9e\u73b0\uff0c\u517c\u5bb9 glibc 2.17+ \u7248\u672c jindo-core-linux-el7-aarch64-x.y.z.jar \u652f\u6301\u5927\u591a\u6570 linux aarch64 \u53d1\u884c\u7248\u7684 lite \u4ea7\u51fa\u7269\uff0c\u5305\u542b native \u5b9e\u73b0\uff0c\u517c\u5bb9 glibc 2.17+ \u7248\u672c jindo-core-linux-el6-x86_64-x.y.z.jar \u652f\u6301 centos6 x86 \u7684 lite \u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.12+ \u7248\u672c jindo-core-linux-ubuntu22-x86_64-x.y.z.jar \u652f\u6301 ubuntu22 x86 \u7684 lite \u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.34+ \u7248\u672c jindo-core-macos-11_0-x86_64-x.y.z.jar \u652f\u6301 macos x86 \u7684 lite \u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 macos 11.0 \u4ee5\u4e0a\u7248\u672c jindo-core-macos-11_0-aarch64-x.y.z.jar \u652f\u6301 macos m1/m2 \u7684 lite \u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 macos 11.0 \u4ee5\u4e0a\u7248\u672c ps: 6.5.0 \u7248\u672c\u5f00\u59cb tar \u5305\u4e2d\u7684 lite jar \u9ed8\u8ba4\u4f7f\u7528 nextarch classifer\uff0c\u5373 jindo-core-x.y.z-nextarch.jar\u3001jindo-core-linux-el7-aarch64-x.y.z-nextarch.jar \u3002","title":"\u4ea7\u7269\u7269\u5e73\u53f0\u8bf4\u660e\u8868"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#jindosdk_2","text":"\u4ee5\u4e0b 6.7.3 \u7248\u672c\u4e3a\u4f8b\u3002","title":"JindoSDK \u90e8\u7f72\u793a\u4f8b"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#linux-x86","text":"\u5b8c\u6574\u5b89\u88c5\uff0c\u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u89e3\u538b\u5728 /usr/lib/jindosdk-6.7.3-linux \u76ee\u5f55\u4e3a\u4f8b\uff1a wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz tar zxvf jindosdk-6.7.3-linux.tar.gz -C /usr/lib/ export JINDOSDK_HOME=/usr/lib/jindosdk-6.7.3-linux export JINDOSDK_CONF_DIR=${JINDOSDK_HOME}/conf export PATH=${PATH}:${JINDOSDK_HOME}/bin export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${JINDOSDK_HOME}/lib/native export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:${JINDOSDK_HOME}/lib/* lite\u5b89\u88c5\uff0c\u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u4e0b\u8f7d\u5230 /share/hadoop/hdfs/lib/ \u76ee\u5f55\u4e3a\u4f8b\uff1a wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-sdk/6.7.3/jindo-sdk-6.7.3.jar wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-core/6.7.3/jindo-core-6.7.3.jar cp jindo-core-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"Linux x86 \u73af\u5883\u793a\u4f8b"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#linux-aarch64","text":"\u5b8c\u6574\u5b89\u88c5\uff0c\u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u89e3\u538b\u5728 /usr/lib/jindosdk-6.7.3-linux-el7-aarch64 \u76ee\u5f55\u4e3a\u4f8b\uff1a wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/x.y.z/jindosdk-6.7.3-linux-el7-aarch64.tar.gz tar zxvf jindosdk-6.7.3-linux-el7-aarch64.tar.gz -C /usr/lib/ export JINDOSDK_HOME=/usr/lib/jindosdk-6.7.3-linux-el7-aarch64 export JINDOSDK_CONF_DIR=${JINDOSDK_HOME}/conf export PATH=${PATH}:${JINDOSDK_HOME}/bin export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${JINDOSDK_HOME}/lib/native export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:${JINDOSDK_HOME}/lib/* lite\u5b89\u88c5\uff0c\u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u4e0b\u8f7d\u5230 /share/hadoop/hdfs/lib/ \u76ee\u5f55\u4e3a\u4f8b\uff1a wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-sdk/6.7.3/jindo-sdk-6.7.3.jar wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-core-linux-el7-aarch64/6.7.3/jindo-core-linux-el7-aarch64-6.7.3.jar cp jindo-core-linux-el7-aarch64-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindo-core-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"Linux aarch64 \u73af\u5883\u793a\u4f8b"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#_4","text":"","title":"\u9644\u5f55"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#_5","text":"\u4ee5\u4e0b x.y.z \u8868\u793a\u7248\u672c\u53f7\u3002 . \u251c\u2500\u2500 bin \u2502 \u251c\u2500\u2500 fusermount3 \u2502 \u251c\u2500\u2500 jindo \u2502 \u251c\u2500\u2500 jindo-fuse \u2502 \u251c\u2500\u2500 jindobench \u2502 \u251c\u2500\u2500 jindodiag \u2502 \u251c\u2500\u2500 jindosync \u2502 \u2514\u2500\u2500 jindotable \u251c\u2500\u2500 conf \u2502 \u251c\u2500\u2500 core-site.xml.template \u2502 \u251c\u2500\u2500 jindosdk.cfg.template \u2502 \u2514\u2500\u2500 log4j.properties.template \u251c\u2500\u2500 include \u2502 \u251c\u2500\u2500 jdo_api.h \u2502 \u251c\u2500\u2500 jdo_common.h \u2502 \u251c\u2500\u2500 jdo_content_summary.h \u2502 \u251c\u2500\u2500 jdo_data_types.h \u2502 \u251c\u2500\u2500 jdo_defines.h \u2502 \u251c\u2500\u2500 jdo_error.h \u2502 \u251c\u2500\u2500 jdo_file_status.h \u2502 \u251c\u2500\u2500 jdo_list_dir_result.h \u2502 \u251c\u2500\u2500 jdo_option_keys.h \u2502 \u2514\u2500\u2500 jdo_options.h \u251c\u2500\u2500 lib \u2502 \u251c\u2500\u2500 jindo-core-x.y.z-nextarch.jar \u2502 \u251c\u2500\u2500 jindo-core-linux-el7-aarch64-x.y.z-nextarch.jar \u2502 \u251c\u2500\u2500 jindo-sdk-x.y.z-nextarch.jar \u2502 \u251c\u2500\u2500 native \u2502 \u2502 \u251c\u2500\u2500 libfuse3.so.3 \u2502 \u2502 \u251c\u2500\u2500 libjemalloc.so \u2502 \u2502 \u251c\u2500\u2500 libjindo-csdk.so \u2502 \u2502 \u251c\u2500\u2500 libjindo-tensorflow1.15.so \u2502 \u2502 \u251c\u2500\u2500 libjindo-tensorflow2.8.so \u2502 \u2502 \u251c\u2500\u2500 libjindosdk.so \u2502 \u2502 \u251c\u2500\u2500 libjindosdk_c.so -> libjindosdk_c.so.x \u2502 \u2502 \u251c\u2500\u2500 libjindosdk_c.so.x -> libjindosdk_c.so.x.y.z \u2502 \u2502 \u2514\u2500\u2500 libjindosdk_c.so.x.y.z \u2502 \u2514\u2500\u2500 site-packages \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp310-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp311-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp312-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp36-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp37-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp38-abi3-linux_x86_64.whl \u2502 \u2514\u2500\u2500 pyjindo-x.y.z-cp39-abi3-linux_x86_64.whl \u251c\u2500\u2500 plugins \u2502 \u2514\u2500\u2500 libjindo-auth-client-plugin-nextarch.so \u2502 \u2514\u2500\u2500 flink \u2502 \u251c\u2500\u2500 jindo-flink-x.y.z-full.jar \u2502 \u2514\u2500\u2500 jindo-flink-x.y.z.jar \u2502 \u251c\u2500\u2500 spark2 \u2502 \u2502 \u2514\u2500\u2500 jindo-spark2-x.y.z.jar \u2502 \u2514\u2500\u2500 spark3 \u2502 \u2514\u2500\u2500 jindo-spark3-x.y.z.jar \u251c\u2500\u2500 tools \u2502 \u251c\u2500\u2500 jindo-dependence-shaded-x.y.z.jar \u2502 \u251c\u2500\u2500 jindo-distcp-tool-x.y.z.jar \u2502 \u251c\u2500\u2500 jindo-distjob-tool-x.y.z.jar \u2502 \u251c\u2500\u2500 jindotable-hive-tool-x.y.z.jar \u2502 \u2514\u2500\u2500 jindotable-shell-x.y.z.jar \u2514\u2500\u2500 versions \u251c\u2500\u2500 JINDOSDK_VERSION \u251c\u2500\u2500 JINDOTABLE_VERSION \u2514\u2500\u2500 JINDO_CONNECTORS_VERSION 9 directories, 29 files","title":"\u5b8c\u6574\u4ea7\u51fa\u7269\u793a\u4f8b"},{"location":"jindosdk/jindosdk_download/","text":"JindoData v6.7.3 \u4e0b\u8f7d \u00a7 \u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72\u8bf4\u660e\uff0c\u53c2\u89c1 \u6587\u6863\u94fe\u63a5 \u5386\u53f2\u7248\u672c\u95ee\u9898\u53ca\u53d1\u5e03\u8bf4\u660e\uff0c\u53c2\u89c1 \u6587\u6863\u94fe\u63a5 \u4e0b\u8f7d\u94fe\u63a5 \u00a7 \u5b8c\u6574\u4ea7\u51fa\u7269 \u00a7 jindosdk-6.7.3-linux.tar.gz jindosdk-6.7.3-linux-el7-aarch64.tar.gz jindosdk-6.7.3-linux-el6-x86_64.tar.gz jindosdk-6.7.3-linux-ubuntu22-x86_64.tar.gz jindosdk-6.7.3-macos-11_0-x86_64.tar.gz jindosdk-6.7.3-macos-11_0-aarch64.tar.gz lite \u4ea7\u51fa\u7269 \u00a7 jindo-core-6.7.3.jar jindo-sdk-6.7.3.jar jindo-core-linux-el7-aarch64-6.7.3.jar jindo-core-linux-el6-x86_64-6.7.3.jar jindo-core-linux-ubuntu22-x86_64-6.7.3.jar jindo-core-macos-11_0-x86_64-6.7.3.jar jindo-core-macos-11_0-aarch64-6.7.3.jar lite \u4ea7\u51fa\u7269 (nextarch classifier) \u00a7 jindo-core-6.7.3-nextarch.jar jindo-sdk-6.7.3-nextarch.jar jindo-core-linux-el7-aarch64-6.7.3-nextarch.jar jindo-core-linux-el6-x86_64-6.7.3-nextarch.jar jindo-core-linux-ubuntu22-x86_64-6.7.3-nextarch.jar jindo-core-macos-11_0-x86_64-6.7.3-nextarch.jar jindo-core-macos-11_0-aarch64-6.7.3-nextarch.jar","title":"JindoSDK \u4e0b\u8f7d"},{"location":"jindosdk/jindosdk_download/#jindodata-v673","text":"\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72\u8bf4\u660e\uff0c\u53c2\u89c1 \u6587\u6863\u94fe\u63a5 \u5386\u53f2\u7248\u672c\u95ee\u9898\u53ca\u53d1\u5e03\u8bf4\u660e\uff0c\u53c2\u89c1 \u6587\u6863\u94fe\u63a5","title":"JindoData v6.7.3 \u4e0b\u8f7d"},{"location":"jindosdk/jindosdk_download/#_1","text":"","title":"\u4e0b\u8f7d\u94fe\u63a5"},{"location":"jindosdk/jindosdk_download/#_2","text":"jindosdk-6.7.3-linux.tar.gz jindosdk-6.7.3-linux-el7-aarch64.tar.gz jindosdk-6.7.3-linux-el6-x86_64.tar.gz jindosdk-6.7.3-linux-ubuntu22-x86_64.tar.gz jindosdk-6.7.3-macos-11_0-x86_64.tar.gz jindosdk-6.7.3-macos-11_0-aarch64.tar.gz","title":"\u5b8c\u6574\u4ea7\u51fa\u7269"},{"location":"jindosdk/jindosdk_download/#lite","text":"jindo-core-6.7.3.jar jindo-sdk-6.7.3.jar jindo-core-linux-el7-aarch64-6.7.3.jar jindo-core-linux-el6-x86_64-6.7.3.jar jindo-core-linux-ubuntu22-x86_64-6.7.3.jar jindo-core-macos-11_0-x86_64-6.7.3.jar jindo-core-macos-11_0-aarch64-6.7.3.jar","title":"lite \u4ea7\u51fa\u7269"},{"location":"jindosdk/jindosdk_download/#lite-nextarch-classifier","text":"jindo-core-6.7.3-nextarch.jar jindo-sdk-6.7.3-nextarch.jar jindo-core-linux-el7-aarch64-6.7.3-nextarch.jar jindo-core-linux-el6-x86_64-6.7.3-nextarch.jar jindo-core-linux-ubuntu22-x86_64-6.7.3-nextarch.jar jindo-core-macos-11_0-x86_64-6.7.3-nextarch.jar jindo-core-macos-11_0-aarch64-6.7.3-nextarch.jar","title":"lite \u4ea7\u51fa\u7269 (nextarch classifier)"},{"location":"jindosdk/jindosdk_endpoint_configuration/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Endpoint \u914d\u7f6e \u00a7 \u6709\u4e09\u79cd\u65b9\u5f0f\u53ef\u4ee5\u6307\u5b9a Endpoint\uff0cJindoSDK \u4f1a\u4ee5\u4e0b\u4f18\u5148\u7ea7\u67e5\u627e Endpoint\uff1a \u65b9\u5f0f\u4e00\uff1a \u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u6307\u5b9a Endpoint\u3002\uff08\u63a8\u8350\uff09 \u00a7 \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u4f7f\u7528\u8fd9\u79cd\u683c\u5f0f\u7684\u8bbf\u95ee\u8def\u5f84 oss://<Bucket>.<Endpoint>/<Object> , \u5982: oss://example-oss-bucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u63a5\u53e3\u3002 \u65b9\u5f0f\u4e8c\uff1a \u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint\u3002 \u00a7 \u5982\u679c\u4f7f\u7528 oss://<Bucket>/<Object> \u8fd9\u79cd\u8bbf\u95ee\u8def\u5f84\uff0c\u5373\u8bbf\u95ee\u8def\u5f84\u4e2d\u6ca1\u6709 Endpoint, JindoSDK \u5219\u4f1a\u5728\u914d\u7f6e\u4e2d\u67e5\u627e Bucket \u7ea7\u522b\u7684 Endpoint\u3002 \u53ef\u5728 Hadoop \u7684 core-site.xml \u4e2d\u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint \u6307\u5411 OSS-HDFS \u670d\u52a1\u7684 Endpoint\u3002 <configuration> <property> <name>fs.oss.bucket.XXX.endpoint</name> <value>cn-xxx.oss-dls.aliyuncs.com</value> </property> </configuration> \u8bf4\u660e: XXX \u4e3a OSS-HDFS \u670d\u52a1 Bucket \u540d\u79f0\u3002 \u65b9\u5f0f\u4e09\uff1a\u914d\u7f6e\u5168\u5c40\u9ed8\u8ba4 Endpoint\u3002 \u00a7 \u5982\u679c\u4f7f\u7528 oss://<Bucket>/<Object> \u8fd9\u79cd\u8bbf\u95ee\u8def\u5f84\u800c\u4e14\u6ca1\u6709\u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint\uff0c\u5219\u4f1a\u7528\u5168\u5c40 Endpoint \u8bbf\u95ee\uff0c\u5982\u679c\u8981\u9ed8\u8ba4\u4f7f\u7528\u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff0c\u5219\u53ef\u7528\u8be5\u65b9\u6cd5\u3002 \u53ef\u5728 Hadoop \u7684 core-site.xml \u4e2d\u914d\u7f6e\u5168\u5c40\u9ed8\u8ba4 Endpoint\u3002 <configuration> <property> <name>fs.oss.endpoint</name> <value>cn-xxx.oss-dls.aliyuncs.com</value> </property> </configuration>","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Endpoint \u914d\u7f6e"},{"location":"jindosdk/jindosdk_endpoint_configuration/#oss-hdfs-jindofs-endpoint","text":"\u6709\u4e09\u79cd\u65b9\u5f0f\u53ef\u4ee5\u6307\u5b9a Endpoint\uff0cJindoSDK \u4f1a\u4ee5\u4e0b\u4f18\u5148\u7ea7\u67e5\u627e Endpoint\uff1a","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Endpoint \u914d\u7f6e"},{"location":"jindosdk/jindosdk_endpoint_configuration/#endpoint","text":"\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u4f7f\u7528\u8fd9\u79cd\u683c\u5f0f\u7684\u8bbf\u95ee\u8def\u5f84 oss://<Bucket>.<Endpoint>/<Object> , \u5982: oss://example-oss-bucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u63a5\u53e3\u3002","title":"\u65b9\u5f0f\u4e00\uff1a \u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u6307\u5b9a Endpoint\u3002\uff08\u63a8\u8350\uff09"},{"location":"jindosdk/jindosdk_endpoint_configuration/#bucket-endpoint","text":"\u5982\u679c\u4f7f\u7528 oss://<Bucket>/<Object> \u8fd9\u79cd\u8bbf\u95ee\u8def\u5f84\uff0c\u5373\u8bbf\u95ee\u8def\u5f84\u4e2d\u6ca1\u6709 Endpoint, JindoSDK \u5219\u4f1a\u5728\u914d\u7f6e\u4e2d\u67e5\u627e Bucket \u7ea7\u522b\u7684 Endpoint\u3002 \u53ef\u5728 Hadoop \u7684 core-site.xml \u4e2d\u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint \u6307\u5411 OSS-HDFS \u670d\u52a1\u7684 Endpoint\u3002 <configuration> <property> <name>fs.oss.bucket.XXX.endpoint</name> <value>cn-xxx.oss-dls.aliyuncs.com</value> </property> </configuration> \u8bf4\u660e: XXX \u4e3a OSS-HDFS \u670d\u52a1 Bucket \u540d\u79f0\u3002","title":"\u65b9\u5f0f\u4e8c\uff1a \u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint\u3002"},{"location":"jindosdk/jindosdk_endpoint_configuration/#endpoint_1","text":"\u5982\u679c\u4f7f\u7528 oss://<Bucket>/<Object> \u8fd9\u79cd\u8bbf\u95ee\u8def\u5f84\u800c\u4e14\u6ca1\u6709\u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint\uff0c\u5219\u4f1a\u7528\u5168\u5c40 Endpoint \u8bbf\u95ee\uff0c\u5982\u679c\u8981\u9ed8\u8ba4\u4f7f\u7528\u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff0c\u5219\u53ef\u7528\u8be5\u65b9\u6cd5\u3002 \u53ef\u5728 Hadoop \u7684 core-site.xml \u4e2d\u914d\u7f6e\u5168\u5c40\u9ed8\u8ba4 Endpoint\u3002 <configuration> <property> <name>fs.oss.endpoint</name> <value>cn-xxx.oss-dls.aliyuncs.com</value> </property> </configuration>","title":"\u65b9\u5f0f\u4e09\uff1a\u914d\u7f6e\u5168\u5c40\u9ed8\u8ba4 Endpoint\u3002"},{"location":"jindosdk/jindosdk_memory_best_practice/","text":"JindoSDk \u901a\u8fc7 jemalloc \u63a7\u5236\u5185\u5b58\u4e0a\u6da8\u6700\u4f73\u5b9e\u8df5 \u00a7 \u80cc\u666f \u00a7 jindosdk hadoop sdk \u4f7f\u7528 jni \u8c03\u7528 native \u5b9e\u73b0\uff0c\u56e0\u6b64 jvm \u76f8\u5173 gc \u53c2\u6570\u65e0\u6cd5\u6709\u6548\u63a7\u5236 native \u5185\u5b58\u4e0a\u6da8\uff0c\u53ef\u4ee5\u901a\u8fc7 jemalloc \u914d\u7f6e\u6765\u8fdb\u884c\u63a7\u5236\u3002 \u90e8\u5c5e\u65b9\u6cd5 \u00a7 jindosdk 4.6.8 \u53ca\u4ee5\u4e0a\u7248\u672c\u5b8c\u6574\u4ea7\u51fa\u7269\u5728 tar \u5305\u4e2d\u9ed8\u8ba4\u643a\u5e26\u4e86 jemalloc\uff1a . \u251c\u2500\u2500 bin \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 conf \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 lib \u2502 \u251c\u2500\u2500 jindo-core-6.7.3.jar \u2502 \u251c\u2500\u2500 jindo-sdk-6.7.3.jar \u2502 \u2514\u2500\u2500 native \u2502 \u251c\u2500\u2500 libjemalloc.so \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 plugins \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 tools \u2502 \u251c\u2500\u2500 ... \u2514\u2500\u2500 versions \u2514\u2500\u2500 JINDOSDK_VERSION \u5047\u8bbe\u5b89\u88c5\u76ee\u5f55\u4e3a/opt/apps/JINDOSDK/jindosdk-current/\uff0c\u8981\u4fdd\u8bc1\u6240\u6709\u8c03\u7528jindosdk\u7684\u73af\u5883\uff0c\u90fd\u914d\u7f6e\u4ee5\u4e0b\u7684\u73af\u5883\u53d8\u91cf\uff1a LD_PRELOAD=/opt/apps/JINDOSDK/jindosdk-current/lib/native/libjemalloc.so MALLOC_CONF=narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0,background_thread:true,abort_conf:true \u6548\u679c \u00a7 wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/memcheck/jindo-sdk-memleak-check-1.0.jar \u6267\u884c\u4ee5\u4e0bmemleak-check\u6d4b\u8bd5 HADOOP_HEAPSIZE=2560 HADOOP_ROOT_LOGGER=ERROR,console hadoop jar jindo-sdk-memleak-check-1.0.jar MemLeakTestProgram -baseDir oss:/<your-bucket>/ -duration 8899 \u4f7f\u7528\u9ed8\u8ba4glibc\uff0c\u6700\u7ec8\u5360\u75283627M \u4f7f\u7528jemalloc\u4e0d\u8bbe\u7f6eMALLOC_CONF\uff0c\u6700\u7ec8\u5360\u75282929M \u4f7f\u7528jemalloc\u5e76\u8bbe\u7f6eMALLOC_CONF\uff0c\u6700\u7ec8\u5360\u75282289M \u5176\u4ed6 \u00a7 \u4e0a\u9762\u7684\u914d\u7f6e\u9002\u5408\u5185\u5b58\u7d27\u5f20\u7684\u573a\u666f\uff0c\u53ef\u80fd\u5bf9\u6027\u80fd\u6709\u4e00\u5b9a\u5f71\u54cd\u3002\u5f53\u9700\u8981\u7efc\u5408\u8003\u8651\u5185\u5b58\u5360\u7528\u4e0e\u6027\u80fd\u65f6\uff0c\u53ef\u4ee5\u5c1d\u8bd5\u4ee5\u4e0b\u914d\u7f6e\uff1a LD_PRELOAD=/opt/apps/JINDOSDK/jindosdk-current/lib/native/libjemalloc.so MALLOC_CONF=percpu_arena:percpu,lg_tcache_max:12,dirty_decay_ms:5000,muzzy_decay_ms:5000","title":"JindoSDk \u901a\u8fc7 jemalloc \u63a7\u5236\u5185\u5b58\u4e0a\u6da8\u6700\u4f73\u5b9e\u8df5"},{"location":"jindosdk/jindosdk_memory_best_practice/#jindosdk-jemalloc","text":"","title":"JindoSDk \u901a\u8fc7 jemalloc \u63a7\u5236\u5185\u5b58\u4e0a\u6da8\u6700\u4f73\u5b9e\u8df5"},{"location":"jindosdk/jindosdk_memory_best_practice/#_1","text":"jindosdk hadoop sdk \u4f7f\u7528 jni \u8c03\u7528 native \u5b9e\u73b0\uff0c\u56e0\u6b64 jvm \u76f8\u5173 gc \u53c2\u6570\u65e0\u6cd5\u6709\u6548\u63a7\u5236 native \u5185\u5b58\u4e0a\u6da8\uff0c\u53ef\u4ee5\u901a\u8fc7 jemalloc \u914d\u7f6e\u6765\u8fdb\u884c\u63a7\u5236\u3002","title":"\u80cc\u666f"},{"location":"jindosdk/jindosdk_memory_best_practice/#_2","text":"jindosdk 4.6.8 \u53ca\u4ee5\u4e0a\u7248\u672c\u5b8c\u6574\u4ea7\u51fa\u7269\u5728 tar \u5305\u4e2d\u9ed8\u8ba4\u643a\u5e26\u4e86 jemalloc\uff1a . \u251c\u2500\u2500 bin \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 conf \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 lib \u2502 \u251c\u2500\u2500 jindo-core-6.7.3.jar \u2502 \u251c\u2500\u2500 jindo-sdk-6.7.3.jar \u2502 \u2514\u2500\u2500 native \u2502 \u251c\u2500\u2500 libjemalloc.so \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 plugins \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 tools \u2502 \u251c\u2500\u2500 ... \u2514\u2500\u2500 versions \u2514\u2500\u2500 JINDOSDK_VERSION \u5047\u8bbe\u5b89\u88c5\u76ee\u5f55\u4e3a/opt/apps/JINDOSDK/jindosdk-current/\uff0c\u8981\u4fdd\u8bc1\u6240\u6709\u8c03\u7528jindosdk\u7684\u73af\u5883\uff0c\u90fd\u914d\u7f6e\u4ee5\u4e0b\u7684\u73af\u5883\u53d8\u91cf\uff1a LD_PRELOAD=/opt/apps/JINDOSDK/jindosdk-current/lib/native/libjemalloc.so MALLOC_CONF=narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0,background_thread:true,abort_conf:true","title":"\u90e8\u5c5e\u65b9\u6cd5"},{"location":"jindosdk/jindosdk_memory_best_practice/#_3","text":"wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/memcheck/jindo-sdk-memleak-check-1.0.jar \u6267\u884c\u4ee5\u4e0bmemleak-check\u6d4b\u8bd5 HADOOP_HEAPSIZE=2560 HADOOP_ROOT_LOGGER=ERROR,console hadoop jar jindo-sdk-memleak-check-1.0.jar MemLeakTestProgram -baseDir oss:/<your-bucket>/ -duration 8899 \u4f7f\u7528\u9ed8\u8ba4glibc\uff0c\u6700\u7ec8\u5360\u75283627M \u4f7f\u7528jemalloc\u4e0d\u8bbe\u7f6eMALLOC_CONF\uff0c\u6700\u7ec8\u5360\u75282929M \u4f7f\u7528jemalloc\u5e76\u8bbe\u7f6eMALLOC_CONF\uff0c\u6700\u7ec8\u5360\u75282289M","title":"\u6548\u679c"},{"location":"jindosdk/jindosdk_memory_best_practice/#_4","text":"\u4e0a\u9762\u7684\u914d\u7f6e\u9002\u5408\u5185\u5b58\u7d27\u5f20\u7684\u573a\u666f\uff0c\u53ef\u80fd\u5bf9\u6027\u80fd\u6709\u4e00\u5b9a\u5f71\u54cd\u3002\u5f53\u9700\u8981\u7efc\u5408\u8003\u8651\u5185\u5b58\u5360\u7528\u4e0e\u6027\u80fd\u65f6\uff0c\u53ef\u4ee5\u5c1d\u8bd5\u4ee5\u4e0b\u914d\u7f6e\uff1a LD_PRELOAD=/opt/apps/JINDOSDK/jindosdk-current/lib/native/libjemalloc.so MALLOC_CONF=percpu_arena:percpu,lg_tcache_max:12,dirty_decay_ms:5000,muzzy_decay_ms:5000","title":"\u5176\u4ed6"},{"location":"jindosdk/jindosdk_quickstart/","text":"Hadoop \u73af\u5883\u4f7f\u7528 JindoSDK \u5feb\u901f\u5165\u95e8 \u00a7 JindoSDK \u5168\u9762\u517c\u5bb9 Hadoop FileSystem \u63a5\u53e3\uff0c\u63d0\u4f9b\u4e86\u66f4\u597d\u7684\u517c\u5bb9\u6027\u548c\u6613\u7528\u6027\uff0c\u53c8\u80fd\u517c\u5177OSS\u7684\u6027\u80fd\u548c\u6210\u672c\u4f18\u52bf\u3002 \u76ee\u524d\u652f\u6301\u5e02\u9762\u4e0a\u5927\u90e8\u5206 Hadoop \u7248\u672c\uff0c\u5728 Hadoop 2.3 \u53ca\u4ee5\u4e0a\u7684\u7248\u672c\u4e0a\u9a8c\u8bc1\u901a\u8fc7\uff082.3 \u4ee5\u524d\u7248\u672c\u6682\u672a\u6d4b\u8bd5\uff0c\u5982\u6709\u95ee\u9898\u8bf7 \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988\uff09\u3002 \u672c\u6587\u6863\u5047\u8bbe\u60a8\u5728 Linux x86 \u73af\u5883\u4e2d\uff0c\u5df2\u7ecf\u5b89\u88c5\u90e8\u7f72\u597d\u4e86 Hadoop \u73af\u5883\uff0c\u5982\u9700\u5728\u5176\u4ed6\u5e73\u53f0\u4f7f\u7528\uff0c\u8bf7\u53c2\u8003 \u300a\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK\u300b \u3002 \u6b65\u9aa4 \u00a7 1. \u5b89\u88c5\u90e8\u7f72 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz\uff0c \u4e0b\u8f7d\u9875\u9762 \u3002 \u5b8c\u6574\u90e8\u7f72 JindoSDK\uff0c\u53c2\u89c1 \u6587\u6863\u94fe\u63a5 \u6700\u7b80\u90e8\u7f72 JindoSDK\uff0c\u53c2\u89c1 \u6587\u6863\u94fe\u63a5 2. \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u00a7 \u7528 Hadoop Shell \u8bbf\u95ee OSS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls oss://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<bucket>/<path> \u66f4\u591a Hadoop \u547d\u4ee4\u53c2\u89c1 \u300a\u901a\u8fc7 Hadoop Shell \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS\u300b 3. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u5feb\u901f\u5165\u95e8"},{"location":"jindosdk/jindosdk_quickstart/#hadoop-jindosdk","text":"JindoSDK \u5168\u9762\u517c\u5bb9 Hadoop FileSystem \u63a5\u53e3\uff0c\u63d0\u4f9b\u4e86\u66f4\u597d\u7684\u517c\u5bb9\u6027\u548c\u6613\u7528\u6027\uff0c\u53c8\u80fd\u517c\u5177OSS\u7684\u6027\u80fd\u548c\u6210\u672c\u4f18\u52bf\u3002 \u76ee\u524d\u652f\u6301\u5e02\u9762\u4e0a\u5927\u90e8\u5206 Hadoop \u7248\u672c\uff0c\u5728 Hadoop 2.3 \u53ca\u4ee5\u4e0a\u7684\u7248\u672c\u4e0a\u9a8c\u8bc1\u901a\u8fc7\uff082.3 \u4ee5\u524d\u7248\u672c\u6682\u672a\u6d4b\u8bd5\uff0c\u5982\u6709\u95ee\u9898\u8bf7 \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988\uff09\u3002 \u672c\u6587\u6863\u5047\u8bbe\u60a8\u5728 Linux x86 \u73af\u5883\u4e2d\uff0c\u5df2\u7ecf\u5b89\u88c5\u90e8\u7f72\u597d\u4e86 Hadoop \u73af\u5883\uff0c\u5982\u9700\u5728\u5176\u4ed6\u5e73\u53f0\u4f7f\u7528\uff0c\u8bf7\u53c2\u8003 \u300a\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK\u300b \u3002","title":"Hadoop \u73af\u5883\u4f7f\u7528 JindoSDK \u5feb\u901f\u5165\u95e8"},{"location":"jindosdk/jindosdk_quickstart/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"jindosdk/jindosdk_quickstart/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz\uff0c \u4e0b\u8f7d\u9875\u9762 \u3002 \u5b8c\u6574\u90e8\u7f72 JindoSDK\uff0c\u53c2\u89c1 \u6587\u6863\u94fe\u63a5 \u6700\u7b80\u90e8\u7f72 JindoSDK\uff0c\u53c2\u89c1 \u6587\u6863\u94fe\u63a5","title":"1. \u5b89\u88c5\u90e8\u7f72 JindoSDK"},{"location":"jindosdk/jindosdk_quickstart/#2-jindosdk-oss","text":"\u7528 Hadoop Shell \u8bbf\u95ee OSS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls oss://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<bucket>/<path> \u66f4\u591a Hadoop \u547d\u4ee4\u53c2\u89c1 \u300a\u901a\u8fc7 Hadoop Shell \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS\u300b","title":"2. \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS"},{"location":"jindosdk/jindosdk_quickstart/#3","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"3. \u53c2\u6570\u8c03\u4f18"},{"location":"jindosdk/oss-maven/","text":"JindoData 6.7.3 Maven \u76f8\u5173\u4f9d\u8d56 \u00a7 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 Jindo Flink \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo flink dependency --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-flink</artifactId> <version>${jindodata.version}</version> </dependency> </dependencies> </project> \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoDistCP \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo distcp dependency --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <!-- ... --> <dependencies> <!-- add jindo-core-extended-jar for macos --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian/arm --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> </dependencies> </project>","title":"JindoData 6.7.3 Maven \u76f8\u5173\u4f9d\u8d56"},{"location":"jindosdk/oss-maven/#jindodata-673-maven","text":"\u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 Jindo Flink \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo flink dependency --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-flink</artifactId> <version>${jindodata.version}</version> </dependency> </dependencies> </project> \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoDistCP \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo distcp dependency --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <!-- ... --> <dependencies> <!-- add jindo-core-extended-jar for macos --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian/arm --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> </dependencies> </project>","title":"JindoData 6.7.3 Maven \u76f8\u5173\u4f9d\u8d56"},{"location":"jindosdk/flink/jindosdk_on_flink/","text":"Flink \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 \u5728\u96c6\u7fa4\u4e0a\u6709\u5f00\u6e90\u7248\u672c Flink \u8f6f\u4ef6\uff0c\u7248\u672c\u4e0d\u4f4e\u4e8e 1.10.1\u3002 \u6ce8\uff1aFlink 1.16.0 \u53ca\u66f4\u9ad8\u7248\u672c\u7684\u53ef\u7528\u6027\u5c1a\u672a\u5f97\u5230\u9a8c\u8bc1\u3002 \u4e3a\u4ec0\u4e48 Flink \u9700\u8981\u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u00a7 \u82e5\u9700\u8981 Flink \u4fdd\u6301 EXACTLY_ONCE \u8bed\u4e49\u6d41\u5f0f\u5199\u5165 Aliyun OSS\uff0c\u5219\u9700\u8981\u4f7f\u7528 JindoSDK\u3002 Apache Flink \u662f\u4e00\u79cd\u5f53\u524d\u4e1a\u754c\u6d41\u884c\u7684\u5f00\u6e90\u5927\u6570\u636e\u6d41\u5f0f\u8ba1\u7b97\u5f15\u64ce\uff0c\u652f\u6301 \u201c\u4e25\u683c\u4e00\u6b21\u201d\uff08EXACTLY_ONCE\uff09\u8bed\u4e49\u5199\u5165\u90e8\u5206\u5b58\u50a8\u4ecb\u8d28\u3002Aliyun Object Storage Service (Aliyun OSS) \u5219\u662f\u963f\u91cc\u4e91\u65d7\u4e0b\u7684\u6838\u5fc3\u4ea7\u54c1\u4e4b\u4e00\uff0c\u63d0\u4f9b\u4e91\u539f\u751f\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1\uff0c\u51ed\u501f\u4f18\u5f02\u7684\u6027\u80fd\u548c\u5b89\u5168\u6027\u5df2\u670d\u52a1\u4e8e\u6d77\u91cf\u7528\u6237\u3002 \u76ee\u524d\uff0c\u5f00\u6e90\u7248\u672c Flink \u5bf9\u6d41\u5f0f\u5199\u5165 Aliyun OSS \u5c1a\u4e0d\u80fd\u652f\u6301 EXACTLY_ONCE \u8bed\u4e49\uff0c\u5982\u6709\u8be5\u9700\u6c42\u5219\u9700\u8981\u4f7f\u7528 JindoSDK\u3002 \u6ce8\uff1aApache Flink \u81ea 1.16 \u7248\u672c\u8d77\u539f\u751f\u63d0\u4f9b\u4e86\u6d41\u5f0f\u5199\u5165 Aliyun OSS \u5bf9\u8c61\u5b58\u50a8\u7684\u652f\u6301\u3002JindoSDK \u63d0\u4f9b\u7684\u5b9e\u73b0\u4e0e Apache Flink \u7684\u81ea\u6709\u5b9e\u73b0\u6709\u4e24\u4e2a\u4e3b\u8981\u533a\u522b\uff1a - JindoSDK \u7684\u5b9e\u73b0\u80fd\u591f\u540c\u65f6\u652f\u6301 Aliyun OSS \u5bf9\u8c61\u5b58\u50a8\u548c Aliyun OSS-HDFS \u6587\u4ef6\u5b58\u50a8\u3002\u5bf9 OSS-HDFS \u7684\u4ecb\u7ecd\u53c2\u8003 OSS-HDFS\u670d\u52a1\u6982\u8ff0 - JindoSDK \u7684\u5b9e\u73b0\u5e95\u5c42\u91c7\u7528\u4e86 Native \u65b9\u6848\u3002 JindoSDK \u914d\u7f6e \u00a7 \u9700\u8981\u5728\u6240\u6709 Flink \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flink \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e .jar \u6587\u4ef6\uff1a * jindo-flink-${version}-full.jar \u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 plugins/flink/ \u76ee\u5f55\u4e0b\u627e\u5230\u3002 \u53e6\u5916\uff0c\u5982\u679c\u96c6\u7fa4\u5185\u5b58\u5728 Apache Flink \u63d0\u4f9b\u7684\u81ea\u5e26 Flink OSS Connector\uff0c\u9700\u79fb\u9664\u3002\u79fb\u9664\u65b9\u6cd5\u4e3a\uff0c\u4ece Flink \u96c6\u7fa4\u7684 lib \u76ee\u5f55\u6216 plugins/oss-fs-hadoop \u79fb\u8d70\u5982\u4e0b jar \u5305\uff1a * flink-oss-fs-hadoop-${flink-version}.jar \u6ce8\uff1aJindoSDK \u76ee\u524d\u63d0\u4f9b\u591a\u5e73\u53f0\u652f\u6301\uff0c\u5f53\u524d\u652f\u6301\u7684\u5e73\u53f0\u4ee5\u53ca\u9700\u8981\u989d\u5916\u653e\u7f6e\u7684\u8d44\u6e90\uff0c\u53c2\u8003 \u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK \u5982\u4f55\u4f7f\u7528 \u00a7 \u914d\u7f6e\u597d SDK \u540e\uff0c\u65e0\u9700\u989d\u5916\u914d\u7f6e\uff0c\u4ee5\u5e38\u89c4 Flink \u6d41\u5f0f\u4f5c\u4e1a\u65b9\u6cd5\u4f7f\u7528\u5373\u53ef\uff0c\u6ce8\u610f\u4f7f\u7528\u6b63\u786e\u7684\u8def\u5f84\u3002\u5199\u5165 OSS (Aliyun Object Storage Service) \u987b\u4ee5 oss:// \u4e3a\u524d\u7f00\u3002 \u793a\u4f8b \u00a7 \u4e0b\u9762\u662f\u4e00\u4e2a\u7b80\u5355\u7684\u793a\u4f8b\u7a0b\u5e8f\uff0c\u5982\u679c\u5df2\u7ecf\u6309\u7167\u4e0a\u4e00\u8282\u914d\u7f6e\u597d SDK\uff0c\u5219\u4e0b\u5217\u7a0b\u5e8f\u5e94\u80fd\u6b63\u786e\u8fd0\u884c\uff1a (1) \u901a\u7528\u914d\u7f6e \u6253\u5f00 Flink \u7684 \u201c\u68c0\u67e5\u70b9\u201d\uff08Checkpoint\uff09\u3002\u4f8b\u5982\uff0cenv \u662f\u4e00\u4e2a\u5df2\u7ecf\u5efa\u7acb\u7684 StreamExecutionEnvironment\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u65b9\u5f0f\u5efa\u7acb\uff1a StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); \u5219\u53ef\u4ee5\u8fd9\u6837\u542f\u7528 checkpoint\uff1a env.enableCheckpointing(<userDefinedCheckpointInterval>, CheckpointingMode.EXACTLY_ONCE); \u4f7f\u7528\u53ef\u91cd\u53d1\u7684\u6570\u636e\u6e90\uff0c\u4f8b\u5982 Kafka (2) \u793a\u4f8b\u7a0b\u5e8f \u4e0b\u6587\u4e2d\uff0coutputStream \u662f\u4e00\u4e2a\u9884\u5148\u5f62\u6210\u7684 DataStream<String> \u5bf9\u8c61\uff0c\u5e76\u671f\u671b\u5c06\u5176\u5199\u5165 OSS\uff0c\u90a3\u4e48\u53ef\u4ee5\u8fd9\u6837\u6dfb\u52a0 sink\uff1a String outputPath = \"oss://<user-defined-bucket>/<user-defined-dir>\" StreamingFileSink<String> sink = StreamingFileSink.forRowFormat( new Path(outputPath), new SimpleStringEncoder<String>(\"UTF-8\") ).build(); outputStream.addSink(sink); \u6700\u540e\u7528 env.execute() \u6267\u884c Flink \u4f5c\u4e1a\u5373\u53ef\u3002 \u9ad8\u7ea7\u4f7f\u7528\uff1a\u81ea\u5b9a\u4e49\u53c2\u6570 \u00a7 \u7528\u6237\u5728\u63d0\u4ea4 Flink \u4f5c\u4e1a\u65f6\uff0c\u53ef\u4ee5\u81ea\u5b9a\u4e49\u914d\u7f6e\u4e00\u4e9b\u53c2\u6570\uff0c\u4ee5\u5f00\u542f\u6216\u63a7\u5236\u7279\u5b9a\u529f\u80fd\u3002\u4f8b\u5982\uff0c\u4ee5 yarn-cluster \u6a21\u5f0f\u63d0\u4ea4\u65f6\uff0c\u53ef\u4ee5\u901a\u8fc7 -yD \u8fdb\u884c\u914d\u7f6e\uff0c\u65b9\u5f0f\u7c7b\u4f3c\u4e8e\uff1a <flink_home>/bin/flink run -m yarn-cluster -yD key1=value1 -yD key2=value2 ... \u76ee\u524d\u652f\u6301\u901a\u8fc7\u914d\u7f6e\u5f00\u542f \u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\uff0c\u6216\u8005\u63a7\u5236 \u201c\u5206\u7247\u4e0a\u4f20\u201d \u7684\u5e76\u884c\u5ea6\u3002 \u71b5\u6ce8\u5165 \u00a7 \u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\u7684\u76ee\u7684\u662f\uff0c\u5c06\u5199\u5165\u8def\u5f84\u7684\u4e00\u6bb5\u7279\u5b9a\u5b57\u7b26\u4e32\u5339\u914d\u51fa\u6765\uff0c\u7528\u4e00\u6bb5\u968f\u673a\u7684\u5b57\u7b26\u4e32\u8fdb\u884c\u66ff\u6362\uff0c\u4ee5\u524a\u5f31\u6240\u8c13 \u201c\u7247\u533a\u201d (sharding) \u6548\u5e94\uff0c\u63d0\u9ad8\u5199\u5165\u6548\u7387\u3002\u914d\u7f6e\u53c2\u6570\uff1a oss.entropy.key=<user-defined-key> oss.entropy.length=<user-defined-length> \u5982\u6b64\uff0c\u6bcf\u6b21\u6253\u5f00\u5199\u4e00\u4e2a\u65b0\u6587\u4ef6\u65f6\uff0c\u8def\u5f84\u4e2d\u4e0e <user-defined-key> \u76f8\u540c\u7684\u5b57\u7b26\u4e32\u4f1a\u88ab\u66ff\u6362\u4e3a\u4e00\u4e2a\u968f\u673a\u5b57\u7b26\u4e32\uff0c\u968f\u673a\u4e32\u7684\u957f\u5ea6\u4e3a <user-defined-length> \u3002\u914d\u7f6e\u8be5\u53c2\u6570\u65f6\uff0c <user-defined-length> \u5fc5\u987b\u5927\u4e8e\u96f6\u3002 \u5206\u7247\u4e0a\u4f20\u5e76\u884c\u5ea6 \u00a7 \u5199\u5165 OSS \u4f1a\u81ea\u52a8\u8c03\u7528\u9ad8\u6548\u7684 \u201c\u5206\u7247\u4e0a\u4f20\u201d (Multipart Upload) \u673a\u5236\uff0c\u5c06\u5f85\u4e0a\u4f20\u7684\u6587\u4ef6\u5206\u4e3a\u591a\u4e2a\u6570\u636e\u5757 (part) \u5206\u522b\u4e0a\u4f20\uff0c\u6700\u540e\u7ec4\u5408\u3002\u76ee\u524d\u652f\u6301\u914d\u7f6e\u53c2\u6570 oss.upload.max.concurrent.uploads \uff0c\u7528\u6765\u63a7\u5236\u4e0a\u4f20\u6570\u636e\u5757 (part) \u7684\u5e76\u884c\u5ea6\uff0c\u5982\u679c\u8bbe\u7f6e\u8f83\u9ad8\u7684\u6570\u503c\u5219\u53ef\u80fd\u4f1a\u63d0\u9ad8\u5199\u5165\u6548\u7387\uff08\u4f46\u4e5f\u4f1a\u5360\u7528\u66f4\u591a\u8d44\u6e90\uff09\u3002\u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u8be5\u503c\u4e3a\u5f53\u524d\u53ef\u7528\u7684\u5904\u7406\u5668\u6570\u91cf\u3002","title":"Flink"},{"location":"jindosdk/flink/jindosdk_on_flink/#flink-jindosdk-oss","text":"","title":"Flink \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e"},{"location":"jindosdk/flink/jindosdk_on_flink/#_1","text":"\u5728\u96c6\u7fa4\u4e0a\u6709\u5f00\u6e90\u7248\u672c Flink \u8f6f\u4ef6\uff0c\u7248\u672c\u4e0d\u4f4e\u4e8e 1.10.1\u3002 \u6ce8\uff1aFlink 1.16.0 \u53ca\u66f4\u9ad8\u7248\u672c\u7684\u53ef\u7528\u6027\u5c1a\u672a\u5f97\u5230\u9a8c\u8bc1\u3002","title":"\u73af\u5883\u8981\u6c42"},{"location":"jindosdk/flink/jindosdk_on_flink/#flink-jindosdk-oss_1","text":"\u82e5\u9700\u8981 Flink \u4fdd\u6301 EXACTLY_ONCE \u8bed\u4e49\u6d41\u5f0f\u5199\u5165 Aliyun OSS\uff0c\u5219\u9700\u8981\u4f7f\u7528 JindoSDK\u3002 Apache Flink \u662f\u4e00\u79cd\u5f53\u524d\u4e1a\u754c\u6d41\u884c\u7684\u5f00\u6e90\u5927\u6570\u636e\u6d41\u5f0f\u8ba1\u7b97\u5f15\u64ce\uff0c\u652f\u6301 \u201c\u4e25\u683c\u4e00\u6b21\u201d\uff08EXACTLY_ONCE\uff09\u8bed\u4e49\u5199\u5165\u90e8\u5206\u5b58\u50a8\u4ecb\u8d28\u3002Aliyun Object Storage Service (Aliyun OSS) \u5219\u662f\u963f\u91cc\u4e91\u65d7\u4e0b\u7684\u6838\u5fc3\u4ea7\u54c1\u4e4b\u4e00\uff0c\u63d0\u4f9b\u4e91\u539f\u751f\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1\uff0c\u51ed\u501f\u4f18\u5f02\u7684\u6027\u80fd\u548c\u5b89\u5168\u6027\u5df2\u670d\u52a1\u4e8e\u6d77\u91cf\u7528\u6237\u3002 \u76ee\u524d\uff0c\u5f00\u6e90\u7248\u672c Flink \u5bf9\u6d41\u5f0f\u5199\u5165 Aliyun OSS \u5c1a\u4e0d\u80fd\u652f\u6301 EXACTLY_ONCE \u8bed\u4e49\uff0c\u5982\u6709\u8be5\u9700\u6c42\u5219\u9700\u8981\u4f7f\u7528 JindoSDK\u3002 \u6ce8\uff1aApache Flink \u81ea 1.16 \u7248\u672c\u8d77\u539f\u751f\u63d0\u4f9b\u4e86\u6d41\u5f0f\u5199\u5165 Aliyun OSS \u5bf9\u8c61\u5b58\u50a8\u7684\u652f\u6301\u3002JindoSDK \u63d0\u4f9b\u7684\u5b9e\u73b0\u4e0e Apache Flink \u7684\u81ea\u6709\u5b9e\u73b0\u6709\u4e24\u4e2a\u4e3b\u8981\u533a\u522b\uff1a - JindoSDK \u7684\u5b9e\u73b0\u80fd\u591f\u540c\u65f6\u652f\u6301 Aliyun OSS \u5bf9\u8c61\u5b58\u50a8\u548c Aliyun OSS-HDFS \u6587\u4ef6\u5b58\u50a8\u3002\u5bf9 OSS-HDFS \u7684\u4ecb\u7ecd\u53c2\u8003 OSS-HDFS\u670d\u52a1\u6982\u8ff0 - JindoSDK \u7684\u5b9e\u73b0\u5e95\u5c42\u91c7\u7528\u4e86 Native \u65b9\u6848\u3002","title":"\u4e3a\u4ec0\u4e48 Flink \u9700\u8981\u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS"},{"location":"jindosdk/flink/jindosdk_on_flink/#jindosdk","text":"\u9700\u8981\u5728\u6240\u6709 Flink \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flink \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e .jar \u6587\u4ef6\uff1a * jindo-flink-${version}-full.jar \u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 plugins/flink/ \u76ee\u5f55\u4e0b\u627e\u5230\u3002 \u53e6\u5916\uff0c\u5982\u679c\u96c6\u7fa4\u5185\u5b58\u5728 Apache Flink \u63d0\u4f9b\u7684\u81ea\u5e26 Flink OSS Connector\uff0c\u9700\u79fb\u9664\u3002\u79fb\u9664\u65b9\u6cd5\u4e3a\uff0c\u4ece Flink \u96c6\u7fa4\u7684 lib \u76ee\u5f55\u6216 plugins/oss-fs-hadoop \u79fb\u8d70\u5982\u4e0b jar \u5305\uff1a * flink-oss-fs-hadoop-${flink-version}.jar \u6ce8\uff1aJindoSDK \u76ee\u524d\u63d0\u4f9b\u591a\u5e73\u53f0\u652f\u6301\uff0c\u5f53\u524d\u652f\u6301\u7684\u5e73\u53f0\u4ee5\u53ca\u9700\u8981\u989d\u5916\u653e\u7f6e\u7684\u8d44\u6e90\uff0c\u53c2\u8003 \u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK","title":"JindoSDK \u914d\u7f6e"},{"location":"jindosdk/flink/jindosdk_on_flink/#_2","text":"\u914d\u7f6e\u597d SDK \u540e\uff0c\u65e0\u9700\u989d\u5916\u914d\u7f6e\uff0c\u4ee5\u5e38\u89c4 Flink \u6d41\u5f0f\u4f5c\u4e1a\u65b9\u6cd5\u4f7f\u7528\u5373\u53ef\uff0c\u6ce8\u610f\u4f7f\u7528\u6b63\u786e\u7684\u8def\u5f84\u3002\u5199\u5165 OSS (Aliyun Object Storage Service) \u987b\u4ee5 oss:// \u4e3a\u524d\u7f00\u3002","title":"\u5982\u4f55\u4f7f\u7528"},{"location":"jindosdk/flink/jindosdk_on_flink/#_3","text":"\u4e0b\u9762\u662f\u4e00\u4e2a\u7b80\u5355\u7684\u793a\u4f8b\u7a0b\u5e8f\uff0c\u5982\u679c\u5df2\u7ecf\u6309\u7167\u4e0a\u4e00\u8282\u914d\u7f6e\u597d SDK\uff0c\u5219\u4e0b\u5217\u7a0b\u5e8f\u5e94\u80fd\u6b63\u786e\u8fd0\u884c\uff1a (1) \u901a\u7528\u914d\u7f6e \u6253\u5f00 Flink \u7684 \u201c\u68c0\u67e5\u70b9\u201d\uff08Checkpoint\uff09\u3002\u4f8b\u5982\uff0cenv \u662f\u4e00\u4e2a\u5df2\u7ecf\u5efa\u7acb\u7684 StreamExecutionEnvironment\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u65b9\u5f0f\u5efa\u7acb\uff1a StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); \u5219\u53ef\u4ee5\u8fd9\u6837\u542f\u7528 checkpoint\uff1a env.enableCheckpointing(<userDefinedCheckpointInterval>, CheckpointingMode.EXACTLY_ONCE); \u4f7f\u7528\u53ef\u91cd\u53d1\u7684\u6570\u636e\u6e90\uff0c\u4f8b\u5982 Kafka (2) \u793a\u4f8b\u7a0b\u5e8f \u4e0b\u6587\u4e2d\uff0coutputStream \u662f\u4e00\u4e2a\u9884\u5148\u5f62\u6210\u7684 DataStream<String> \u5bf9\u8c61\uff0c\u5e76\u671f\u671b\u5c06\u5176\u5199\u5165 OSS\uff0c\u90a3\u4e48\u53ef\u4ee5\u8fd9\u6837\u6dfb\u52a0 sink\uff1a String outputPath = \"oss://<user-defined-bucket>/<user-defined-dir>\" StreamingFileSink<String> sink = StreamingFileSink.forRowFormat( new Path(outputPath), new SimpleStringEncoder<String>(\"UTF-8\") ).build(); outputStream.addSink(sink); \u6700\u540e\u7528 env.execute() \u6267\u884c Flink \u4f5c\u4e1a\u5373\u53ef\u3002","title":"\u793a\u4f8b"},{"location":"jindosdk/flink/jindosdk_on_flink/#_4","text":"\u7528\u6237\u5728\u63d0\u4ea4 Flink \u4f5c\u4e1a\u65f6\uff0c\u53ef\u4ee5\u81ea\u5b9a\u4e49\u914d\u7f6e\u4e00\u4e9b\u53c2\u6570\uff0c\u4ee5\u5f00\u542f\u6216\u63a7\u5236\u7279\u5b9a\u529f\u80fd\u3002\u4f8b\u5982\uff0c\u4ee5 yarn-cluster \u6a21\u5f0f\u63d0\u4ea4\u65f6\uff0c\u53ef\u4ee5\u901a\u8fc7 -yD \u8fdb\u884c\u914d\u7f6e\uff0c\u65b9\u5f0f\u7c7b\u4f3c\u4e8e\uff1a <flink_home>/bin/flink run -m yarn-cluster -yD key1=value1 -yD key2=value2 ... \u76ee\u524d\u652f\u6301\u901a\u8fc7\u914d\u7f6e\u5f00\u542f \u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\uff0c\u6216\u8005\u63a7\u5236 \u201c\u5206\u7247\u4e0a\u4f20\u201d \u7684\u5e76\u884c\u5ea6\u3002","title":"\u9ad8\u7ea7\u4f7f\u7528\uff1a\u81ea\u5b9a\u4e49\u53c2\u6570"},{"location":"jindosdk/flink/jindosdk_on_flink/#_5","text":"\u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\u7684\u76ee\u7684\u662f\uff0c\u5c06\u5199\u5165\u8def\u5f84\u7684\u4e00\u6bb5\u7279\u5b9a\u5b57\u7b26\u4e32\u5339\u914d\u51fa\u6765\uff0c\u7528\u4e00\u6bb5\u968f\u673a\u7684\u5b57\u7b26\u4e32\u8fdb\u884c\u66ff\u6362\uff0c\u4ee5\u524a\u5f31\u6240\u8c13 \u201c\u7247\u533a\u201d (sharding) \u6548\u5e94\uff0c\u63d0\u9ad8\u5199\u5165\u6548\u7387\u3002\u914d\u7f6e\u53c2\u6570\uff1a oss.entropy.key=<user-defined-key> oss.entropy.length=<user-defined-length> \u5982\u6b64\uff0c\u6bcf\u6b21\u6253\u5f00\u5199\u4e00\u4e2a\u65b0\u6587\u4ef6\u65f6\uff0c\u8def\u5f84\u4e2d\u4e0e <user-defined-key> \u76f8\u540c\u7684\u5b57\u7b26\u4e32\u4f1a\u88ab\u66ff\u6362\u4e3a\u4e00\u4e2a\u968f\u673a\u5b57\u7b26\u4e32\uff0c\u968f\u673a\u4e32\u7684\u957f\u5ea6\u4e3a <user-defined-length> \u3002\u914d\u7f6e\u8be5\u53c2\u6570\u65f6\uff0c <user-defined-length> \u5fc5\u987b\u5927\u4e8e\u96f6\u3002","title":"\u71b5\u6ce8\u5165"},{"location":"jindosdk/flink/jindosdk_on_flink/#_6","text":"\u5199\u5165 OSS \u4f1a\u81ea\u52a8\u8c03\u7528\u9ad8\u6548\u7684 \u201c\u5206\u7247\u4e0a\u4f20\u201d (Multipart Upload) \u673a\u5236\uff0c\u5c06\u5f85\u4e0a\u4f20\u7684\u6587\u4ef6\u5206\u4e3a\u591a\u4e2a\u6570\u636e\u5757 (part) \u5206\u522b\u4e0a\u4f20\uff0c\u6700\u540e\u7ec4\u5408\u3002\u76ee\u524d\u652f\u6301\u914d\u7f6e\u53c2\u6570 oss.upload.max.concurrent.uploads \uff0c\u7528\u6765\u63a7\u5236\u4e0a\u4f20\u6570\u636e\u5757 (part) \u7684\u5e76\u884c\u5ea6\uff0c\u5982\u679c\u8bbe\u7f6e\u8f83\u9ad8\u7684\u6570\u503c\u5219\u53ef\u80fd\u4f1a\u63d0\u9ad8\u5199\u5165\u6548\u7387\uff08\u4f46\u4e5f\u4f1a\u5360\u7528\u66f4\u591a\u8d44\u6e90\uff09\u3002\u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u8be5\u503c\u4e3a\u5f53\u524d\u53ef\u7528\u7684\u5904\u7406\u5668\u6570\u91cf\u3002","title":"\u5206\u7247\u4e0a\u4f20\u5e76\u884c\u5ea6"},{"location":"jindosdk/flume/jindosdk_on_flume/","text":"Flume \u4f7f\u7528 JindoSDK \u5199\u5165 OSS \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 \u5728\u96c6\u7fa4\u4e0a\u5df2\u7ecf\u90e8\u7f72 Flume\uff0c\u5df2\u90e8\u7f72 JindoSDK 6.0 \u4ee5\u4e0a\u7248\u672c\u3002 JindoSDK \u5b89\u88c5\u90e8\u7f72 \u00a7 \u9700\u8981\u5728 Flume \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flume \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e JindoSDK\u3002 \u4e0b\u8f7d\u5e76\u89e3\u538b\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.y.z.tar.gz\uff08x.y.z\u8868\u7248\u672c\u53f7\uff09\uff0c ( \u4e0b\u8f7d\u9875\u9762 ) cp jindosdk-x.y.z/lib/*.jar $FLUME_HOME/jars/ \u914d\u7f6e \u00a7 Flume Sink \u914d\u7f6e \u00a7 # \u914d\u7f6eOSS Sink\u3002your_bucket\u586b\u5199\u4e3a\u5df2\u5f00\u542fOSS-HDFS\u670d\u52a1\u7684Bucket\u3002 xxx.sinks.oss_sink.hdfs.path = oss://${your_bucket}/flume_dir/%Y-%m-%d/%H # \u5199\u5165Flume\u4e8b\u52a1\u4e2d\u6700\u5927\u7684event\u6570\u91cf\u3002\u63a8\u8350\u6bcf\u6b21Flush\u7684\u91cf\u572832 MB\u4ee5\u4e0a\uff0c\u907f\u514dFlush\u8fc7\u5c0f\u5f71\u54cd\u6574\u4f53\u6027\u80fd\u4ee5\u53ca\u4ea7\u751f\u5927\u91cf\u7684staging\u6587\u4ef6\u3002 # batchSize\u5355\u4f4d\u4e3aevent\u6570\u91cf\uff08\u5373\u65e5\u5fd7\u884c\u6570\uff09\uff0c\u8bbe\u7f6e\u65f6\u9700\u8981\u5148\u8bc4\u4f30event\u5e73\u5747\u5927\u5c0f\uff08\u4f8b\u5982200 Byte\uff09\uff0c\u5047\u8bbe\u6bcf\u6b21Flush\u7684\u5927\u5c0f\u9884\u671f\u4e3a32 MB\uff0c\u5219batchSize\u7ea6\u4e3a160000\uff0832 MB / 200 Byte\uff09\u3002 xxx.sinks.oss_sink.hdfs.batchSize = 100000 ... xxx.sinks.oss_sink.hdfs.round = true xxx.sinks.oss_sink.hdfs.roundValue = 15 xxx.sinks.oss_sink.hdfs.Unit = minute xxx.sinks.oss_sink.hdfs.filePrefix = your_topic xxx.sinks.oss_sink.rollSize = 3600 xxx.sinks.oss_sink.threadsPoolSize = 30 ... OSS-HDFS \u573a\u666f core-sites.xml \u914d\u7f6e \u00a7 \u63a8\u8350\u6bcf\u6b21 Flush \u7684\u91cf\u5728 32 MB\u4ee5\u4e0a\uff0c\u907f\u514d Flush \u8fc7\u4e8e\u9891\u7e41\u5f71\u54cd\u6574\u4f53\u6027\u80fd\u4ee5\u53ca\u4ea7\u751f\u5c0f\u5757\u3002 <configuration> <property> <!-- false\u8868\u793a\u4e0d\u6267\u884chflush\uff0c\u4ec5\u4f9d\u9760close\u6765\u786e\u4fdd\u6267\u884c\u6587\u4ef6\u5199\u5165\uff0c\u4e0d\u63a8\u8350\u5173\u95ed --> <name>fs.oss.hflush.enable</name> <value>true</value> </property> <property> <!-- \u5f53\u524d\u6587\u4ef6\u6267\u884chflush\u7684\u6b21\u6570\u8fbe\u5230\u8be5\u6570\u91cf\u540e\uff0c\u624d\u89e6\u53d1\u4e00\u6b21\u771f\u6b63\u7684flush\uff080\u8868\u793a\u4e0d\u6839\u636e\u8be5\u6570\u91cf\u6765\u89e6\u53d1Flush\uff09 --> <name>fs.oss.hflush.interval.count</name> <value>0</value> </property> <property> <!-- \u5f53\u524d\u6587\u4ef6\u5199\u5165\u8fbe\u5230\u8be5\u5927\u5c0f\u540e\uff0c\u624d\u89e6\u53d1\u4e00\u6b21\u771f\u6b63\u7684flush\uff080\u8868\u793a\u4e0d\u6839\u636e\u5199\u5165\u5927\u5c0f\u6765\u89e6\u53d1Flush\uff09\uff0c\u5355\u4f4d\uff1a\u5b57\u8282--> <name>fs.oss.hflush.interval.size</name> <value>0</value> </property> </configuration> ps: \u5982 fs.oss.hflush.interval.count \u548c fs.oss.hflush.interval.size \u5747\u4e0d\u4e3a0\uff0c\u5219\u4e24\u4e2a\u6761\u4ef6\u5747\u6ee1\u8db3\u65f6\u624d\u6267\u884chflush\u3002","title":"Flume"},{"location":"jindosdk/flume/jindosdk_on_flume/#flume-jindosdk-oss","text":"","title":"Flume \u4f7f\u7528 JindoSDK \u5199\u5165 OSS"},{"location":"jindosdk/flume/jindosdk_on_flume/#_1","text":"\u5728\u96c6\u7fa4\u4e0a\u5df2\u7ecf\u90e8\u7f72 Flume\uff0c\u5df2\u90e8\u7f72 JindoSDK 6.0 \u4ee5\u4e0a\u7248\u672c\u3002","title":"\u73af\u5883\u8981\u6c42"},{"location":"jindosdk/flume/jindosdk_on_flume/#jindosdk","text":"\u9700\u8981\u5728 Flume \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flume \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e JindoSDK\u3002 \u4e0b\u8f7d\u5e76\u89e3\u538b\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.y.z.tar.gz\uff08x.y.z\u8868\u7248\u672c\u53f7\uff09\uff0c ( \u4e0b\u8f7d\u9875\u9762 ) cp jindosdk-x.y.z/lib/*.jar $FLUME_HOME/jars/","title":"JindoSDK \u5b89\u88c5\u90e8\u7f72"},{"location":"jindosdk/flume/jindosdk_on_flume/#_2","text":"","title":"\u914d\u7f6e"},{"location":"jindosdk/flume/jindosdk_on_flume/#flume-sink","text":"# \u914d\u7f6eOSS Sink\u3002your_bucket\u586b\u5199\u4e3a\u5df2\u5f00\u542fOSS-HDFS\u670d\u52a1\u7684Bucket\u3002 xxx.sinks.oss_sink.hdfs.path = oss://${your_bucket}/flume_dir/%Y-%m-%d/%H # \u5199\u5165Flume\u4e8b\u52a1\u4e2d\u6700\u5927\u7684event\u6570\u91cf\u3002\u63a8\u8350\u6bcf\u6b21Flush\u7684\u91cf\u572832 MB\u4ee5\u4e0a\uff0c\u907f\u514dFlush\u8fc7\u5c0f\u5f71\u54cd\u6574\u4f53\u6027\u80fd\u4ee5\u53ca\u4ea7\u751f\u5927\u91cf\u7684staging\u6587\u4ef6\u3002 # batchSize\u5355\u4f4d\u4e3aevent\u6570\u91cf\uff08\u5373\u65e5\u5fd7\u884c\u6570\uff09\uff0c\u8bbe\u7f6e\u65f6\u9700\u8981\u5148\u8bc4\u4f30event\u5e73\u5747\u5927\u5c0f\uff08\u4f8b\u5982200 Byte\uff09\uff0c\u5047\u8bbe\u6bcf\u6b21Flush\u7684\u5927\u5c0f\u9884\u671f\u4e3a32 MB\uff0c\u5219batchSize\u7ea6\u4e3a160000\uff0832 MB / 200 Byte\uff09\u3002 xxx.sinks.oss_sink.hdfs.batchSize = 100000 ... xxx.sinks.oss_sink.hdfs.round = true xxx.sinks.oss_sink.hdfs.roundValue = 15 xxx.sinks.oss_sink.hdfs.Unit = minute xxx.sinks.oss_sink.hdfs.filePrefix = your_topic xxx.sinks.oss_sink.rollSize = 3600 xxx.sinks.oss_sink.threadsPoolSize = 30 ...","title":"Flume Sink \u914d\u7f6e"},{"location":"jindosdk/flume/jindosdk_on_flume/#oss-hdfs-core-sitesxml","text":"\u63a8\u8350\u6bcf\u6b21 Flush \u7684\u91cf\u5728 32 MB\u4ee5\u4e0a\uff0c\u907f\u514d Flush \u8fc7\u4e8e\u9891\u7e41\u5f71\u54cd\u6574\u4f53\u6027\u80fd\u4ee5\u53ca\u4ea7\u751f\u5c0f\u5757\u3002 <configuration> <property> <!-- false\u8868\u793a\u4e0d\u6267\u884chflush\uff0c\u4ec5\u4f9d\u9760close\u6765\u786e\u4fdd\u6267\u884c\u6587\u4ef6\u5199\u5165\uff0c\u4e0d\u63a8\u8350\u5173\u95ed --> <name>fs.oss.hflush.enable</name> <value>true</value> </property> <property> <!-- \u5f53\u524d\u6587\u4ef6\u6267\u884chflush\u7684\u6b21\u6570\u8fbe\u5230\u8be5\u6570\u91cf\u540e\uff0c\u624d\u89e6\u53d1\u4e00\u6b21\u771f\u6b63\u7684flush\uff080\u8868\u793a\u4e0d\u6839\u636e\u8be5\u6570\u91cf\u6765\u89e6\u53d1Flush\uff09 --> <name>fs.oss.hflush.interval.count</name> <value>0</value> </property> <property> <!-- \u5f53\u524d\u6587\u4ef6\u5199\u5165\u8fbe\u5230\u8be5\u5927\u5c0f\u540e\uff0c\u624d\u89e6\u53d1\u4e00\u6b21\u771f\u6b63\u7684flush\uff080\u8868\u793a\u4e0d\u6839\u636e\u5199\u5165\u5927\u5c0f\u6765\u89e6\u53d1Flush\uff09\uff0c\u5355\u4f4d\uff1a\u5b57\u8282--> <name>fs.oss.hflush.interval.size</name> <value>0</value> </property> </configuration> ps: \u5982 fs.oss.hflush.interval.count \u548c fs.oss.hflush.interval.size \u5747\u4e0d\u4e3a0\uff0c\u5219\u4e24\u4e2a\u6761\u4ef6\u5747\u6ee1\u8db3\u65f6\u624d\u6267\u884chflush\u3002","title":"OSS-HDFS \u573a\u666f core-sites.xml \u914d\u7f6e"},{"location":"jindosdk/hadoop/jindosdk_ide_hadoop/","text":"Hadoop \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 \u00a7 \u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.net.URI; public class TestJindoSDK { public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. FileSystem fs = FileSystem.get(URI.create(\"oss://<bucket>/\"), conf); FSDataInputStream in = fs.open(new Path(\"/uttest/file1\")); in.read(); in.close(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <!-- ... --> <dependencies> <!-- add jindo-core-extended-jar for macos --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> </dependencies> </project>","title":"\u5f00\u53d1\u548c\u4f7f\u7528"},{"location":"jindosdk/hadoop/jindosdk_ide_hadoop/#hadoop-jindosdk-ide","text":"\u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.net.URI; public class TestJindoSDK { public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. FileSystem fs = FileSystem.get(URI.create(\"oss://<bucket>/\"), conf); FSDataInputStream in = fs.open(new Path(\"/uttest/file1\")); in.read(); in.close(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <!-- ... --> <dependencies> <!-- add jindo-core-extended-jar for macos --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> </dependencies> </project>","title":"Hadoop \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5"},{"location":"jindosdk/hbase/jindosdk_on_hbase/","text":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 \u00a7 HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002 OSS-HDFS \u670d\u52a1\u662f\u963f\u91cc\u4e91\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9 HDFS \u63a5\u53e3, JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\u53ca WAL \u6587\u4ef6\u7684\u5b58\u50a8\u5b9e\u73b0\u5b58\u7b97\u5206\u79bb\uff0c\u76f8\u5bf9\u4e8e\u672c\u5730HDFS\u5b58\u50a8\uff0c\u4f7f\u7528\u66f4\u52a0\u7075\u6d3b\uff0c\u51cf\u5c11\u8fd0\u7ef4\u6210\u672c\u3002 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-6.7.3 \u4e3a\u4f8b: cp jindosdk-6.7.3/lib/jindo-core-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-6.7.3/lib/jindo-sdk-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06\u5df2\u5f00\u542f OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u00a7 \u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 5. \u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84 \u00a7 \u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u548c hbase.wal.dir \u7684\u503c\u4e3a OSS-HDFS \u670d\u52a1\u5730\u5740\uff0c\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84\u548c WAL \u6587\u4ef6\u7684\u5b58\u50a8\u8def\u5f84\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 OSS-HDFS \u670d\u52a1\u3002\u53c2\u6570\u503c\u4e3a oss://bucket.endpoint/hbase-root-dir hbase.wal.dir \u6307\u5b9a HBase \u7684 WAL \u5b58\u50a8\u76ee\u5f55\u5230 OSS-HDFS \u670d\u52a1\u3002 \u53c2\u6570\u503c\u4e3a oss://bucket.endpoint/hbase","title":"Hbase"},{"location":"jindosdk/hbase/jindosdk_on_hbase/#hbase-oss-hdfs-jindofs","text":"HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002 OSS-HDFS \u670d\u52a1\u662f\u963f\u91cc\u4e91\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9 HDFS \u63a5\u53e3, JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\u53ca WAL \u6587\u4ef6\u7684\u5b58\u50a8\u5b9e\u73b0\u5b58\u7b97\u5206\u79bb\uff0c\u76f8\u5bf9\u4e8e\u672c\u5730HDFS\u5b58\u50a8\uff0c\u4f7f\u7528\u66f4\u52a0\u7075\u6d3b\uff0c\u51cf\u5c11\u8fd0\u7ef4\u6210\u672c\u3002","title":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8"},{"location":"jindosdk/hbase/jindosdk_on_hbase/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"jindosdk/hbase/jindosdk_on_hbase/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-6.7.3 \u4e3a\u4f8b: cp jindosdk-6.7.3/lib/jindo-core-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-6.7.3/lib/jindo-sdk-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"jindosdk/hbase/jindosdk_on_hbase/#3-oss-hdfs-access-key","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06\u5df2\u5f00\u542f OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002","title":"3. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"jindosdk/hbase/jindosdk_on_hbase/#4-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002","title":"4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint"},{"location":"jindosdk/hbase/jindosdk_on_hbase/#5-hbase","text":"\u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u548c hbase.wal.dir \u7684\u503c\u4e3a OSS-HDFS \u670d\u52a1\u5730\u5740\uff0c\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84\u548c WAL \u6587\u4ef6\u7684\u5b58\u50a8\u8def\u5f84\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 OSS-HDFS \u670d\u52a1\u3002\u53c2\u6570\u503c\u4e3a oss://bucket.endpoint/hbase-root-dir hbase.wal.dir \u6307\u5b9a HBase \u7684 WAL \u5b58\u50a8\u76ee\u5f55\u5230 OSS-HDFS \u670d\u52a1\u3002 \u53c2\u6570\u503c\u4e3a oss://bucket.endpoint/hbase","title":"5. \u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84"},{"location":"jindosdk/hive/jindosdk_on_hive/","text":"Hive \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e \u00a7 Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002 \u5b89\u88c5\u914d\u7f6e \u00a7 1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/ 2. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u914d\u7f6e JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b \u5c06 JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key \u5c06\u5df2\u5f00\u542f OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 3. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u00a7 \u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u6700\u540e\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 4. Hive on MR \u00a7 \u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002 5. Hive on Tez \u00a7 \u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002 6. Hive on Spark \u00a7 \u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002 OSS-HDFS \u670d\u52a1\u7528\u4e8e\u8868\u7684\u5b58\u50a8 \u00a7 \u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS-HDFS \u670d\u52a1\u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'oss://bucket_name.endpoint_name/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'oss://bucket_name.endpoint_name/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS-HDFS \u670d\u52a1\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>oss://bucket_name.endpoint_name/path/to/warehouse</value> </property> </configuration> \u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a \u00a7 ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'oss://bucket_name.endpoint_name/path/to/us/part210301cn'; \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Hive"},{"location":"jindosdk/hive/jindosdk_on_hive/#hive-jindosdk-oss-hdfs-jindofs","text":"Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002","title":"Hive \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"jindosdk/hive/jindosdk_on_hive/#_1","text":"","title":"\u5b89\u88c5\u914d\u7f6e"},{"location":"jindosdk/hive/jindosdk_on_hive/#1-hive-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/","title":"1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002"},{"location":"jindosdk/hive/jindosdk_on_hive/#2-oss-hdfs-access-key","text":"\u914d\u7f6e JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b \u5c06 JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key \u5c06\u5df2\u5f00\u542f OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002","title":"2. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"jindosdk/hive/jindosdk_on_hive/#3-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u6700\u540e\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002","title":"3. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint"},{"location":"jindosdk/hive/jindosdk_on_hive/#4-hive-on-mr","text":"\u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002","title":"4. Hive on MR"},{"location":"jindosdk/hive/jindosdk_on_hive/#5-hive-on-tez","text":"\u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002","title":"5. Hive on Tez"},{"location":"jindosdk/hive/jindosdk_on_hive/#6-hive-on-spark","text":"\u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002","title":"6. Hive on Spark"},{"location":"jindosdk/hive/jindosdk_on_hive/#oss-hdfs","text":"\u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS-HDFS \u670d\u52a1\u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'oss://bucket_name.endpoint_name/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'oss://bucket_name.endpoint_name/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS-HDFS \u670d\u52a1\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>oss://bucket_name.endpoint_name/path/to/warehouse</value> </property> </configuration>","title":"OSS-HDFS \u670d\u52a1\u7528\u4e8e\u8868\u7684\u5b58\u50a8"},{"location":"jindosdk/hive/jindosdk_on_hive/#oss","text":"ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'oss://bucket_name.endpoint_name/path/to/us/part210301cn';","title":"\u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a"},{"location":"jindosdk/hive/jindosdk_on_hive/#_2","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"jindosdk/presto/jindosdk_on_presto/","text":"Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u6570\u636e \u00a7 Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002 \u6b65\u9aa4 \u00a7 1. \u5b89\u88c5 jar \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/ 2. \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> 3. \u914d\u7f6e OSS Access Key \u00a7 \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002 4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u00a7 \u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 5. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 \u4f7f\u7528\u793a\u4f8b \u00a7 \u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236\u53f0 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='oss://<Bucket>.<Endpoint>/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"Presto"},{"location":"jindosdk/presto/jindosdk_on_presto/#presto-jindosdk-oss-hdfs","text":"Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002","title":"Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u6570\u636e"},{"location":"jindosdk/presto/jindosdk_on_presto/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"jindosdk/presto/jindosdk_on_presto/#1-jar","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/","title":"1. \u5b89\u88c5 jar \u5305"},{"location":"jindosdk/presto/jindosdk_on_presto/#2-jindosdk-oss","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration>","title":"2. \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b"},{"location":"jindosdk/presto/jindosdk_on_presto/#3-oss-access-key","text":"\u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002","title":"3. \u914d\u7f6e OSS Access Key"},{"location":"jindosdk/presto/jindosdk_on_presto/#4-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002","title":"4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint"},{"location":"jindosdk/presto/jindosdk_on_presto/#5-presto","text":"","title":"5. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"jindosdk/presto/jindosdk_on_presto/#_2","text":"\u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236\u53f0 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='oss://<Bucket>.<Endpoint>/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"\u4f7f\u7528\u793a\u4f8b"},{"location":"jindosdk/python/pyjindo_quickstart/","text":"\u4f7f\u7528 PyJindo \u8bbf\u95ee\u963f\u91cc\u4e91 OSS/OSS-HDFS \u00a7 \u80cc\u666f \u00a7 \u672c\u6587\u6307\u5bfc\u5982\u4f55\u4f7f\u7528 Python \u7684\u5de5\u5177\u5305 PyJindo \u64cd\u4f5c OSS/OSS-HDFS\u3002PyJindo \u517c\u5bb9 python3.6 \u4ee5\u4e0a\u7248\u672c\u3002 \u90e8\u7f72\u73af\u5883 \u00a7 \u4e0b\u8f7d\u5bf9\u5e94 Python \u7248\u672c\u7684 PyJindo \u5b89\u88c5\u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.y.z.tar.gz\uff08x.y.z\u8868\u7248\u672c\u53f7\uff09\uff0c\u4e0b\u8f7d\u94fe\u63a5\u89c1 \u4e0b\u8f7d\u9875\u9762 \u3002 \u90e8\u7f72 jindosdk-6.7.0.tar.gz\uff0cwhl\u5b89\u88c5\u5305\u4f4d\u4e8e\u5b8c\u6574\u4ea7\u51fa\u7269 jindosdk-x.x.x/lib/site-packages/ \u7684\u5b50\u76ee\u5f55\u4e2d\u3002\u591a\u5e73\u53f0\u90e8\u7f72\u8bf4\u660e\u53c2\u89c1 \u90e8\u7f72\u6587\u6863 \u3002 \u4ee5 Python3.6 \u7248\u672c\u4e3a\u4f8b\uff0c\u8bf7\u5b89\u88c5pyjindo-x.y.z-cp 36 -abi3-linux_x86_64.whl . \u251c\u2500\u2500 bin \u2502 \u251c\u2500\u2500 xxx \u251c\u2500\u2500 conf \u2502 \u251c\u2500\u2500 xxx \u251c\u2500\u2500 include \u2502 \u251c\u2500\u2500 xxx \u251c\u2500\u2500 lib \u2502 \u251c\u2500\u2500 xxx \u2502 \u251c\u2500\u2500 native \u2502 \u2502 \u251c\u2500\u2500 xxxx \u2502 \u2514\u2500\u2500 site-packages \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp310-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp311-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp312-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp36-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp37-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp38-abi3-linux_x86_64.whl \u2502 \u2514\u2500\u2500 pyjindo-x.y.z-cp39-abi3-linux_x86_64.whl \u251c\u2500\u2500 plugins \u2502 \u2514\u2500\u2500 xxxx \u251c\u2500\u2500 tools \u2502 \u251c\u2500\u2500 xxx \u2514\u2500\u2500 versions \u251c\u2500\u2500 xxx \u786e\u8ba4\u5b58\u5728\u4ee5\u4e0b\u73af\u5883\u53d8\u91cf\u7528\u4ee5\u8bfb\u53d6\u914d\u7f6e \u00a7 export JINDOSDK_CONF_DIR=/etc/taihao-apps/jindosdk-conf export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf \u5728\u963f\u91cc\u4e91EMR\u73af\u5883\u4e2d\u9ed8\u8ba4\u5b58\u5728\u4ee5\u4e0a\u914d\u7f6e\uff0c\u65e0\u9700\u914d\u7f6e\u3002 \u975eEMR\u4e2d\u914d\u7f6e\u65b9\u5f0f\u53c2\u89c1\uff1a \u300a\u5728\u975eEMR\u96c6\u7fa4\u4e2d\u90e8\u7f72JindoSDK\u300b \u3002\u5176\u4e2d\uff0cHadoop\u914d\u7f6e\u6587\u4ef6\u53caHADOOP_CONF_DIR\u4e0d\u662f\u5fc5\u987b\uff0c\u4ec5\u4e3a\u517c\u5bb9HADOOP\u73af\u5883\u4e2d\u7684\u914d\u7f6e\u3002 \u5b89\u88c5 PyJindo \u00a7 \u4ee5 Python3.8 \u73af\u5883\u5b89\u88c5\u6700\u65b0\u7248\u672c\u7684 pyjindo-6.7.3 \u4e3a\u4f8b python3.8 -m ensurepip python3.8 -m pip install pip --upgrade --trusted-host mirrors.aliyun.com -i http://mirrors.aliyun.com/pypi/simple/ python3.8 -m pip install pyjindo-6.7.3-cp38-abi3-linux_x86_64.whl \u7f16\u5199 python \u6d4b\u8bd5\u7a0b\u5e8f \u00a7 from pyjindo import fs bucket = \"jindosdk-yanbin-sh\" endpoint = bucket + \".cn-shanghai.oss-dls.aliyuncs.com\" root_path = \"oss://\" + endpoint + \"/\" sub_dir = root_path + \"pyjindotest/\" file_path = root_path + \"hello.txt\" file_path2 = sub_dir + \"hello.txt\" config = fs.read_config() fs = fs.connect(root_path, \"root\", config) # open\uff08path, mode\uff09\u6a21\u5f0f w,\u6587\u4ef6\u4e0d\u5b58\u5728\u521b\u5efa\u4e00\u4e2a\u6587\u4ef6 out_file = fs.open(file_path,\"wb\") out_file.write(str.encode(\"hello world, pyjindo\")) #\u5199 out_file.close() in_file = fs.open(file_path, \"rb\") data = in_file.read() # \u8bfb print(\"\u5199\u5165\u7684\u6570\u636e\u4e3a%s.\"%(data)) in_file.close() # \u5217\u51fa\u6587\u4ef6 ls_file = fs.listdir(root_path) print(\"\u76ee\u5f55\u6587\u4ef6\u4e3a%s.\" %(ls_file)) # \u521b\u5efa\u76ee\u5f55 fs.mkdir(sub_dir) # \u79fb\u52a8\u5e76\u91cd\u547d\u540d\u6587\u4ef6 fs.rename(file_path, file_path2) # \u5217\u51fa\u6587\u4ef6 mv_file = fs.listdir(sub_dir) print(\"\u79fb\u52a8\u540e\u7684\u76ee\u5f55\u6587\u4ef6\u4e3a%s.\" %(mv_file)) # \u5220\u9664\u6d4b\u8bd5\u6587\u4ef6\uff0c\u91cd\u65b0\u5217\u51fa\u6587\u4ef6 fs.remove(file_path2) de_file = fs.listdir(sub_dir) print(\"\u5220\u9664\u6587\u4ef6\u540e\u7684pyjindotest\u76ee\u5f55\u4e0b\u6587\u4ef6\u4e3a%s.\" %(de_file)) \u6267\u884c\u6d4b\u8bd5 \u00a7 python3.8 fs_test.py \u6267\u884c\u7ed3\u679c \u00a7 \u5199\u5165\u7684\u6570\u636e\u4e3ab'hello world, pyjindo'. \u76ee\u5f55\u6587\u4ef6\u4e3a[<FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/.sysinfo/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/apps/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/flume/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/hbase/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/hello.txt': type=File, size=20>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/pyarrowtest/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/spark-history/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/tmp/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/user/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/yarn/': type=Directory>]. \u79fb\u52a8\u540e\u7684\u76ee\u5f55\u6587\u4ef6\u4e3a[<FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/pyjindotest/hello.txt': type=File, size=20>]. \u5220\u9664\u6587\u4ef6\u540e\u7684pyjindotest\u76ee\u5f55\u4e0b\u6587\u4ef6\u4e3a[]. \u65e5\u5fd7\u7b49\u7ea7 \u00a7 \u8c03\u6574JINDOSDK_CONF_DIR\u4e0b\u7684jindosdk.cfg\u914d\u7f6e\uff0cemr\u4e0a\u5bf9\u5e94/etc/taihao-apps/jindosdk-conf [common] logger.dir = /var/log/emr/jindosdk logger.level = 2 logger.verbose = 0 logger.sync = false logger.jnilogger = true logger.consolelogger = false logger.cleaner.enable = true \u914d\u7f6e\u9879 \u63cf\u8ff0 logger.dir \u65e5\u5fd7\u8f93\u51fa\u76ee\u5f55\u3002 logger.level \u63a8\u8350\u4e3a2\uff0clevel<=1\uff0c\u8868\u793aWARN\uff1blevel>1\uff0c\u8868\u793aINFO\u3002 logger.verbose \u8be6\u7ec6\u65e5\u5fd7\u7b49\u7ea7\uff0c\u8303\u56f4\u4e3a0-99\uff0c\u503c\u8d8a\u5927\u65e5\u5fd7\u8d8a\u8be6\u7ec6\u3002 logger.sync \u63a8\u8350\u4e3afalse\u3002true\u8868\u793a\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\u3002 logger.jnilogger \u4f7f\u7528jni\u65f6\u751f\u6548\uff0c\u4e0epyjindo\u65e0\u5173\u3002 logger.consolelogger \u7ec8\u7aef\u8f93\u51fa\u65f6\u751f\u6548\uff0c\u4e0epyjindo\u65e0\u5173\u3002 logger.cleaner.enable \u63a8\u8350\u4e3atrue\uff0cfalse\u8868\u793a\u5173\u95ed\u65e5\u5fd7\u81ea\u52a8\u6e05\u7406\u3002 API \u8bf4\u660e \u00a7 Config \u7c7b \u00a7 \u6210\u5458\u51fd\u6570 \u8fd4\u56de\u503c\u7c7b\u578b \u63cf\u8ff0 set(key, val) \u65e0 \u8bbe\u7f6e\u5b57\u7b26\u4e32\u7c7b\u578b\u914d\u7f6e\uff0ckey\u4e0eval\u5747\u4e3astr get(key, default='') str \u83b7\u53d6\u5b57\u7b26\u4e32\u7c7b\u578b\u914d\u7f6e contains(key) bool \u662f\u5426\u5b58\u5728\u914d\u7f6ekey \u914d\u7f6e\u8bf4\u660e\u53c2\u89c1\uff1a \u5ba2\u6237\u7aef\u5e38\u7528\u914d\u7f6e FileType \u679a\u4e3e \u00a7 \u679a\u4e3e\u7c7b\u578b \u679a\u4e3e\u503c \u63cf\u8ff0 Unknown 0 - Directory 1 \u76ee\u5f55 File 2 \u6587\u4ef6 Symlink 3 \u8f6f\u94fe FileInfo \u7c7b \u00a7 \u6210\u5458\u5c5e\u6027 \u8fd4\u56de\u503c\u7c7b\u578b \u63cf\u8ff0 type FileType \u6587\u4ef6\u7c7b\u578b is_file bool \u662f\u5426\u4e3a\u6587\u4ef6 is_dir bool \u662f\u5426\u4e3a\u76ee\u5f55 is_symlink bool \u662f\u5426\u4e3a\u8f6f\u94fe path str \u8def\u5f84 user str \u7528\u6237 group str \u7528\u6237\u7ec4 size int \u6587\u4ef6\u5927\u5c0f perm int \u6587\u4ef6\u6743\u9650 atime datetime \u6587\u4ef6\u6700\u540e\u8bbf\u95ee\u65f6\u95f4 mtime datetime \u6587\u4ef6\u6700\u540e\u4fee\u6539\u65f6\u95f4 FileStream \u7c7b \u00a7 \u6210\u5458\u51fd\u6570 \u8fd4\u56de\u503c\u7c7b\u578b \u63cf\u8ff0 readable() bool \u662f\u5426\u53ef\u8bfb writable() bool \u662f\u5426\u53ef\u5199 seekable() bool \u662f\u5426\u53efseek closed() bool \u662f\u5426\u5df2\u5173\u95ed close() \u65e0\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u5173\u95ed\u6587\u4ef6 size() int, \u5931\u8d25\u65f6\u629b\u51faIOError \u6587\u4ef6\u5927\u5c0f\uff08\u4ec5\u53ef\u8bfb\u65f6\u4f7f\u7528\uff09 tell() int, \u5931\u8d25\u65f6\u629b\u51faIOError \u6587\u4ef6\u6d41\u4f4d\u7f6e flush() \u65e0\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u5237\u65b0\u7f13\u5b58 write(data) \u65e0\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u5199\u6570\u636e\uff0cdata\u4e3abytes\u7c7b\u578b\uff0c\u4e3a\u5199\u5165buffer read(nbytes) bytes, \u5931\u8d25\u65f6\u629b\u51faIOError \u8bfb\u6570\u636e\uff0cnbytes\u4e3aint\uff0c\u4e3a\u8bfb\u53d6\u5927\u5c0f pread(nbytes, offset) bytes, \u5931\u8d25\u65f6\u629b\u51faIOError \u968f\u673a\u8bfb\u6570\u636e\uff0cnbytes\u4e3aint\uff0c\u8868\u8bfb\u53d6\u5927\u5c0f\uff1boffset\u4e3aint\uff0c\u8868\u6587\u4ef6\u4f4d\u79fb readall() bytes, \u5931\u8d25\u65f6\u629b\u51faIOError \u8bfb\u53d6\u6574\u4e2a\u6587\u4ef6 download(stream_or_path, buffer_size) \u65e0, \u5931\u8d25\u65f6\u629b\u51faIOError \u4e0b\u8f7d\uff0c\u8bfb\u53d6\u5f53\u524d\u6587\u4ef6\uff0c\u5199\u5165\u5230\u672c\u5730\u8def\u5f84\u6216\u7740\u76ee\u6807\u6d41\u3002stream_or_path\u53ef\u4ee5\u662f\u672c\u5730\u8def\u5f84\u4e5f\u53ef\u4ee5\u6587\u4ef6\u6d41\u3002 upload(stream, buffer_size) \u65e0\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u4e0a\u4f20\uff0c\u8bfb\u53d6stream\u6d41\uff0c\u5199\u5165\u5f53\u524d\u6587\u4ef6\u3002 FileSystem \u7c7b\u578b \u00a7 \u6210\u5458\u51fd\u6570 \u8fd4\u56de\u503c\u7c7b\u578b \u63cf\u8ff0 mkdir(path, recursive) bool\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u521b\u5efa\u76ee\u5f55\uff0crecursive\u4e3abool\uff0c\u8868\u662f\u5426\u9012\u5f52\u521b\u5efa\u7236\u76ee\u5f55 rename(src, dest) bool\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError rename\u6587\u4ef6\uff0c\u628asrc\u8def\u5f84\u79fb\u52a8\u5230dest\u8def\u5f84\uff0csrc\u548cdest\u8def\u5f84\u4e3astr get_file_info(path) FileInfo\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u83b7\u53d6\u6587\u4ef6\u4fe1\u606f exists(path) bool\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u6587\u4ef6\u662f\u5426\u5b58\u5728 listdir(path, recursive) FileInfo\u5217\u8868\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u5217\u4e3e\u6587\u4ef6\uff0crecursive\u4e3abool\uff0c\u8868\u662f\u5426\u9012\u5f52 chmod(path, perm) bool\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u7c7b\u4f3csetPermission\uff0cperm\u4e3a\u6743\u9650\uff0c\u59820o777 chown(path, owner, group) bool\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u7c7b\u4f3csetOwner\uff0cowner\u4e3astr\uff0c\u8868\u7528\u6237\u540d\uff0cgroup\u4e3astr\uff0c\u8868\u7528\u6237\u7ec4 open(path, mode, buffer_size=None) FileStream\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u6253\u5f00\u6587\u4ef6\uff0cmode\u652f\u6301 rb , wb \u3002buffer_size\u9ed8\u8ba4\u4e3a64k\uff0c\u4e3aupload\u3001download\u3001copy_file\u65f6\u7684buffer\u5927\u5c0f\u3002 download(path, stream_or_path, buffer_size=None) \u65e0, \u5931\u8d25\u65f6\u629b\u51faIOError \u4e0b\u8f7d\uff0c\u8bfb\u53d6\u8fdc\u7aefpath\u7684\u6587\u4ef6\uff0c\u5230\u672c\u5730\u8def\u5f84\u6216\u7740\u76ee\u6807\u6d41\u3002stream_or_path\u53ef\u4ee5\u662f\u672c\u5730\u8def\u5f84\u4e5f\u53ef\u4ee5\u6587\u4ef6\u6d41\u3002buffer_size\u9ed8\u8ba4\u4e3a64k\uff0c\u5982\u914d\u7f6e\u4e2d\u5b58\u5728 fs.oss.read.buffer.size \uff0c\u5219\u4ee5\u914d\u7f6e\u4e3a\u51c6\u3002 upload(path, stream, buffer_size=None) \u65e0, \u5931\u8d25\u65f6\u629b\u51faIOError \u4e0a\u4f20stream\u6d41\u5230\u8fdc\u7aefpath\u3002buffer_size\u9ed8\u8ba4\u4e3a64k\uff0c\u5982\u914d\u7f6e\u4e2d\u5b58\u5728 fs.oss.write.buffer.size \uff0c\u5219\u4ee5\u914d\u7f6e\u4e3a\u51c6\u3002 copy_file(src, dest, buffer_size=None) \u65e0, \u5931\u8d25\u65f6\u629b\u51faIOError \u62f7\u8d1d\u6587\u4ef6\uff0c\u4ecesrc\u8def\u5f84\u62f7\u8d1d\u5230dest\u8def\u5f84\u3002buffer_size\u9ed8\u8ba4\u4e3a64k\uff0c\u5982\u914d\u7f6e\u4e2d\u5b58\u5728 fs.oss.read.buffer.size \uff0c\u5219\u4ee5\u914d\u7f6e\u4e3a\u51c6\u3002 fs \u6a21\u5757 \u00a7 \u5168\u5c40\u51fd\u6570 \u8fd4\u56de\u503c\u7c7b\u578b \u63cf\u8ff0 read_config() Config \u8bfb\u53d6\u914d\u7f6e 1. \u68c0\u67e5\u73af\u5883\u53d8\u91cfJINDOSDK_CONF_DIR\uff0c\u5982\u5b58\u5728\uff0c\u5219\u4ece$JINDOSDK_CONF_DIR/jindosdk.cfg\u4e2d\u8bfb\u53d6\u914d\u7f6e\u3002 2. \u68c0\u67e5\u73af\u5883\u53d8\u91cfHADOOP_CONF_DIR\uff0c\u5982\u5b58\u5728\u4e14jindosdk.cfg\u4e2d\u672a\u914d\u7f6ehadoopConf.enable\u4e3afalse\uff0c\u5219\u4ece$HADOOP_CONF_DIR/core-sites.xml\u4e2d\u8bfb\u53d6\u914d\u7f6e\u3002 connect(uri, user, config) FileSystem\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u521d\u59cb\u5316 FileSystem API \u517c\u5bb9\u6027 \u00a7 ossfs \u6a21\u5757\u517c\u5bb9 fsspec \u63a5\u53e3 \u00a7 pyjindo \u540c\u6837\u517c\u5bb9 fsspec \u540c\u6b65\u63a5\u53e3\uff0c\u4f46\u4ec5\u517c\u5bb9 python3.7 \u4ee5\u4e0a\u7248\u672c\u3002\u4ec5\u63a5\u53e3\u517c\u5bb9\uff0c\u8bfb\u53d6\u914d\u7f6e\u3001\u65e5\u5fd7\u7b49\u7ea7\u7b49\u4f7f\u7528\u65b9\u5f0f\u4e0d\u53d8\u3002 \u4f7f\u7528\u524d\u9700\u8981\u5b89\u88c5\u4f9d\u8d56 fsspec \u00a7 \u4ee5 Python3.8 \u73af\u5883\u5b89\u88c5 fssepc \u4e3a\u4f8b python3.8 -m pip install fsspec --trusted-host mirrors.aliyun.com -i http://mirrors.aliyun.com/pypi/simple/ \u7f16\u5199 python \u6d4b\u8bd5\u7a0b\u5e8f \u00a7 from pyjindo.ossfs import JindoOssFileSystem bucket = \"jindosdk-yanbin-sh\" endpoint = bucket + \".cn-shanghai.oss-dls.aliyuncs.com\" root_path = \"oss://\" + endpoint + \"/\" sub_dir = root_path + \"pyjindotest/\" file_path = root_path + \"hello.txt\" file_path2 = sub_dir + \"hello.txt\" fs = JindoOssFileSystem(root_path) # open\uff08path, mode\uff09\u6a21\u5f0f w,\u6587\u4ef6\u4e0d\u5b58\u5728\u521b\u5efa\u4e00\u4e2a\u6587\u4ef6 out_file = fs.open(file_path,\"wb\") out_file.write(str.encode(\"hello world, pyjindo\")) #\u5199 out_file.close() in_file = fs.open(file_path, \"rb\") data = in_file.read() # \u8bfb print(\"\u5199\u5165\u7684\u6570\u636e\u4e3a%s.\"%(data)) in_file.close() # \u5217\u51fa\u6587\u4ef6 ls_file = fs.ls(root_path, detail=False) print(\"\u76ee\u5f55\u6587\u4ef6\u4e3a%s.\" %(ls_file)) assert file_path in fs.glob(root_path + \"*\") # \u521b\u5efa\u76ee\u5f55 fs.mkdir(sub_dir) # \u79fb\u52a8\u5e76\u91cd\u547d\u540d\u6587\u4ef6 fs.rename(file_path, file_path2) # \u5217\u51fa\u6587\u4ef6 mv_file = fs.listdir(sub_dir, detail=False) print(\"\u79fb\u52a8\u540e\u7684\u76ee\u5f55\u6587\u4ef6\u4e3a%s.\" %(mv_file)) # \u5220\u9664\u6d4b\u8bd5\u6587\u4ef6\uff0c\u91cd\u65b0\u5217\u51fa\u6587\u4ef6 fs.rm(file_path2) de_file = fs.ls(sub_dir) print(\"\u5220\u9664\u6587\u4ef6\u540e\u7684pyjindotest\u76ee\u5f55\u4e0b\u6587\u4ef6\u4e3a%s.\" %(de_file)) \u6267\u884c\u6d4b\u8bd5 \u00a7 python3.8 ossfs_test.py \u6267\u884c\u7ed3\u679c\uff1a \u00a7 \u5199\u5165\u7684\u6570\u636e\u4e3ab'hello world, pyjindo'. \u76ee\u5f55\u6587\u4ef6\u4e3a['oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/.sysinfo/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/apps/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/flume/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/hbase/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/hello.txt', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/pyarrowtest/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/pyjindotest/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/spark-history/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/test/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/tmp/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/user/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/yarn/']. \u79fb\u52a8\u540e\u7684\u76ee\u5f55\u6587\u4ef6\u4e3a['oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/pyjindotest/hello.txt']. \u5220\u9664\u6587\u4ef6\u540e\u7684pyjindotest\u76ee\u5f55\u4e0b\u6587\u4ef6\u4e3a[]. \u66f4\u591a\u63a5\u53e3\u8bf4\u660e\u53c2\u89c1 fssepc \u5b98\u7f51\u6587\u6863 \u3002","title":"Python"},{"location":"jindosdk/python/pyjindo_quickstart/#pyjindo-ossoss-hdfs","text":"","title":"\u4f7f\u7528 PyJindo \u8bbf\u95ee\u963f\u91cc\u4e91 OSS/OSS-HDFS"},{"location":"jindosdk/python/pyjindo_quickstart/#_1","text":"\u672c\u6587\u6307\u5bfc\u5982\u4f55\u4f7f\u7528 Python \u7684\u5de5\u5177\u5305 PyJindo \u64cd\u4f5c OSS/OSS-HDFS\u3002PyJindo \u517c\u5bb9 python3.6 \u4ee5\u4e0a\u7248\u672c\u3002","title":"\u80cc\u666f"},{"location":"jindosdk/python/pyjindo_quickstart/#_2","text":"","title":"\u90e8\u7f72\u73af\u5883"},{"location":"jindosdk/python/pyjindo_quickstart/#python-pyjindo","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.y.z.tar.gz\uff08x.y.z\u8868\u7248\u672c\u53f7\uff09\uff0c\u4e0b\u8f7d\u94fe\u63a5\u89c1 \u4e0b\u8f7d\u9875\u9762 \u3002 \u90e8\u7f72 jindosdk-6.7.0.tar.gz\uff0cwhl\u5b89\u88c5\u5305\u4f4d\u4e8e\u5b8c\u6574\u4ea7\u51fa\u7269 jindosdk-x.x.x/lib/site-packages/ \u7684\u5b50\u76ee\u5f55\u4e2d\u3002\u591a\u5e73\u53f0\u90e8\u7f72\u8bf4\u660e\u53c2\u89c1 \u90e8\u7f72\u6587\u6863 \u3002 \u4ee5 Python3.6 \u7248\u672c\u4e3a\u4f8b\uff0c\u8bf7\u5b89\u88c5pyjindo-x.y.z-cp 36 -abi3-linux_x86_64.whl . \u251c\u2500\u2500 bin \u2502 \u251c\u2500\u2500 xxx \u251c\u2500\u2500 conf \u2502 \u251c\u2500\u2500 xxx \u251c\u2500\u2500 include \u2502 \u251c\u2500\u2500 xxx \u251c\u2500\u2500 lib \u2502 \u251c\u2500\u2500 xxx \u2502 \u251c\u2500\u2500 native \u2502 \u2502 \u251c\u2500\u2500 xxxx \u2502 \u2514\u2500\u2500 site-packages \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp310-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp311-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp312-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp36-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp37-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp38-abi3-linux_x86_64.whl \u2502 \u2514\u2500\u2500 pyjindo-x.y.z-cp39-abi3-linux_x86_64.whl \u251c\u2500\u2500 plugins \u2502 \u2514\u2500\u2500 xxxx \u251c\u2500\u2500 tools \u2502 \u251c\u2500\u2500 xxx \u2514\u2500\u2500 versions \u251c\u2500\u2500 xxx","title":"\u4e0b\u8f7d\u5bf9\u5e94 Python \u7248\u672c\u7684 PyJindo \u5b89\u88c5\u5305"},{"location":"jindosdk/python/pyjindo_quickstart/#_3","text":"export JINDOSDK_CONF_DIR=/etc/taihao-apps/jindosdk-conf export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf \u5728\u963f\u91cc\u4e91EMR\u73af\u5883\u4e2d\u9ed8\u8ba4\u5b58\u5728\u4ee5\u4e0a\u914d\u7f6e\uff0c\u65e0\u9700\u914d\u7f6e\u3002 \u975eEMR\u4e2d\u914d\u7f6e\u65b9\u5f0f\u53c2\u89c1\uff1a \u300a\u5728\u975eEMR\u96c6\u7fa4\u4e2d\u90e8\u7f72JindoSDK\u300b \u3002\u5176\u4e2d\uff0cHadoop\u914d\u7f6e\u6587\u4ef6\u53caHADOOP_CONF_DIR\u4e0d\u662f\u5fc5\u987b\uff0c\u4ec5\u4e3a\u517c\u5bb9HADOOP\u73af\u5883\u4e2d\u7684\u914d\u7f6e\u3002","title":"\u786e\u8ba4\u5b58\u5728\u4ee5\u4e0b\u73af\u5883\u53d8\u91cf\u7528\u4ee5\u8bfb\u53d6\u914d\u7f6e"},{"location":"jindosdk/python/pyjindo_quickstart/#pyjindo","text":"\u4ee5 Python3.8 \u73af\u5883\u5b89\u88c5\u6700\u65b0\u7248\u672c\u7684 pyjindo-6.7.3 \u4e3a\u4f8b python3.8 -m ensurepip python3.8 -m pip install pip --upgrade --trusted-host mirrors.aliyun.com -i http://mirrors.aliyun.com/pypi/simple/ python3.8 -m pip install pyjindo-6.7.3-cp38-abi3-linux_x86_64.whl","title":"\u5b89\u88c5 PyJindo"},{"location":"jindosdk/python/pyjindo_quickstart/#python","text":"from pyjindo import fs bucket = \"jindosdk-yanbin-sh\" endpoint = bucket + \".cn-shanghai.oss-dls.aliyuncs.com\" root_path = \"oss://\" + endpoint + \"/\" sub_dir = root_path + \"pyjindotest/\" file_path = root_path + \"hello.txt\" file_path2 = sub_dir + \"hello.txt\" config = fs.read_config() fs = fs.connect(root_path, \"root\", config) # open\uff08path, mode\uff09\u6a21\u5f0f w,\u6587\u4ef6\u4e0d\u5b58\u5728\u521b\u5efa\u4e00\u4e2a\u6587\u4ef6 out_file = fs.open(file_path,\"wb\") out_file.write(str.encode(\"hello world, pyjindo\")) #\u5199 out_file.close() in_file = fs.open(file_path, \"rb\") data = in_file.read() # \u8bfb print(\"\u5199\u5165\u7684\u6570\u636e\u4e3a%s.\"%(data)) in_file.close() # \u5217\u51fa\u6587\u4ef6 ls_file = fs.listdir(root_path) print(\"\u76ee\u5f55\u6587\u4ef6\u4e3a%s.\" %(ls_file)) # \u521b\u5efa\u76ee\u5f55 fs.mkdir(sub_dir) # \u79fb\u52a8\u5e76\u91cd\u547d\u540d\u6587\u4ef6 fs.rename(file_path, file_path2) # \u5217\u51fa\u6587\u4ef6 mv_file = fs.listdir(sub_dir) print(\"\u79fb\u52a8\u540e\u7684\u76ee\u5f55\u6587\u4ef6\u4e3a%s.\" %(mv_file)) # \u5220\u9664\u6d4b\u8bd5\u6587\u4ef6\uff0c\u91cd\u65b0\u5217\u51fa\u6587\u4ef6 fs.remove(file_path2) de_file = fs.listdir(sub_dir) print(\"\u5220\u9664\u6587\u4ef6\u540e\u7684pyjindotest\u76ee\u5f55\u4e0b\u6587\u4ef6\u4e3a%s.\" %(de_file))","title":"\u7f16\u5199 python \u6d4b\u8bd5\u7a0b\u5e8f"},{"location":"jindosdk/python/pyjindo_quickstart/#_4","text":"python3.8 fs_test.py","title":"\u6267\u884c\u6d4b\u8bd5"},{"location":"jindosdk/python/pyjindo_quickstart/#_5","text":"\u5199\u5165\u7684\u6570\u636e\u4e3ab'hello world, pyjindo'. \u76ee\u5f55\u6587\u4ef6\u4e3a[<FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/.sysinfo/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/apps/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/flume/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/hbase/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/hello.txt': type=File, size=20>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/pyarrowtest/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/spark-history/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/tmp/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/user/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/yarn/': type=Directory>]. \u79fb\u52a8\u540e\u7684\u76ee\u5f55\u6587\u4ef6\u4e3a[<FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/pyjindotest/hello.txt': type=File, size=20>]. \u5220\u9664\u6587\u4ef6\u540e\u7684pyjindotest\u76ee\u5f55\u4e0b\u6587\u4ef6\u4e3a[].","title":"\u6267\u884c\u7ed3\u679c"},{"location":"jindosdk/python/pyjindo_quickstart/#_6","text":"\u8c03\u6574JINDOSDK_CONF_DIR\u4e0b\u7684jindosdk.cfg\u914d\u7f6e\uff0cemr\u4e0a\u5bf9\u5e94/etc/taihao-apps/jindosdk-conf [common] logger.dir = /var/log/emr/jindosdk logger.level = 2 logger.verbose = 0 logger.sync = false logger.jnilogger = true logger.consolelogger = false logger.cleaner.enable = true \u914d\u7f6e\u9879 \u63cf\u8ff0 logger.dir \u65e5\u5fd7\u8f93\u51fa\u76ee\u5f55\u3002 logger.level \u63a8\u8350\u4e3a2\uff0clevel<=1\uff0c\u8868\u793aWARN\uff1blevel>1\uff0c\u8868\u793aINFO\u3002 logger.verbose \u8be6\u7ec6\u65e5\u5fd7\u7b49\u7ea7\uff0c\u8303\u56f4\u4e3a0-99\uff0c\u503c\u8d8a\u5927\u65e5\u5fd7\u8d8a\u8be6\u7ec6\u3002 logger.sync \u63a8\u8350\u4e3afalse\u3002true\u8868\u793a\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\u3002 logger.jnilogger \u4f7f\u7528jni\u65f6\u751f\u6548\uff0c\u4e0epyjindo\u65e0\u5173\u3002 logger.consolelogger \u7ec8\u7aef\u8f93\u51fa\u65f6\u751f\u6548\uff0c\u4e0epyjindo\u65e0\u5173\u3002 logger.cleaner.enable \u63a8\u8350\u4e3atrue\uff0cfalse\u8868\u793a\u5173\u95ed\u65e5\u5fd7\u81ea\u52a8\u6e05\u7406\u3002","title":"\u65e5\u5fd7\u7b49\u7ea7"},{"location":"jindosdk/python/pyjindo_quickstart/#api","text":"","title":"API \u8bf4\u660e"},{"location":"jindosdk/python/pyjindo_quickstart/#config","text":"\u6210\u5458\u51fd\u6570 \u8fd4\u56de\u503c\u7c7b\u578b \u63cf\u8ff0 set(key, val) \u65e0 \u8bbe\u7f6e\u5b57\u7b26\u4e32\u7c7b\u578b\u914d\u7f6e\uff0ckey\u4e0eval\u5747\u4e3astr get(key, default='') str \u83b7\u53d6\u5b57\u7b26\u4e32\u7c7b\u578b\u914d\u7f6e contains(key) bool \u662f\u5426\u5b58\u5728\u914d\u7f6ekey \u914d\u7f6e\u8bf4\u660e\u53c2\u89c1\uff1a \u5ba2\u6237\u7aef\u5e38\u7528\u914d\u7f6e","title":"Config \u7c7b"},{"location":"jindosdk/python/pyjindo_quickstart/#filetype","text":"\u679a\u4e3e\u7c7b\u578b \u679a\u4e3e\u503c \u63cf\u8ff0 Unknown 0 - Directory 1 \u76ee\u5f55 File 2 \u6587\u4ef6 Symlink 3 \u8f6f\u94fe","title":"FileType \u679a\u4e3e"},{"location":"jindosdk/python/pyjindo_quickstart/#fileinfo","text":"\u6210\u5458\u5c5e\u6027 \u8fd4\u56de\u503c\u7c7b\u578b \u63cf\u8ff0 type FileType \u6587\u4ef6\u7c7b\u578b is_file bool \u662f\u5426\u4e3a\u6587\u4ef6 is_dir bool \u662f\u5426\u4e3a\u76ee\u5f55 is_symlink bool \u662f\u5426\u4e3a\u8f6f\u94fe path str \u8def\u5f84 user str \u7528\u6237 group str \u7528\u6237\u7ec4 size int \u6587\u4ef6\u5927\u5c0f perm int \u6587\u4ef6\u6743\u9650 atime datetime \u6587\u4ef6\u6700\u540e\u8bbf\u95ee\u65f6\u95f4 mtime datetime \u6587\u4ef6\u6700\u540e\u4fee\u6539\u65f6\u95f4","title":"FileInfo \u7c7b"},{"location":"jindosdk/python/pyjindo_quickstart/#filestream","text":"\u6210\u5458\u51fd\u6570 \u8fd4\u56de\u503c\u7c7b\u578b \u63cf\u8ff0 readable() bool \u662f\u5426\u53ef\u8bfb writable() bool \u662f\u5426\u53ef\u5199 seekable() bool \u662f\u5426\u53efseek closed() bool \u662f\u5426\u5df2\u5173\u95ed close() \u65e0\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u5173\u95ed\u6587\u4ef6 size() int, \u5931\u8d25\u65f6\u629b\u51faIOError \u6587\u4ef6\u5927\u5c0f\uff08\u4ec5\u53ef\u8bfb\u65f6\u4f7f\u7528\uff09 tell() int, \u5931\u8d25\u65f6\u629b\u51faIOError \u6587\u4ef6\u6d41\u4f4d\u7f6e flush() \u65e0\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u5237\u65b0\u7f13\u5b58 write(data) \u65e0\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u5199\u6570\u636e\uff0cdata\u4e3abytes\u7c7b\u578b\uff0c\u4e3a\u5199\u5165buffer read(nbytes) bytes, \u5931\u8d25\u65f6\u629b\u51faIOError \u8bfb\u6570\u636e\uff0cnbytes\u4e3aint\uff0c\u4e3a\u8bfb\u53d6\u5927\u5c0f pread(nbytes, offset) bytes, \u5931\u8d25\u65f6\u629b\u51faIOError \u968f\u673a\u8bfb\u6570\u636e\uff0cnbytes\u4e3aint\uff0c\u8868\u8bfb\u53d6\u5927\u5c0f\uff1boffset\u4e3aint\uff0c\u8868\u6587\u4ef6\u4f4d\u79fb readall() bytes, \u5931\u8d25\u65f6\u629b\u51faIOError \u8bfb\u53d6\u6574\u4e2a\u6587\u4ef6 download(stream_or_path, buffer_size) \u65e0, \u5931\u8d25\u65f6\u629b\u51faIOError \u4e0b\u8f7d\uff0c\u8bfb\u53d6\u5f53\u524d\u6587\u4ef6\uff0c\u5199\u5165\u5230\u672c\u5730\u8def\u5f84\u6216\u7740\u76ee\u6807\u6d41\u3002stream_or_path\u53ef\u4ee5\u662f\u672c\u5730\u8def\u5f84\u4e5f\u53ef\u4ee5\u6587\u4ef6\u6d41\u3002 upload(stream, buffer_size) \u65e0\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u4e0a\u4f20\uff0c\u8bfb\u53d6stream\u6d41\uff0c\u5199\u5165\u5f53\u524d\u6587\u4ef6\u3002","title":"FileStream \u7c7b"},{"location":"jindosdk/python/pyjindo_quickstart/#filesystem","text":"\u6210\u5458\u51fd\u6570 \u8fd4\u56de\u503c\u7c7b\u578b \u63cf\u8ff0 mkdir(path, recursive) bool\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u521b\u5efa\u76ee\u5f55\uff0crecursive\u4e3abool\uff0c\u8868\u662f\u5426\u9012\u5f52\u521b\u5efa\u7236\u76ee\u5f55 rename(src, dest) bool\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError rename\u6587\u4ef6\uff0c\u628asrc\u8def\u5f84\u79fb\u52a8\u5230dest\u8def\u5f84\uff0csrc\u548cdest\u8def\u5f84\u4e3astr get_file_info(path) FileInfo\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u83b7\u53d6\u6587\u4ef6\u4fe1\u606f exists(path) bool\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u6587\u4ef6\u662f\u5426\u5b58\u5728 listdir(path, recursive) FileInfo\u5217\u8868\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u5217\u4e3e\u6587\u4ef6\uff0crecursive\u4e3abool\uff0c\u8868\u662f\u5426\u9012\u5f52 chmod(path, perm) bool\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u7c7b\u4f3csetPermission\uff0cperm\u4e3a\u6743\u9650\uff0c\u59820o777 chown(path, owner, group) bool\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u7c7b\u4f3csetOwner\uff0cowner\u4e3astr\uff0c\u8868\u7528\u6237\u540d\uff0cgroup\u4e3astr\uff0c\u8868\u7528\u6237\u7ec4 open(path, mode, buffer_size=None) FileStream\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u6253\u5f00\u6587\u4ef6\uff0cmode\u652f\u6301 rb , wb \u3002buffer_size\u9ed8\u8ba4\u4e3a64k\uff0c\u4e3aupload\u3001download\u3001copy_file\u65f6\u7684buffer\u5927\u5c0f\u3002 download(path, stream_or_path, buffer_size=None) \u65e0, \u5931\u8d25\u65f6\u629b\u51faIOError \u4e0b\u8f7d\uff0c\u8bfb\u53d6\u8fdc\u7aefpath\u7684\u6587\u4ef6\uff0c\u5230\u672c\u5730\u8def\u5f84\u6216\u7740\u76ee\u6807\u6d41\u3002stream_or_path\u53ef\u4ee5\u662f\u672c\u5730\u8def\u5f84\u4e5f\u53ef\u4ee5\u6587\u4ef6\u6d41\u3002buffer_size\u9ed8\u8ba4\u4e3a64k\uff0c\u5982\u914d\u7f6e\u4e2d\u5b58\u5728 fs.oss.read.buffer.size \uff0c\u5219\u4ee5\u914d\u7f6e\u4e3a\u51c6\u3002 upload(path, stream, buffer_size=None) \u65e0, \u5931\u8d25\u65f6\u629b\u51faIOError \u4e0a\u4f20stream\u6d41\u5230\u8fdc\u7aefpath\u3002buffer_size\u9ed8\u8ba4\u4e3a64k\uff0c\u5982\u914d\u7f6e\u4e2d\u5b58\u5728 fs.oss.write.buffer.size \uff0c\u5219\u4ee5\u914d\u7f6e\u4e3a\u51c6\u3002 copy_file(src, dest, buffer_size=None) \u65e0, \u5931\u8d25\u65f6\u629b\u51faIOError \u62f7\u8d1d\u6587\u4ef6\uff0c\u4ecesrc\u8def\u5f84\u62f7\u8d1d\u5230dest\u8def\u5f84\u3002buffer_size\u9ed8\u8ba4\u4e3a64k\uff0c\u5982\u914d\u7f6e\u4e2d\u5b58\u5728 fs.oss.read.buffer.size \uff0c\u5219\u4ee5\u914d\u7f6e\u4e3a\u51c6\u3002","title":"FileSystem \u7c7b\u578b"},{"location":"jindosdk/python/pyjindo_quickstart/#fs","text":"\u5168\u5c40\u51fd\u6570 \u8fd4\u56de\u503c\u7c7b\u578b \u63cf\u8ff0 read_config() Config \u8bfb\u53d6\u914d\u7f6e 1. \u68c0\u67e5\u73af\u5883\u53d8\u91cfJINDOSDK_CONF_DIR\uff0c\u5982\u5b58\u5728\uff0c\u5219\u4ece$JINDOSDK_CONF_DIR/jindosdk.cfg\u4e2d\u8bfb\u53d6\u914d\u7f6e\u3002 2. \u68c0\u67e5\u73af\u5883\u53d8\u91cfHADOOP_CONF_DIR\uff0c\u5982\u5b58\u5728\u4e14jindosdk.cfg\u4e2d\u672a\u914d\u7f6ehadoopConf.enable\u4e3afalse\uff0c\u5219\u4ece$HADOOP_CONF_DIR/core-sites.xml\u4e2d\u8bfb\u53d6\u914d\u7f6e\u3002 connect(uri, user, config) FileSystem\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u521d\u59cb\u5316 FileSystem","title":"fs \u6a21\u5757"},{"location":"jindosdk/python/pyjindo_quickstart/#api_1","text":"","title":"API \u517c\u5bb9\u6027"},{"location":"jindosdk/python/pyjindo_quickstart/#ossfs-fsspec","text":"pyjindo \u540c\u6837\u517c\u5bb9 fsspec \u540c\u6b65\u63a5\u53e3\uff0c\u4f46\u4ec5\u517c\u5bb9 python3.7 \u4ee5\u4e0a\u7248\u672c\u3002\u4ec5\u63a5\u53e3\u517c\u5bb9\uff0c\u8bfb\u53d6\u914d\u7f6e\u3001\u65e5\u5fd7\u7b49\u7ea7\u7b49\u4f7f\u7528\u65b9\u5f0f\u4e0d\u53d8\u3002","title":"ossfs \u6a21\u5757\u517c\u5bb9 fsspec \u63a5\u53e3"},{"location":"jindosdk/python/pyjindo_quickstart/#fsspec","text":"\u4ee5 Python3.8 \u73af\u5883\u5b89\u88c5 fssepc \u4e3a\u4f8b python3.8 -m pip install fsspec --trusted-host mirrors.aliyun.com -i http://mirrors.aliyun.com/pypi/simple/","title":"\u4f7f\u7528\u524d\u9700\u8981\u5b89\u88c5\u4f9d\u8d56 fsspec"},{"location":"jindosdk/python/pyjindo_quickstart/#python_1","text":"from pyjindo.ossfs import JindoOssFileSystem bucket = \"jindosdk-yanbin-sh\" endpoint = bucket + \".cn-shanghai.oss-dls.aliyuncs.com\" root_path = \"oss://\" + endpoint + \"/\" sub_dir = root_path + \"pyjindotest/\" file_path = root_path + \"hello.txt\" file_path2 = sub_dir + \"hello.txt\" fs = JindoOssFileSystem(root_path) # open\uff08path, mode\uff09\u6a21\u5f0f w,\u6587\u4ef6\u4e0d\u5b58\u5728\u521b\u5efa\u4e00\u4e2a\u6587\u4ef6 out_file = fs.open(file_path,\"wb\") out_file.write(str.encode(\"hello world, pyjindo\")) #\u5199 out_file.close() in_file = fs.open(file_path, \"rb\") data = in_file.read() # \u8bfb print(\"\u5199\u5165\u7684\u6570\u636e\u4e3a%s.\"%(data)) in_file.close() # \u5217\u51fa\u6587\u4ef6 ls_file = fs.ls(root_path, detail=False) print(\"\u76ee\u5f55\u6587\u4ef6\u4e3a%s.\" %(ls_file)) assert file_path in fs.glob(root_path + \"*\") # \u521b\u5efa\u76ee\u5f55 fs.mkdir(sub_dir) # \u79fb\u52a8\u5e76\u91cd\u547d\u540d\u6587\u4ef6 fs.rename(file_path, file_path2) # \u5217\u51fa\u6587\u4ef6 mv_file = fs.listdir(sub_dir, detail=False) print(\"\u79fb\u52a8\u540e\u7684\u76ee\u5f55\u6587\u4ef6\u4e3a%s.\" %(mv_file)) # \u5220\u9664\u6d4b\u8bd5\u6587\u4ef6\uff0c\u91cd\u65b0\u5217\u51fa\u6587\u4ef6 fs.rm(file_path2) de_file = fs.ls(sub_dir) print(\"\u5220\u9664\u6587\u4ef6\u540e\u7684pyjindotest\u76ee\u5f55\u4e0b\u6587\u4ef6\u4e3a%s.\" %(de_file))","title":"\u7f16\u5199 python \u6d4b\u8bd5\u7a0b\u5e8f"},{"location":"jindosdk/python/pyjindo_quickstart/#_7","text":"python3.8 ossfs_test.py","title":"\u6267\u884c\u6d4b\u8bd5"},{"location":"jindosdk/python/pyjindo_quickstart/#_8","text":"\u5199\u5165\u7684\u6570\u636e\u4e3ab'hello world, pyjindo'. \u76ee\u5f55\u6587\u4ef6\u4e3a['oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/.sysinfo/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/apps/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/flume/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/hbase/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/hello.txt', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/pyarrowtest/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/pyjindotest/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/spark-history/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/test/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/tmp/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/user/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/yarn/']. \u79fb\u52a8\u540e\u7684\u76ee\u5f55\u6587\u4ef6\u4e3a['oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/pyjindotest/hello.txt']. \u5220\u9664\u6587\u4ef6\u540e\u7684pyjindotest\u76ee\u5f55\u4e0b\u6587\u4ef6\u4e3a[]. \u66f4\u591a\u63a5\u53e3\u8bf4\u660e\u53c2\u89c1 fssepc \u5b98\u7f51\u6587\u6863 \u3002","title":"\u6267\u884c\u7ed3\u679c\uff1a"},{"location":"jindosdk/spark/jindosdk_ide_spark/","text":"Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 \u00a7 \u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.spark.sql.SparkSession; public class TestJindoSDK { public static void main(String[] args) throws Exception { SparkSession spark = SparkSession .builder() .config(\"spark.hadoop.fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\") .config(\"spark.hadoop.fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\") .config(\"spark.hadoop.fs.oss.accessKeyId\", \"xxx\") .config(\"spark.hadoop.fs.oss.accessKeySecret\", \"xxx\") // set accessKey, secret, endpoint and so on. .appName(\"TestJindoSDK\") .getOrCreate(); spark.read().parquet(\"oss://bucket.endpoint/xxx\").count(); spark.stop(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <!-- ... --> <dependencies> <!-- add jindo-core-extended-jar for macos --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> </dependencies> </project>","title":"Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5"},{"location":"jindosdk/spark/jindosdk_ide_spark/#spark-jindosdk-ide","text":"\u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.spark.sql.SparkSession; public class TestJindoSDK { public static void main(String[] args) throws Exception { SparkSession spark = SparkSession .builder() .config(\"spark.hadoop.fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\") .config(\"spark.hadoop.fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\") .config(\"spark.hadoop.fs.oss.accessKeyId\", \"xxx\") .config(\"spark.hadoop.fs.oss.accessKeySecret\", \"xxx\") // set accessKey, secret, endpoint and so on. .appName(\"TestJindoSDK\") .getOrCreate(); spark.read().parquet(\"oss://bucket.endpoint/xxx\").count(); spark.stop(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <!-- ... --> <dependencies> <!-- add jindo-core-extended-jar for macos --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> </dependencies> </project>","title":"Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5"},{"location":"jindosdk/spark/jindosdk_on_spark/","text":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e \u00a7 JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411Hadoop/Spark\u751f\u6001\u7684OSS\u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684Hadoop FileSystem\u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fddSpark\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/ 3. \u914d\u7f6e JindoSDK \u00a7 \u5168\u5c40\u914d\u7f6e, \u4fee\u6539 Spark \u4f7f\u7528\u7684 core-site.xml \u00a7 \u914d\u7f6e JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u4efb\u52a1\u7ea7\u522b\u914d\u7f6e \u00a7 \u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx 4. \u4f7f\u7528 Spark \u8bbf\u95ee OSS \u00a7 \u521b\u5efa\u8868 create table test_oss (c1 string) location \"oss://bucket.endpoint/dir\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss; 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"jindosdk/spark/jindosdk_on_spark/#spark-jindosdk-oss-hdfs-jindofs","text":"JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411Hadoop/Spark\u751f\u6001\u7684OSS\u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684Hadoop FileSystem\u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"jindosdk/spark/jindosdk_on_spark/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"jindosdk/spark/jindosdk_on_spark/#1-spark-hadoop","text":"\u786e\u4fddSpark\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"jindosdk/spark/jindosdk_on_spark/#2-spark-classpath-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/","title":"2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK"},{"location":"jindosdk/spark/jindosdk_on_spark/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"jindosdk/spark/jindosdk_on_spark/#spark-core-sitexml","text":"\u914d\u7f6e JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002","title":"\u5168\u5c40\u914d\u7f6e, \u4fee\u6539 Spark \u4f7f\u7528\u7684core-site.xml"},{"location":"jindosdk/spark/jindosdk_on_spark/#_2","text":"\u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx","title":"\u4efb\u52a1\u7ea7\u522b\u914d\u7f6e"},{"location":"jindosdk/spark/jindosdk_on_spark/#4-spark-oss","text":"\u521b\u5efa\u8868 create table test_oss (c1 string) location \"oss://bucket.endpoint/dir\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss;","title":"4. \u4f7f\u7528 Spark \u8bbf\u95ee OSS"},{"location":"jindosdk/spark/jindosdk_on_spark/#5","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/","text":"\u4f7f\u7528 Jindo Tensorflow Collector \u8bbf\u95ee OSS/OSS-HDFS \u00a7 Jindo Tensorflow Collector \u652f\u6301\u4f7f\u7528 Tensoflow \u539f\u751f API \u652f\u6301\u8bbf\u95ee OSS/OSS-HDFS\u3002 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.y.z.tar.gz\uff08x.y.z\u8868\u7248\u672c\u53f7\uff09\uff0c ( \u4e0b\u8f7d\u9875\u9762 )\u3002 \u90e8\u7f72\u65b9\u5f0f\uff0c\u53c2\u89c1 \u90e8\u7f72\u6587\u6863 2. \u4fee\u6539\u914d\u7f6e \u00a7 2.1 \u5b89\u88c5 tensorflow \u00a7 \u76ee\u524d\u4ec5\u652f\u6301 tensorflow 1.15 \u6216 tensorflow 2.8 \u7248\u672c\uff0c\u4ee5\u4e0b\u4ee5 tensorflow 2.8 \u4e3a\u4f8b\u3002 pip3 install tensorflow==2.8 2.2 \u6dfb\u52a0\u73af\u5883\u53d8\u91cf \u00a7 2.2.1 JindoSDK \u73af\u5883\u53d8\u91cf \u00a7 \u786e\u8ba4 LD_LIBRARY_PATH \u5305\u542b /path/to/jindosdk-x.x.x/lib/native 2.2.2 \u4f9d\u8d56\u7684 Tensorflow \u73af\u5883\u53d8\u91cf \u00a7 \u540c\u65f6\uff0cLD_LIBRARY_PATH \u9700\u8981\u6dfb\u52a0 tensorflow \u6240\u5728\u7684\u4f9d\u8d56\u8def\u5f84\uff0c\u4e0d\u540c\u7684 python \u73af\u5883\u5b89\u88c5\u8def\u5f84\u53ef\u80fd\u4e0d\u540c\uff0c\u5982\uff1a # tensorflow1.15 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow/\":$LD_LIBRARY_PATH # tensorflow2.8 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow/\":$LD_LIBRARY_PATH 2.3 \u914d\u7f6e\u6587\u4ef6 \u00a7 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \u3002\u53ef\u4ee5\u9009\u62e9\u56fa\u5b9a\u6216\u8005\u514d\u5bc6\u4e24\u79cd\u914d\u7f6e\u65b9\u5f0f\u3002 2.3.1 \u56fa\u5b9a AK \u8bbf\u95ee \u00a7 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/tfio-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95eeOSS\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> \u914d\u7f6e\u9009\u9879 \u00a7 \u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u914d\u7f6e\u8282 \u8bf4\u660e logger.dir /tmp/tfio-log common \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false common \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false common \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\u3002 \u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002 logger.verbose 0 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false common \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u5730\u5740\uff0c\u5982cn-xxx.oss-dls.aliyuncs.com fs.oss.accessKeyId jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeySecret \u66f4\u591a jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 2.3.2 \u514d\u5bc6\u8bbf\u95ee \u00a7 \u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7\u89d2\u8272\u6388\u6743\u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/tensorflow-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE 3. \u4f7f\u7528 Jindo Tensorflow \u00a7 \u4ee5\u52a0\u8f7d jindo-tensorflow2.8 \u4e3a\u4f8b 3.1 \u901a\u8fc7 load_library \u6dfb\u52a0 collector \u00a7 import tensorflow as tf tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\") 3.2 \u901a\u8fc7 gfile \u8bbf\u95ee oss-hdfs \u00a7 \u4e3a\u907f\u514d\u4e0e tfio \u4e2d\u7684 oss \u524d\u7f00\u51b2\u7a81\uff0cjindo-tensorflow \u4f7f\u7528 joss \u524d\u7f00\u3002 # \u76ee\u5f55\u64cd\u4f5c tf.io.gfile.mkdir(\"joss://<bucket>/test_mkdir\") content = tf.io.gfile.listdir(\"joss://<bucket>/test_mkdir\") # \u8bfb gmnist = tf.io.gfile.GFile(\"joss://<bucket>/mnist.npz\", \"rb\") gmnist_file = gmnist.read() # \u5199 fh = gfile.Open(f, mode=\"w\") content = \"file content\" fh.write(content) fh.close() 3.3 \u8bfb\u53d6 TFRecordDataset \u00a7 tfRecordOssPath = \"joss://<bucket>/test.tfrecord\" filenames = [tfRecordOssPath] raw_dataset = tf.data.TFRecordDataset(filenames) 3.4 \u5b8c\u6574\u793a\u4f8b \u00a7 import tensorflow as tf import numpy as np tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\") gmnist = tf.io.gfile.GFile(\"joss://<bucket>/mnist.npz\", 'rb') print(gmnist.size()) with np.load(gmnist) as f: x_train, y_train = f['x_train'], f['y_train'] x_test, y_test = f['x_test'], f['y_test'] model = tf.keras.models.Sequential([ tf.keras.layers.Flatten(input_shape=(28, 28)), tf.keras.layers.Dense(128, activation='relu'), tf.keras.layers.Dropout(0.2), tf.keras.layers.Dense(10, activation='softmax') ]) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) model.fit(x_train, y_train, epochs=5)","title":"Tensorflow"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#jindo-tensorflow-collector-ossoss-hdfs","text":"Jindo Tensorflow Collector \u652f\u6301\u4f7f\u7528 Tensoflow \u539f\u751f API \u652f\u6301\u8bbf\u95ee OSS/OSS-HDFS\u3002","title":"\u4f7f\u7528 Jindo Tensorflow Collector \u8bbf\u95ee OSS/OSS-HDFS"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.y.z.tar.gz\uff08x.y.z\u8868\u7248\u672c\u53f7\uff09\uff0c ( \u4e0b\u8f7d\u9875\u9762 )\u3002 \u90e8\u7f72\u65b9\u5f0f\uff0c\u53c2\u89c1 \u90e8\u7f72\u6587\u6863","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#2","text":"","title":"2. \u4fee\u6539\u914d\u7f6e"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#21-tensorflow","text":"\u76ee\u524d\u4ec5\u652f\u6301 tensorflow 1.15 \u6216 tensorflow 2.8 \u7248\u672c\uff0c\u4ee5\u4e0b\u4ee5 tensorflow 2.8 \u4e3a\u4f8b\u3002 pip3 install tensorflow==2.8","title":"2.1 \u5b89\u88c5 tensorflow"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#22","text":"","title":"2.2 \u6dfb\u52a0\u73af\u5883\u53d8\u91cf"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#221-jindosdk","text":"\u786e\u8ba4 LD_LIBRARY_PATH \u5305\u542b /path/to/jindosdk-x.x.x/lib/native","title":"2.2.1 JindoSDK \u73af\u5883\u53d8\u91cf"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#222-tensorflow","text":"\u540c\u65f6\uff0cLD_LIBRARY_PATH \u9700\u8981\u6dfb\u52a0 tensorflow \u6240\u5728\u7684\u4f9d\u8d56\u8def\u5f84\uff0c\u4e0d\u540c\u7684 python \u73af\u5883\u5b89\u88c5\u8def\u5f84\u53ef\u80fd\u4e0d\u540c\uff0c\u5982\uff1a # tensorflow1.15 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow/\":$LD_LIBRARY_PATH # tensorflow2.8 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow/\":$LD_LIBRARY_PATH","title":"2.2.2 \u4f9d\u8d56\u7684 Tensorflow \u73af\u5883\u53d8\u91cf"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#23","text":"\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \u3002\u53ef\u4ee5\u9009\u62e9\u56fa\u5b9a\u6216\u8005\u514d\u5bc6\u4e24\u79cd\u914d\u7f6e\u65b9\u5f0f\u3002","title":"2.3 \u914d\u7f6e\u6587\u4ef6"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#231-ak","text":"\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/tfio-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95eeOSS\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret>","title":"2.3.1 \u56fa\u5b9a AK \u8bbf\u95ee"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#_1","text":"\u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u914d\u7f6e\u8282 \u8bf4\u660e logger.dir /tmp/tfio-log common \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false common \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false common \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\u3002 \u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002 logger.verbose 0 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false common \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u5730\u5740\uff0c\u5982cn-xxx.oss-dls.aliyuncs.com fs.oss.accessKeyId jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeySecret \u66f4\u591a jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002","title":"\u914d\u7f6e\u9009\u9879"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#232","text":"\u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7\u89d2\u8272\u6388\u6743\u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/tensorflow-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE","title":"2.3.2 \u514d\u5bc6\u8bbf\u95ee"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#3-jindo-tensorflow","text":"\u4ee5\u52a0\u8f7d jindo-tensorflow2.8 \u4e3a\u4f8b","title":"3. \u4f7f\u7528 Jindo Tensorflow"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#31-load_library-collector","text":"import tensorflow as tf tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\")","title":"3.1 \u901a\u8fc7 load_library \u6dfb\u52a0 collector"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#32-gfile-oss-hdfs","text":"\u4e3a\u907f\u514d\u4e0e tfio \u4e2d\u7684 oss \u524d\u7f00\u51b2\u7a81\uff0cjindo-tensorflow \u4f7f\u7528 joss \u524d\u7f00\u3002 # \u76ee\u5f55\u64cd\u4f5c tf.io.gfile.mkdir(\"joss://<bucket>/test_mkdir\") content = tf.io.gfile.listdir(\"joss://<bucket>/test_mkdir\") # \u8bfb gmnist = tf.io.gfile.GFile(\"joss://<bucket>/mnist.npz\", \"rb\") gmnist_file = gmnist.read() # \u5199 fh = gfile.Open(f, mode=\"w\") content = \"file content\" fh.write(content) fh.close()","title":"3.2 \u901a\u8fc7 gfile \u8bbf\u95ee oss-hdfs"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#33-tfrecorddataset","text":"tfRecordOssPath = \"joss://<bucket>/test.tfrecord\" filenames = [tfRecordOssPath] raw_dataset = tf.data.TFRecordDataset(filenames)","title":"3.3 \u8bfb\u53d6 TFRecordDataset"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#34","text":"import tensorflow as tf import numpy as np tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\") gmnist = tf.io.gfile.GFile(\"joss://<bucket>/mnist.npz\", 'rb') print(gmnist.size()) with np.load(gmnist) as f: x_train, y_train = f['x_train'], f['y_train'] x_test, y_test = f['x_test'], f['y_test'] model = tf.keras.models.Sequential([ tf.keras.layers.Flatten(input_shape=(28, 28)), tf.keras.layers.Dense(128, activation='relu'), tf.keras.layers.Dropout(0.2), tf.keras.layers.Dense(10, activation='softmax') ]) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) model.fit(x_train, y_train, epochs=5)","title":"3.4 \u5b8c\u6574\u793a\u4f8b"},{"location":"jindotools/jindodistcp_api_quickstart/","text":"JindoDistCp API \u4f7f\u7528\u8bf4\u660e \u00a7 \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 \u5b89\u88c5\u4f9d\u8d56\u5305 \u00a7 \u76ee\u524d JindoSDK/JindoDistCP \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff0cMac M1 \u7cfb\u5217\u4e5f\u6682\u4e0d\u652f\u6301\uff09 \u4ee56.7.3\u7248\u672c\u4e3a\u4f8b\uff0c\u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoDistCP \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-distcp --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop with provided scope, your runtime should install hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> </dependencies> </project> \u793a\u4f8b\u4ee3\u7801 \u00a7 \u4ee5\u62f7\u8d1d HDFS \u6587\u4ef6\u5939\u5230\u963f\u91cc\u4e91 OSS \u4e3a\u4f8b\u3002 import com.aliyun.jindodata.distcp.util.DistCpCounter; import com.aliyun.jindodata.distjob.request.DistJobResponse; import org.apache.hadoop.conf.Configuration; import java.io.IOException; public class DistcpExample { static public void main(String[] args) throws IOException { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. conf.set(\"fs.oss.credentials.provider\", \"com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider\"); conf.set(\"fs.oss.accessKeyId\", \"XXX\"); conf.set(\"fs.oss.accessKeySecret\", \"XXX\"); DistCpRequest request = new DistCpRequest(); request.setSrcDir(\"hdfs:///user/root/random-data\"); request.setDestDir(\"oss://dls-bucket.cn-xxx.oss-dls.aliyuncs.com/dest\"); request.setWorkTempDir(\"hdfs:///user/root/tmp\"); request.setConf(conf); // Jindo DistCp Service DistCpService service = new DistCpService(); service.buildAllJobs(request); service.runAllJobs(); DistJobResponse response = service.collectAllJobResult(); printResponse(response); } static public void printResponse(DistJobResponse response) { System.out.println(\"ErrorCode: \" + response.getErrorCode()); System.out.println(\"ErrorMsg: \" + response.getErrorMsg()); System.out.println(\"FileCopied: \" + response.getCount(DistCpCounter.FILES_COPIED.getName())); System.out.println(\"BytesCopied: \" + response.getCount(DistCpCounter.BYTES_COPIED.getName())); } } \u8bf7\u6c42\u53c2\u6570 \u00a7 \u76f8\u5173\u65b9\u6cd5\u7684\u5177\u4f53\u63cf\u8ff0\u53ef\u53c2\u8003 \u300a\u4f7f\u7528Jindo DistCp\u300b \u65b9\u6cd5 \u53c2\u6570\u7c7b\u578b \u8bf4\u660e \u9ed8\u8ba4\u503c \u517c\u5bb9 void setSrc(String src) \u5fc5\u9009 \u8bbe\u7f6e\u6e90\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setDest(String dest) \u5fc5\u9009 \u8bbe\u7f6e\u76ee\u6807\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setConf(Configuration conf) \u5fc5\u9009 \u8bbe\u7f6e Configuration - 4.3.0+ void setMode(DistCpMode mode) \u53ef\u9009 \u8bbe\u7f6e DistCp \u6a21\u5f0f\uff0c\u652f\u6301\u7684\u6a21\u5f0f\u6709 COPY\uff08\u62f7\u8d1d\uff09 DIFF\uff08\u5dee\u5f02\u6bd4\u8f83\uff09 UPADTE\uff08\u589e\u91cf\u62f7\u8d1d\uff09 DistCpMode.COPY 4.3.0+ void setBandWidthLimit(int bandWidthLimit) \u53ef\u9009 \u8bbe\u7f6e\u5355\u4e2a\u8282\u70b9\u7684\u5e26\u5bbd\u9650\u5236\uff0c\u5355\u4f4dM -1 4.3.0+ void setOutputCodec(String outputCodec) \u53ef\u9009 \u8bbe\u7f6e\u538b\u7f29\u7c7b\u578b\uff0c\u652f\u6301\u7f16\u89e3\u7801\u5668\u6709 gzip\u3001gz\u3001lzo\u3001lzop\u3001lzop\u3001snappy keep\uff08\u4e0d\u66f4\u6539\u538b\u7f29\u7c7b\u578b\uff09 4.3.0+ void setStorageClass(String storageClass) \u53ef\u9009 \u8bbe\u7f6e\u76ee\u6807\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301Standard\u3001IA\u3001Archive\u3001ColdArchive Standard 4.3.0+ void setIncludeMatcherFile(String includeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setExcludeMatcherFile(String excludeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setNumWorkers(int numWorkers) \u53ef\u9009 \u8bbe\u7f6eDistCp\u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\uff0c\u5bf9\u5e94MR\u4efb\u52a1\u4e2d\u7684 mapreduce.job.maps 10 4.3.0+ void setBatchPerJob(int batchPerJob) \u53ef\u9009 \u8bbe\u7f6e\u6bcf\u4e2a distcp job \u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf 10 4.3.0+ void setBatchPerTask(int batchPerTask) \u53ef\u9009 \u8bbe\u7f6e\u6bcf\u4e2a distcp task \u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf 10 4.3.0+ void setWorkTempDir(String workTempDir) \u53ef\u9009 \u8bbe\u7f6e\u4e34\u65f6\u76ee\u5f55 /tmp 4.3.0+ void setDisableChecksum(boolean disableChecksum) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5173\u95ed checksum \u68c0\u67e5 false 4.3.0+ void setDeleteOnSuccess(boolean deleteOnSuccess) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5220\u9664\u6e90\u6587\u4ef6\uff0c\u7528\u4e8e\u79fb\u52a8\u6570\u636e false 4.3.0+ void setEnableTransaction(boolean enableTransaction) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4e8b\u52a1\uff0c\u4ee5\u4fdd\u8bc1Job\u7ea7\u522b\u7684\u539f\u5b50\u6027 false 4.3.0+ void setIgnoreFailures(boolean ignoreFailures) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5ffd\u7565\u62f7\u8d1d\u4efb\u52a1\u4e2d\u629b\u51fa\u7684\u5f02\u5e38\uff0c\u907f\u514d\u4e2d\u65ad\u4efb\u52a1 False 4.3.0+ void setEnableCMS(boolean enableCMS) \u53ef\u9009 \u662f\u5426\u5f00\u542f\u76d1\u63a7\u544a\u8b66 false 4.3.0+ void setEnablePreserveMetaStatus(boolean enablePreserveMetaStatus) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f false 4.4.0+ \u54cd\u5e94\u53c2\u6570 \u00a7 \u65b9\u6cd5 \u8bf4\u660e \u9ed8\u8ba4\u503c int getErrorCode() \u83b7\u53d6\u9519\u8bef\u7801 0 String getErrorMsg() \u83b7\u53d6\u9519\u8bef\u539f\u56e0 - long getStartTime() \u83b7\u53d6\u4efb\u52a1\u7684\u5f00\u59cb\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - long getEndTime() \u83b7\u53d6\u4efb\u52a1\u7684\u7ed3\u675f\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - getCount(String key) \u83b7\u53d6\u8ba1\u6570\u5668\u503c\uff0ckey\u4e3a\u8ba1\u6570\u5668\u540d\u79f0\uff0c - \u8ba1\u6570\u5668\u8bf4\u660e \u00a7 \u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf BYTES_EXPECTED \u9884\u671f\u7684copy\u5b57\u8282\u6570 FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 BYTES_COPIED copy\u6210\u529f\u7684\u5b57\u8282\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"\u4f7f\u7528 JindoDistcp API \u5f00\u53d1"},{"location":"jindotools/jindodistcp_api_quickstart/#jindodistcp-api","text":"","title":"JindoDistCp API \u4f7f\u7528\u8bf4\u660e"},{"location":"jindotools/jindodistcp_api_quickstart/#jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"\u4e0b\u8f7d JindoSDK \u5305"},{"location":"jindotools/jindodistcp_api_quickstart/#_1","text":"\u76ee\u524d JindoSDK/JindoDistCP \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff0cMac M1 \u7cfb\u5217\u4e5f\u6682\u4e0d\u652f\u6301\uff09 \u4ee56.7.3\u7248\u672c\u4e3a\u4f8b\uff0c\u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoDistCP \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-distcp --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop with provided scope, your runtime should install hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> </dependencies> </project>","title":"\u5b89\u88c5\u4f9d\u8d56\u5305"},{"location":"jindotools/jindodistcp_api_quickstart/#_2","text":"\u4ee5\u62f7\u8d1d HDFS \u6587\u4ef6\u5939\u5230\u963f\u91cc\u4e91 OSS \u4e3a\u4f8b\u3002 import com.aliyun.jindodata.distcp.util.DistCpCounter; import com.aliyun.jindodata.distjob.request.DistJobResponse; import org.apache.hadoop.conf.Configuration; import java.io.IOException; public class DistcpExample { static public void main(String[] args) throws IOException { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. conf.set(\"fs.oss.credentials.provider\", \"com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider\"); conf.set(\"fs.oss.accessKeyId\", \"XXX\"); conf.set(\"fs.oss.accessKeySecret\", \"XXX\"); DistCpRequest request = new DistCpRequest(); request.setSrcDir(\"hdfs:///user/root/random-data\"); request.setDestDir(\"oss://dls-bucket.cn-xxx.oss-dls.aliyuncs.com/dest\"); request.setWorkTempDir(\"hdfs:///user/root/tmp\"); request.setConf(conf); // Jindo DistCp Service DistCpService service = new DistCpService(); service.buildAllJobs(request); service.runAllJobs(); DistJobResponse response = service.collectAllJobResult(); printResponse(response); } static public void printResponse(DistJobResponse response) { System.out.println(\"ErrorCode: \" + response.getErrorCode()); System.out.println(\"ErrorMsg: \" + response.getErrorMsg()); System.out.println(\"FileCopied: \" + response.getCount(DistCpCounter.FILES_COPIED.getName())); System.out.println(\"BytesCopied: \" + response.getCount(DistCpCounter.BYTES_COPIED.getName())); } }","title":"\u793a\u4f8b\u4ee3\u7801"},{"location":"jindotools/jindodistcp_api_quickstart/#_3","text":"\u76f8\u5173\u65b9\u6cd5\u7684\u5177\u4f53\u63cf\u8ff0\u53ef\u53c2\u8003 \u300a\u4f7f\u7528Jindo DistCp\u300b \u65b9\u6cd5 \u53c2\u6570\u7c7b\u578b \u8bf4\u660e \u9ed8\u8ba4\u503c \u517c\u5bb9 void setSrc(String src) \u5fc5\u9009 \u8bbe\u7f6e\u6e90\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setDest(String dest) \u5fc5\u9009 \u8bbe\u7f6e\u76ee\u6807\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setConf(Configuration conf) \u5fc5\u9009 \u8bbe\u7f6e Configuration - 4.3.0+ void setMode(DistCpMode mode) \u53ef\u9009 \u8bbe\u7f6e DistCp \u6a21\u5f0f\uff0c\u652f\u6301\u7684\u6a21\u5f0f\u6709 COPY\uff08\u62f7\u8d1d\uff09 DIFF\uff08\u5dee\u5f02\u6bd4\u8f83\uff09 UPADTE\uff08\u589e\u91cf\u62f7\u8d1d\uff09 DistCpMode.COPY 4.3.0+ void setBandWidthLimit(int bandWidthLimit) \u53ef\u9009 \u8bbe\u7f6e\u5355\u4e2a\u8282\u70b9\u7684\u5e26\u5bbd\u9650\u5236\uff0c\u5355\u4f4dM -1 4.3.0+ void setOutputCodec(String outputCodec) \u53ef\u9009 \u8bbe\u7f6e\u538b\u7f29\u7c7b\u578b\uff0c\u652f\u6301\u7f16\u89e3\u7801\u5668\u6709 gzip\u3001gz\u3001lzo\u3001lzop\u3001lzop\u3001snappy keep\uff08\u4e0d\u66f4\u6539\u538b\u7f29\u7c7b\u578b\uff09 4.3.0+ void setStorageClass(String storageClass) \u53ef\u9009 \u8bbe\u7f6e\u76ee\u6807\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301Standard\u3001IA\u3001Archive\u3001ColdArchive Standard 4.3.0+ void setIncludeMatcherFile(String includeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setExcludeMatcherFile(String excludeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setNumWorkers(int numWorkers) \u53ef\u9009 \u8bbe\u7f6eDistCp\u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\uff0c\u5bf9\u5e94MR\u4efb\u52a1\u4e2d\u7684 mapreduce.job.maps 10 4.3.0+ void setBatchPerJob(int batchPerJob) \u53ef\u9009 \u8bbe\u7f6e\u6bcf\u4e2a distcp job \u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf 10 4.3.0+ void setBatchPerTask(int batchPerTask) \u53ef\u9009 \u8bbe\u7f6e\u6bcf\u4e2a distcp task \u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf 10 4.3.0+ void setWorkTempDir(String workTempDir) \u53ef\u9009 \u8bbe\u7f6e\u4e34\u65f6\u76ee\u5f55 /tmp 4.3.0+ void setDisableChecksum(boolean disableChecksum) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5173\u95ed checksum \u68c0\u67e5 false 4.3.0+ void setDeleteOnSuccess(boolean deleteOnSuccess) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5220\u9664\u6e90\u6587\u4ef6\uff0c\u7528\u4e8e\u79fb\u52a8\u6570\u636e false 4.3.0+ void setEnableTransaction(boolean enableTransaction) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4e8b\u52a1\uff0c\u4ee5\u4fdd\u8bc1Job\u7ea7\u522b\u7684\u539f\u5b50\u6027 false 4.3.0+ void setIgnoreFailures(boolean ignoreFailures) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5ffd\u7565\u62f7\u8d1d\u4efb\u52a1\u4e2d\u629b\u51fa\u7684\u5f02\u5e38\uff0c\u907f\u514d\u4e2d\u65ad\u4efb\u52a1 False 4.3.0+ void setEnableCMS(boolean enableCMS) \u53ef\u9009 \u662f\u5426\u5f00\u542f\u76d1\u63a7\u544a\u8b66 false 4.3.0+ void setEnablePreserveMetaStatus(boolean enablePreserveMetaStatus) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f false 4.4.0+","title":"\u8bf7\u6c42\u53c2\u6570"},{"location":"jindotools/jindodistcp_api_quickstart/#_4","text":"\u65b9\u6cd5 \u8bf4\u660e \u9ed8\u8ba4\u503c int getErrorCode() \u83b7\u53d6\u9519\u8bef\u7801 0 String getErrorMsg() \u83b7\u53d6\u9519\u8bef\u539f\u56e0 - long getStartTime() \u83b7\u53d6\u4efb\u52a1\u7684\u5f00\u59cb\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - long getEndTime() \u83b7\u53d6\u4efb\u52a1\u7684\u7ed3\u675f\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - getCount(String key) \u83b7\u53d6\u8ba1\u6570\u5668\u503c\uff0ckey\u4e3a\u8ba1\u6570\u5668\u540d\u79f0\uff0c -","title":"\u54cd\u5e94\u53c2\u6570"},{"location":"jindotools/jindodistcp_api_quickstart/#_5","text":"\u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf BYTES_EXPECTED \u9884\u671f\u7684copy\u5b57\u8282\u6570 FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 BYTES_COPIED copy\u6210\u529f\u7684\u5b57\u8282\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"\u8ba1\u6570\u5668\u8bf4\u660e"},{"location":"jindotools/jindodistcp_faq/","text":"JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u00a7 1\u3001list \u6027\u80fd\u4f4e \u00a7 \u73b0\u8c61 \u00a7 \u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u53d1\u73b0 list \u6027\u80fd\u8f83\u6162\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Successfully list objects with prefix xxx/yyy/ in bucket xxx recursive 0 result 315 dur 100036.615031MS \u5176\u4e2d dur 100036.615031MS \u4ee3\u8868\u6b64\u6b21 list \u7528\u65f6\uff0c\u5355\u4f4d\u6beb\u79d2\uff0c\u5982\u4e0a\u8ff0\u4e00\u6b21 list \u8017\u65f6 100 \u79d2\uff0cOSS \u4e0a\u6587\u4ef6\u6b63\u5e38\u7684 list \u901f\u5ea6\u662f 1000 \u6587\u4ef6\u5728 1s \u4ee5\u4e0b\uff0c\u60a8\u53ef\u6839\u636e\u5f53\u524d\u76ee\u5f55\u4e0b\u6587\u4ef6\u6570\u91cf\u6765\u5224\u65ad\u8be5 list \u8017\u65f6\u662f\u5426\u5f02\u5e38\uff0c\u5982\u4e0a\u8ff0\u4fe1\u606f\u663e\u793a list 315 \u4e2a\u6587\u4ef6\u7684\u76ee\u5f55\u9700\u8981 100s \u663e\u7136\u662f\u4e0d\u592a\u6b63\u5e38\u7684\u3002 \u89e3\u51b3\u529e\u6cd5 \u00a7 \u589e\u52a0\u5ba2\u6237\u7aef\u5185\u5b58\uff0c\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u589e\u5927\u5ba2\u6237\u7aef\u5185\u5b58 export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx4096m\" 2. checksum \u62a5\u9519 \u00a7 \u73b0\u8c61 \u00a7 \u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Failed to get checksum store. \u89e3\u51b3\u529e\u6cd5 \u00a7 OSS-HDFS \u9ed8\u8ba4\u7684 checksum \u7b97\u6cd5\u662f COMPOSITE_CRC\uff0c\u5982\u679c HDFS \u914d\u7f6e\u7684 dfs.checksum.combine.mode \u4e3a MD5MD5CRC\uff0c\u5219\u9700\u8981\u53d8\u66f4 OSS-HDFS \u7684 fs.oss.checksum.combine.mode \u914d\u7f6e\u4e3a MD5MD5CRC\u3002 hadoop jar jindo-distcp-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.checksum.combine.mode=MD5MD5CRC 3. OSS \u62f7\u8d1d\u5230 OSS-HDFS \u65f6\u62a5 verify checksum failed \u00a7 \u73b0\u8c61 \u00a7 \u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Exception raised while copying data file, verify checksum failed \u89e3\u51b3\u529e\u6cd5 \u00a7 \u5982\u679c OSS \u4e2d\u7684\u7684\u6587\u4ef6\u4e0d\u662f\u901a\u8fc7 JindoDistCp \u4ece HDFS \u8fc1\u79fb\u5230 OSS\u7684\uff0c\u5219\u9700\u8981\u901a\u8fc7 --disableChecksum \u8df3\u8fc7 checksum \u68c0\u67e5\u3002 hadoop jar jindo-distcp-${version}.jar --src oss://ossBucket/ --dest oss://dlsBucket/ --disableChecksum 4. \u6267\u884c JindoDistCp \u540e\uff0c\u67e5\u770b\u662f\u5426\u6210\u529f \u00a7 \u5982\u679c\u6ca1\u6709\u6dfb\u52a0 --ignore \u53c2\u6570\uff0c\u6267\u884c\u8fc7\u7a0b\u4e2d\u5982\u679c\u6709\u95ee\u9898\u4f1a\u62a5\u9519\u9000\u51fa\u3002\u5982\u679c\u6ca1\u6dfb\u52a0\u4e86--ignore \u53c2\u6570\uff0c\u67e5\u770b JindoDistCp Counters\uff0c\u5982 COPY_FAILED\uff0cCHECKSUM_DIFF \u7b49\uff0c\u8be6\u89c1 JindoDistCp Counters \u8bf4\u660e\u3002","title":"\u5e38\u89c1\u95ee\u9898"},{"location":"jindotools/jindodistcp_faq/#jindodistcp","text":"","title":"JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357"},{"location":"jindotools/jindodistcp_faq/#1list","text":"","title":"1\u3001list \u6027\u80fd\u4f4e"},{"location":"jindotools/jindodistcp_faq/#_1","text":"\u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u53d1\u73b0 list \u6027\u80fd\u8f83\u6162\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Successfully list objects with prefix xxx/yyy/ in bucket xxx recursive 0 result 315 dur 100036.615031MS \u5176\u4e2d dur 100036.615031MS \u4ee3\u8868\u6b64\u6b21 list \u7528\u65f6\uff0c\u5355\u4f4d\u6beb\u79d2\uff0c\u5982\u4e0a\u8ff0\u4e00\u6b21 list \u8017\u65f6 100 \u79d2\uff0cOSS \u4e0a\u6587\u4ef6\u6b63\u5e38\u7684 list \u901f\u5ea6\u662f 1000 \u6587\u4ef6\u5728 1s \u4ee5\u4e0b\uff0c\u60a8\u53ef\u6839\u636e\u5f53\u524d\u76ee\u5f55\u4e0b\u6587\u4ef6\u6570\u91cf\u6765\u5224\u65ad\u8be5 list \u8017\u65f6\u662f\u5426\u5f02\u5e38\uff0c\u5982\u4e0a\u8ff0\u4fe1\u606f\u663e\u793a list 315 \u4e2a\u6587\u4ef6\u7684\u76ee\u5f55\u9700\u8981 100s \u663e\u7136\u662f\u4e0d\u592a\u6b63\u5e38\u7684\u3002","title":"\u73b0\u8c61"},{"location":"jindotools/jindodistcp_faq/#_2","text":"\u589e\u52a0\u5ba2\u6237\u7aef\u5185\u5b58\uff0c\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u589e\u5927\u5ba2\u6237\u7aef\u5185\u5b58 export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx4096m\"","title":"\u89e3\u51b3\u529e\u6cd5"},{"location":"jindotools/jindodistcp_faq/#2-checksum","text":"","title":"2. checksum \u62a5\u9519"},{"location":"jindotools/jindodistcp_faq/#_3","text":"\u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Failed to get checksum store.","title":"\u73b0\u8c61"},{"location":"jindotools/jindodistcp_faq/#_4","text":"OSS-HDFS \u9ed8\u8ba4\u7684 checksum \u7b97\u6cd5\u662f COMPOSITE_CRC\uff0c\u5982\u679c HDFS \u914d\u7f6e\u7684 dfs.checksum.combine.mode \u4e3a MD5MD5CRC\uff0c\u5219\u9700\u8981\u53d8\u66f4 OSS-HDFS \u7684 fs.oss.checksum.combine.mode \u914d\u7f6e\u4e3a MD5MD5CRC\u3002 hadoop jar jindo-distcp-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.checksum.combine.mode=MD5MD5CRC","title":"\u89e3\u51b3\u529e\u6cd5"},{"location":"jindotools/jindodistcp_faq/#3-oss-oss-hdfs-verify-checksum-failed","text":"","title":"3. OSS \u62f7\u8d1d\u5230 OSS-HDFS \u65f6\u62a5 verify checksum failed"},{"location":"jindotools/jindodistcp_faq/#_5","text":"\u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Exception raised while copying data file, verify checksum failed","title":"\u73b0\u8c61"},{"location":"jindotools/jindodistcp_faq/#_6","text":"\u5982\u679c OSS \u4e2d\u7684\u7684\u6587\u4ef6\u4e0d\u662f\u901a\u8fc7 JindoDistCp \u4ece HDFS \u8fc1\u79fb\u5230 OSS\u7684\uff0c\u5219\u9700\u8981\u901a\u8fc7 --disableChecksum \u8df3\u8fc7 checksum \u68c0\u67e5\u3002 hadoop jar jindo-distcp-${version}.jar --src oss://ossBucket/ --dest oss://dlsBucket/ --disableChecksum","title":"\u89e3\u51b3\u529e\u6cd5"},{"location":"jindotools/jindodistcp_faq/#4-jindodistcp","text":"\u5982\u679c\u6ca1\u6709\u6dfb\u52a0 --ignore \u53c2\u6570\uff0c\u6267\u884c\u8fc7\u7a0b\u4e2d\u5982\u679c\u6709\u95ee\u9898\u4f1a\u62a5\u9519\u9000\u51fa\u3002\u5982\u679c\u6ca1\u6dfb\u52a0\u4e86--ignore \u53c2\u6570\uff0c\u67e5\u770b JindoDistCp Counters\uff0c\u5982 COPY_FAILED\uff0cCHECKSUM_DIFF \u7b49\uff0c\u8be6\u89c1 JindoDistCp Counters \u8bf4\u660e\u3002","title":"4. \u6267\u884c JindoDistCp \u540e\uff0c\u67e5\u770b\u662f\u5426\u6210\u529f"},{"location":"jindotools/jindodistcp_hdfs_to_oss/","text":"\u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS/OSS-HDFS \u00a7 \u4f7f\u7528\u524d\u987b\u77e5 \u00a7 \u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d\u3002 \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\u3002 1\u3001\u62f7\u8d1d\u6570\u636e\u5230\u963f\u91cc\u4e91 OSS/OSS-HDFS \u4e0a \u00a7 \u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06 HDFS \u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS/OSS-HDFS \u4e0a hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src HDFS \u7684\u6e90\u8def\u5f84\u3002 /data --dest OSS/OSS-HDFS \u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/ --hadoopConf \u6307\u5b9a OSS/OSS-HDFS \u7684 Access Key ID , Access Key Secret * \u914d\u7f6e OSS/OSS-HDFS \u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS/OSS-HDFS \u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10 2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6 \u00a7 \u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53eaCopy\u5269\u4e0b\u672aCopy\u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a \u4f7f\u7528 --update \u547d\u4ee4\uff0c\u83b7\u5f97\u589e\u91cf\u7684\u6587\u4ef6\u5217\u8868 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --update --parallelism 20 3\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9 \u00a7 \u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB 4\u3001\u914d\u7f6e OSS/OSS-HDFS \u7684 AK \u00a7 \u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS/OSS-HDFS AccessKey/AccessSecret \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662f JindoDistCp \u53ef\u4ee5\u5c06\u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> 5\u3001\u914d\u7f6e OSS-HDFS Endpoint \u00a7 \u8bbf\u95ee OSS-HDFS \u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS/OSS-HDFS Endpoint \u914d\u7f6e \u3002 \u6ce8\u610f \uff1a JindoSDK 4.4.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u9ed8\u8ba4\u4f7f\u7528\u57df\u540d\u5206\u79bb\u7684\u65b9\u5f0f\uff0c\u6570\u636e\u8bfb\u5199\u57df\u540d\u9ed8\u8ba4\u4f7f\u7528\u5185\u7f51\u6807\u51c6 OSS \u57df\u540d\uff0c\u5982\u679c\u5728\u975e\u963f\u91cc\u4e91\u5185\u7f51\u73af\u5883\u6267\u884c distcp \u547d\u4ee4\uff0c\u5219\u9700\u8981\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u914d\u7f6e\u6570\u636e\u6d41\u8bbf\u95ee\u7684\u516c\u7f51 OSS endpoint\u3002 <configuration> <property> <name>fs.oss.data.endpoint</name> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> 6\u3001\u5176\u4ed6\u529f\u80fd \u00a7 \u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003: * JindoDistCp \u4f7f\u7528\u8bf4\u660e","title":"\u4ece HDFS \u8fc1\u79fb\u5230 OSS \u4e0a"},{"location":"jindotools/jindodistcp_hdfs_to_oss/#hdfs-ossoss-hdfs","text":"","title":"\u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS/OSS-HDFS"},{"location":"jindotools/jindodistcp_hdfs_to_oss/#_1","text":"\u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d\u3002 \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\u3002","title":"\u4f7f\u7528\u524d\u987b\u77e5"},{"location":"jindotools/jindodistcp_hdfs_to_oss/#1-ossoss-hdfs","text":"\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06 HDFS \u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS/OSS-HDFS \u4e0a hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src HDFS \u7684\u6e90\u8def\u5f84\u3002 /data --dest OSS/OSS-HDFS \u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/ --hadoopConf \u6307\u5b9a OSS/OSS-HDFS \u7684 Access Key ID , Access Key Secret * \u914d\u7f6e OSS/OSS-HDFS \u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS/OSS-HDFS \u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10","title":"1\u3001\u62f7\u8d1d\u6570\u636e\u5230\u963f\u91cc\u4e91 OSS/OSS-HDFS \u4e0a"},{"location":"jindotools/jindodistcp_hdfs_to_oss/#2","text":"\u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53eaCopy\u5269\u4e0b\u672aCopy\u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a","title":"2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6"},{"location":"jindotools/jindodistcp_hdfs_to_oss/#-update","text":"hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --update --parallelism 20","title":"\u4f7f\u7528 --update \u547d\u4ee4\uff0c\u83b7\u5f97\u589e\u91cf\u7684\u6587\u4ef6\u5217\u8868"},{"location":"jindotools/jindodistcp_hdfs_to_oss/#3yarn","text":"\u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB","title":"3\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9"},{"location":"jindotools/jindodistcp_hdfs_to_oss/#4-ossoss-hdfs-ak","text":"\u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS/OSS-HDFS AccessKey/AccessSecret \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662f JindoDistCp \u53ef\u4ee5\u5c06\u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration>","title":"4\u3001\u914d\u7f6e OSS/OSS-HDFS \u7684 AK"},{"location":"jindotools/jindodistcp_hdfs_to_oss/#5-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS-HDFS \u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS/OSS-HDFS Endpoint \u914d\u7f6e \u3002 \u6ce8\u610f \uff1a JindoSDK 4.4.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u9ed8\u8ba4\u4f7f\u7528\u57df\u540d\u5206\u79bb\u7684\u65b9\u5f0f\uff0c\u6570\u636e\u8bfb\u5199\u57df\u540d\u9ed8\u8ba4\u4f7f\u7528\u5185\u7f51\u6807\u51c6 OSS \u57df\u540d\uff0c\u5982\u679c\u5728\u975e\u963f\u91cc\u4e91\u5185\u7f51\u73af\u5883\u6267\u884c distcp \u547d\u4ee4\uff0c\u5219\u9700\u8981\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u914d\u7f6e\u6570\u636e\u6d41\u8bbf\u95ee\u7684\u516c\u7f51 OSS endpoint\u3002 <configuration> <property> <name>fs.oss.data.endpoint</name> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration>","title":"5\u3001\u914d\u7f6e OSS-HDFS Endpoint"},{"location":"jindotools/jindodistcp_hdfs_to_oss/#6","text":"\u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003: * JindoDistCp \u4f7f\u7528\u8bf4\u660e","title":"6\u3001\u5176\u4ed6\u529f\u80fd"},{"location":"jindotools/jindodistcp_how_to_cms/","text":"JindoDistCp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66 \u00a7 JindoDistCp\uff08\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u5de5\u5177\uff09\u662f\u7528\u4e8e\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u62f7\u8d1d\u6587\u4ef6\u7684\u5de5\u5177\u3002\u4f7f\u7528MapReduce\u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3amap/reduce\u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002 CMS\uff08\u4e91\u76d1\u63a7\u670d\u52a1\uff09\uff0c\u53ef\u7528\u4e8e\u6536\u96c6\u963f\u91cc\u4e91\u8d44\u6e90\u7684\u76d1\u63a7\u6307\u6807\u6216\u7528\u6237\u81ea\u5b9a\u4e49\u7684\u76d1\u63a7\u6307\u6807\uff0c\u63a2\u6d4b\u670d\u52a1\u53ef\u7528\u6027\uff0c\u4ee5\u53ca\u9488\u5bf9\u6307\u6807\u8bbe\u7f6e\u8b66\u62a5\u3002\u4f7f\u60a8\u5168\u9762\u4e86\u89e3\u963f\u91cc\u4e91\u4e0a\u8d44\u6e90\u7684\u4f7f\u7528\u60c5\u51b5\u548c\u4e1a\u52a1\u8fd0\u884c\u72b6\u51b5\uff0c\u5e76\u53ca\u65f6\u5bf9\u6545\u969c\u8d44\u6e90\u8fdb\u884c\u5904\u7406\uff0c\u4fdd\u8bc1\u4e1a\u52a1\u6b63\u5e38\u8fd0\u884c\u3002 \u60a8\u53ef\u4ee5\u6307\u5b9a\u672c\u6b21JindoDistCp\u4efb\u52a1\u7ed3\u675f\u540e\u662f\u5426\u9700\u8981\u5f00\u542fCMS\uff0c\u4ee5\u4e0a\u62a5\u4efb\u52a1\u5931\u8d25\u4fe1\u606f\uff0c\u5e76\u4f7f\u7528CMS\u63a7\u5236\u53f0\u6765\u914d\u7f6e\u544a\u8b66\u529f\u80fd\u3002\u544a\u8b66\u6761\u4ef6\u53c2\u89c1 \u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668 \u3002 \u6b65\u9aa4 \u00a7 1. \u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u548c\u62a5\u8b66\u8054\u7cfb\u7ec4 \u00a7 \u5177\u4f53\u8bf7\u53c2\u8003\u6587\u6863 \u300a\u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u6216\u62a5\u8b66\u8054\u7cfb\u7ec4\u300b 2. \u83b7\u53d6\u62a5\u8b66Token \u00a7 \u70b9\u51fb\u5de6\u4fa7\u5bfc\u822a\u680f\u201c\u62a5\u8b66\u670d\u52a1\u201d->\"\u62a5\u8b66\u8054\u7cfb\u4eba\"->\u5207\u6362\u4e0a\u6d4b\u9875\u7b7e\u5230\"\u62a5\u8b66\u8054\u7cfb\u7ec4\"->\u9009\u62e9\u5bf9\u5e94\u7684\u62a5\u8b66\u8054\u7cfb\u7ec4->\u70b9\u51fb\u53f3\u4fa7\u63a5\u5165\u5916\u90e8\u62a5\u8b66 \u67e5\u770b \u62a5\u8b66\u670d\u52a1\u8c03\u7528\u5730\u5740 \u7684\u5185\u5bb9\uff0c\u83b7\u53d6 token= \u4e4b\u540e & \u4e4b\u524d\u7684\u5185\u5bb9\u3002 \u793a\u4f8b\uff1a\u5982\u4e0b\u53ef\u83b7\u5f97 cmsToken \u4e3a XXXX \uff1a https://metrichub-cms-cn-hangzhou.aliyuncs.com/event/notify?token=XXXX&level=CRITICAL 3. \u914d\u7f6e\u73af\u5883\u53d8\u91cf \u00a7 \u73af\u5883\u53d8\u91cf \u8bf4\u660e cmsAccessKeyId \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94AccessKey ID cmsAccessSecret \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94Access Key Secret cmsRegion \u8bbe\u7f6eCMS\u6240\u5728Region\uff0c\u9700\u8981\u4e0e\u5176\u4ed6\u76f8\u5173\u7684\u4e91\u4e0a\u8d44\u6e90\u4e00\u81f4\uff0c\u4f8b\u5982ECS\uff0cOSS\uff0cEMR\u6240\u5bf9\u5e94\u7684Region cmsLevel \u8bbe\u7f6e\u6807\u8bc6\u544a\u8b66\u901a\u77e5\u65b9\u5f0f\uff0c\u5982\u679c\u672a\u8bbe\u7f6e\uff0c\u9ed8\u8ba4\u4e3aWARN\u3002INFO\u4e3a \u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cWARN\u4e3a \u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cCRITICAL\u4e3a \u7535\u8bdd+\u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \u793a\u4f8b\u5982\u4e0b export cmsAccessKeyId=<your_key_id> export cmsAccessSecret=<your_key_secret> export cmsRegion=cn-hangzhou export cmsToken=<your_cms_token> export cmsLevel=WARN 4. \u5f00\u542fCMS\u5bf9\u5931\u8d25\u4efb\u52a1\u8fdb\u884c\u544a\u8b66 \u00a7 copy\u64cd\u4f5c \u00a7 hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://example-oss-bucket/hourly_table \\ --enableCMS diff\u64cd\u4f5c \u00a7 hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://example-oss-bucket/hourly_table \\ --diff \\ --enableCMS update\u64cd\u4f5c \u00a7 hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://example-oss-bucket/hourly_table \\ --update \\ --enableCMS \u544a\u8b66\u5185\u5bb9 \u00a7 \u544a\u8b66\u5185\u5bb9\u683c\u5f0f \u00a7 { \"JobTime\": \"2021-03-09 19:45:27~2021-03-09 19:45:28\", \"SrcPath\": \"hdfs://cluster-xxx:9000//data/incoming/hourly_table\", \"DestPath\": \"oss://example-oss-bucket/hourly_table\", \"Counter\": { \"COUNTER_1\": XXX, \"COUNTER_2\": XXX, ... } } ... \u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668 \u00a7 \u544a\u8b66\u5185\u5bb9\u4e2d\u7684counter\u8ba1\u6570\u5668\u6709\u4ee5\u4e0b\u51e0\u79cd\uff1a copy\u64cd\u4f5c \u00a7 \u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 diff\u64cd\u4f5c \u00a7 \u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"JindoDistCp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66"},{"location":"jindotools/jindodistcp_how_to_cms/#jindodistcpcms","text":"JindoDistCp\uff08\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u5de5\u5177\uff09\u662f\u7528\u4e8e\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u62f7\u8d1d\u6587\u4ef6\u7684\u5de5\u5177\u3002\u4f7f\u7528MapReduce\u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3amap/reduce\u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002 CMS\uff08\u4e91\u76d1\u63a7\u670d\u52a1\uff09\uff0c\u53ef\u7528\u4e8e\u6536\u96c6\u963f\u91cc\u4e91\u8d44\u6e90\u7684\u76d1\u63a7\u6307\u6807\u6216\u7528\u6237\u81ea\u5b9a\u4e49\u7684\u76d1\u63a7\u6307\u6807\uff0c\u63a2\u6d4b\u670d\u52a1\u53ef\u7528\u6027\uff0c\u4ee5\u53ca\u9488\u5bf9\u6307\u6807\u8bbe\u7f6e\u8b66\u62a5\u3002\u4f7f\u60a8\u5168\u9762\u4e86\u89e3\u963f\u91cc\u4e91\u4e0a\u8d44\u6e90\u7684\u4f7f\u7528\u60c5\u51b5\u548c\u4e1a\u52a1\u8fd0\u884c\u72b6\u51b5\uff0c\u5e76\u53ca\u65f6\u5bf9\u6545\u969c\u8d44\u6e90\u8fdb\u884c\u5904\u7406\uff0c\u4fdd\u8bc1\u4e1a\u52a1\u6b63\u5e38\u8fd0\u884c\u3002 \u60a8\u53ef\u4ee5\u6307\u5b9a\u672c\u6b21JindoDistCp\u4efb\u52a1\u7ed3\u675f\u540e\u662f\u5426\u9700\u8981\u5f00\u542fCMS\uff0c\u4ee5\u4e0a\u62a5\u4efb\u52a1\u5931\u8d25\u4fe1\u606f\uff0c\u5e76\u4f7f\u7528CMS\u63a7\u5236\u53f0\u6765\u914d\u7f6e\u544a\u8b66\u529f\u80fd\u3002\u544a\u8b66\u6761\u4ef6\u53c2\u89c1 \u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668 \u3002","title":"JindoDistCp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66"},{"location":"jindotools/jindodistcp_how_to_cms/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"jindotools/jindodistcp_how_to_cms/#1","text":"\u5177\u4f53\u8bf7\u53c2\u8003\u6587\u6863 \u300a\u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u6216\u62a5\u8b66\u8054\u7cfb\u7ec4\u300b","title":"1. \u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u548c\u62a5\u8b66\u8054\u7cfb\u7ec4"},{"location":"jindotools/jindodistcp_how_to_cms/#2-token","text":"\u70b9\u51fb\u5de6\u4fa7\u5bfc\u822a\u680f\u201c\u62a5\u8b66\u670d\u52a1\u201d->\"\u62a5\u8b66\u8054\u7cfb\u4eba\"->\u5207\u6362\u4e0a\u6d4b\u9875\u7b7e\u5230\"\u62a5\u8b66\u8054\u7cfb\u7ec4\"->\u9009\u62e9\u5bf9\u5e94\u7684\u62a5\u8b66\u8054\u7cfb\u7ec4->\u70b9\u51fb\u53f3\u4fa7\u63a5\u5165\u5916\u90e8\u62a5\u8b66 \u67e5\u770b \u62a5\u8b66\u670d\u52a1\u8c03\u7528\u5730\u5740 \u7684\u5185\u5bb9\uff0c\u83b7\u53d6 token= \u4e4b\u540e & \u4e4b\u524d\u7684\u5185\u5bb9\u3002 \u793a\u4f8b\uff1a\u5982\u4e0b\u53ef\u83b7\u5f97 cmsToken \u4e3a XXXX \uff1a https://metrichub-cms-cn-hangzhou.aliyuncs.com/event/notify?token=XXXX&level=CRITICAL","title":"2. \u83b7\u53d6\u62a5\u8b66Token"},{"location":"jindotools/jindodistcp_how_to_cms/#3","text":"\u73af\u5883\u53d8\u91cf \u8bf4\u660e cmsAccessKeyId \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94AccessKey ID cmsAccessSecret \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94Access Key Secret cmsRegion \u8bbe\u7f6eCMS\u6240\u5728Region\uff0c\u9700\u8981\u4e0e\u5176\u4ed6\u76f8\u5173\u7684\u4e91\u4e0a\u8d44\u6e90\u4e00\u81f4\uff0c\u4f8b\u5982ECS\uff0cOSS\uff0cEMR\u6240\u5bf9\u5e94\u7684Region cmsLevel \u8bbe\u7f6e\u6807\u8bc6\u544a\u8b66\u901a\u77e5\u65b9\u5f0f\uff0c\u5982\u679c\u672a\u8bbe\u7f6e\uff0c\u9ed8\u8ba4\u4e3aWARN\u3002INFO\u4e3a \u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cWARN\u4e3a \u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cCRITICAL\u4e3a \u7535\u8bdd+\u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \u793a\u4f8b\u5982\u4e0b export cmsAccessKeyId=<your_key_id> export cmsAccessSecret=<your_key_secret> export cmsRegion=cn-hangzhou export cmsToken=<your_cms_token> export cmsLevel=WARN","title":"3. \u914d\u7f6e\u73af\u5883\u53d8\u91cf"},{"location":"jindotools/jindodistcp_how_to_cms/#4-cms","text":"","title":"4. \u5f00\u542fCMS\u5bf9\u5931\u8d25\u4efb\u52a1\u8fdb\u884c\u544a\u8b66"},{"location":"jindotools/jindodistcp_how_to_cms/#copy","text":"hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://example-oss-bucket/hourly_table \\ --enableCMS","title":"copy\u64cd\u4f5c"},{"location":"jindotools/jindodistcp_how_to_cms/#diff","text":"hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://example-oss-bucket/hourly_table \\ --diff \\ --enableCMS","title":"diff\u64cd\u4f5c"},{"location":"jindotools/jindodistcp_how_to_cms/#update","text":"hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://example-oss-bucket/hourly_table \\ --update \\ --enableCMS","title":"update\u64cd\u4f5c"},{"location":"jindotools/jindodistcp_how_to_cms/#_2","text":"","title":"\u544a\u8b66\u5185\u5bb9"},{"location":"jindotools/jindodistcp_how_to_cms/#_3","text":"{ \"JobTime\": \"2021-03-09 19:45:27~2021-03-09 19:45:28\", \"SrcPath\": \"hdfs://cluster-xxx:9000//data/incoming/hourly_table\", \"DestPath\": \"oss://example-oss-bucket/hourly_table\", \"Counter\": { \"COUNTER_1\": XXX, \"COUNTER_2\": XXX, ... } } ...","title":"\u544a\u8b66\u5185\u5bb9\u683c\u5f0f"},{"location":"jindotools/jindodistcp_how_to_cms/#_4","text":"\u544a\u8b66\u5185\u5bb9\u4e2d\u7684counter\u8ba1\u6570\u5668\u6709\u4ee5\u4e0b\u51e0\u79cd\uff1a","title":"\u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668"},{"location":"jindotools/jindodistcp_how_to_cms/#copy_1","text":"\u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570","title":"copy\u64cd\u4f5c"},{"location":"jindotools/jindodistcp_how_to_cms/#diff_1","text":"\u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"diff\u64cd\u4f5c"},{"location":"jindotools/jindodistcp_quickstart/","text":"JindoDistCp \u4f7f\u7528\u8bf4\u660e \u00a7 JindoDistCp \u4ecb\u7ecd \u00a7 JindoDistCp \u662f\u963f\u91cc\u4e91\u6570\u636e\u6e56\u5b58\u50a8\u56e2\u961f\u5f00\u53d1\u7684\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u7684\u5de5\u5177\u3002\u5176\u4f7f\u7528 MapReduce \u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3a MapReduce \u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002\u76ee\u524d\u5168\u91cf\u652f\u6301 HDFS/OSS-HDFS/OSS/S3 \u4e4b\u95f4\u7684\u7684\u6570\u636e\u62f7\u8d1d\u573a\u666f\uff0c\u63d0\u4f9b\u591a\u79cd\u4e2a\u6027\u5316\u62f7\u8d1d\u53c2\u6570\u548c\u591a\u79cd\u62f7\u8d1d\u7b56\u7565\u3002\u91cd\u70b9\u4f18\u5316 HDFS \u5230 OSS-HDFS \u7684\u6570\u636e\u62f7\u8d1d\uff0c\u901a\u8fc7\u5b9a\u5236\u5316 CopyCommitter\uff0c\u5b9e\u73b0 No-Rename \u62f7\u8d1d\uff0c\u5e76\u4fdd\u8bc1\u6570\u636e\u62f7\u8d1d\u843d\u5730\u7684\u4e00\u81f4\u6027\u3002\u529f\u80fd\u5168\u91cf\u5bf9\u9f50 S3 DistCp \u548c HDFS DistCp\uff0c\u6027\u80fd\u8f83 HDFS DistCp \u6709\u8f83\u5927\u63d0\u5347\uff0c\u81f4\u529b\u4e8e\u63d0\u4f9b\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177\u3002 \u73af\u5883\u8981\u6c42 \u00a7 JDK 1.8\u53ca\u4ee5\u4e0a Hadoop 2.3+ \u7248\u672c\uff0c\u8bf7\u4e0b\u8f7d\u6700\u65b0\u7248\u7684 jindo-distcp-tool-x.x.x.jar\uff0c\u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz \u5185\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 tools/ \u76ee\u5f55\u4e0b\u627e\u5230\uff0c \u4e0b\u8f7d\u9875\u9762 \u3002 \u6ce8\uff1aE-MapReduce EMR-5.6.0/EMR-3.40.0 \u6216\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u5728\u96c6\u7fa4\u4e2d\u5df2\u90e8\u7f72 JindoDistCp\uff0c\u53ef\u5728 /opt/apps/JINDOSDK/jindosdk-current/tools \u76ee\u5f55\u4e0b\u627e\u5230 jindo-distcp-tool-x.x.x.jar\u3002 \u53c2\u6570\u8bf4\u660e \u00a7 JindoDistCp \u63d0\u4f9b jar \u5305\u5f62\u5f0f\u4f7f\u7528\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 hadoop jar \u547d\u4ee4\u914d\u5408\u4e00\u7cfb\u5217\u53c2\u6570\u6765\u5b8c\u6210\u8fc1\u79fb\u64cd\u4f5c\u3002 \u53c2\u6570 \u53c2\u6570\u7c7b\u578b \u8bf4\u660e \u9ed8\u8ba4\u503c \u7248\u672c OSS OSS-HDFS --src \u5fc5\u9009 \u8bbe\u7f6e\u6e90\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ \u652f\u6301 \u652f\u6301 --dest \u5fc5\u9009 \u8bbe\u7f6e\u76ee\u6807\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ \u652f\u6301 \u652f\u6301 --bandWidth \u53ef\u9009 \u8bbe\u7f6e\u5355\u4e2a\u8282\u70b9\u7684\u5e26\u5bbd\u9650\u5236\uff0c\u5355\u4f4d M -1 4.3.0+ \u652f\u6301 \u652f\u6301 --codec \u53ef\u9009 \u8bbe\u7f6e\u538b\u7f29\u7c7b\u578b\uff0c\u652f\u6301\u7f16\u89e3\u7801\u5668\u6709 gzip\u3001gz\u3001lzo\u3001lzop\u3001lzop\u3001snappy keep\uff08\u4e0d\u66f4\u6539\u538b\u7f29\u7c7b\u578b\uff09 4.3.0+ \u652f\u6301 \u652f\u6301 --policy \u53ef\u9009 \u8bbe\u7f6e\u76ee\u6807\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301 Standard\u3001IA\u3001Archive\u3001ColdArchive Standard 4.3.0+ \u652f\u6301 \u4e0d\u652f\u6301 --filters \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ \u652f\u6301 \u652f\u6301 --srcPrefixesFile \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ \u652f\u6301 \u652f\u6301 --parallelism \u53ef\u9009 \u8bbe\u7f6eDistCp\u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\uff0c\u5bf9\u5e94MR\u4efb\u52a1\u4e2d\u7684 mapreduce.job.maps 10 4.3.0+ \u652f\u6301 \u652f\u6301 --jobBatch \u53ef\u9009 \u8bbe\u7f6e\u6bcf\u4e2a distcp job \u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf 10000 4.5.1+ \u652f\u6301 \u652f\u6301 --taskBatch \u53ef\u9009 \u8bbe\u7f6e\u6bcf\u4e2a distcp task \u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf 1 4.3.0+ \u652f\u6301 \u652f\u6301 --tmp \u53ef\u9009 \u8bbe\u7f6e\u4e34\u65f6\u76ee\u5f55 /tmp 4.3.0+ \u652f\u6301 \u652f\u6301 --hadoopConf \u53ef\u9009 \u8bbe\u7f6e Configuration - 4.3.0+ \u652f\u6301 \u652f\u6301 --disableChecksum \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5173\u95ed Checksum \u68c0\u67e5 false 4.3.0+ \u652f\u6301 \u652f\u6301 --deleteOnSuccess \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5220\u9664\u6e90\u6587\u4ef6\uff0c\u7528\u4e8e\u79fb\u52a8\u6570\u636e false 4.3.0+ \u652f\u6301 \u652f\u6301 --enableTransaction \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4e8b\u52a1\uff0c\u4ee5\u4fdd\u8bc1 Job \u7ea7\u522b\u7684\u539f\u5b50\u6027 false 4.3.0+ \u652f\u6301 \u652f\u6301 --ignore \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5ffd\u7565\u62f7\u8d1d\u4efb\u52a1\u4e2d\u629b\u51fa\u7684\u5f02\u5e38\uff0c\u907f\u514d\u4e2d\u65ad\u4efb\u52a1 false 4.3.0+ \u652f\u6301 \u652f\u6301 --enableCMS \u53ef\u9009 \u662f\u5426\u5f00\u542f\u76d1\u63a7\u544a\u8b66 false 4.5.1+ \u652f\u6301 \u652f\u6301 --diff \u53ef\u9009 \u8bbe\u7f6e DistCp \u6a21\u5f0f\u4e3a DIFF, \u67e5\u770b src \u548c dest \u7684\u6587\u4ef6\u5dee\u5f02 DistCpMode.COPY 4.3.0+ \u652f\u6301 \u652f\u6301 --update \u53ef\u9009 \u8bbe\u7f6e DistCp \u6a21\u5f0f\u4e3a UPDATE, \u6307\u5b9a\u589e\u91cf\u540c\u6b65\u529f\u80fd\uff0c\u8df3\u8fc7\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u548c\u76ee\u5f55\uff0c\u76f4\u63a5\u5c06 src \u4e2d\u65b0\u589e\u6216\u53d1\u751f\u6539\u53d8\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u540c\u6b65\u5230 dest \u4e0a DistCpMode.COPY 4.3.0+ \u652f\u6301 \u652f\u6301 --preserveMeta \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f false 4.4.0+ \u4e0d\u652f\u6301 \u652f\u6301 1\u3001\u4f7f\u7528 --src \u548c --dest (\u5fc5\u9009) \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --src \u8868\u793a\u6307\u5b9a\u6e90\u6587\u4ef6\u7684\u8def\u5f84\u3002 --dest \u8868\u793a\u76ee\u6807\u6587\u4ef6\u7684\u8def\u5f84\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table \u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9a dest \u8def\u5f84\u6765\u786e\u5b9a\u62f7\u8d1d\u540e\u7684\u6587\u4ef6\u5c42\u6b21\uff0c\u5982\u60a8\u9700\u8981\u5c06 /data/hourly_table \u4e0b\u7684\u6587\u4ef6\u62f7\u8d1d\u5230 example-oss-bucket \u8fd9\u4e2a bucket \u4e0b\u7684 hourly_table \u76ee\u5f55\u4e0b\uff0c\u5219\u53ef\u4ee5\u4f7f\u7528\u4e0a\u8ff0\u8bed\u53e5\u6765\u5b8c\u6210\u3002\u6b64\u5904\u548c Hadoop \u7684 DistCp \u884c\u4e3a\u6709\u6240\u4e0d\u540c\uff0cJindoDistCp \u4f1a\u9ed8\u8ba4\u5c06 src \u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6587\u4ef6\u62f7\u8d1d\u5230\u60a8\u6307\u5b9a dest \u8def\u5f84\u4e0b\uff0c\u5e76\u4e0d\u5305\u62ec\u5f53\u524d\u7684\u6839\u76ee\u5f55\u540d\u79f0\uff0c\u60a8\u53ef\u4ee5\u5728 dest \u4e2d\u6307\u5b9a\u62f7\u8d1d\u8def\u5f84\u7684\u6839\u76ee\u5f55\uff0c\u5982\u679c\u4e0d\u5b58\u5728\u4f1a\u81ea\u52a8\u521b\u5efa\u3002 \u6ce8\u610f\uff1a\u5982\u9700\u62f7\u8d1d\u5355\u4e2a\u6587\u4ef6, dest \u9700\u6307\u5b9a\u4e3a\u76ee\u5f55\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /test.txt --dest oss://example-oss-bucket/tmp 2\u3001\u4f7f\u7528 --bandWidth \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --bandWidth \u8868\u793a\u672c\u6b21distcp\u4efb\u52a1\u6240\u7528\u7684\u5355\u673a\u5e26\u5bbd(\u4ee5MB\u4e3a\u5355\u4f4d)\uff0c\u907f\u514d\u5355\u673a\u5360\u7528\u8fc7\u5927\u5e26\u5bbd\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --bandWidth 6 3\u3001\u4f7f\u7528 --codec \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 \u539f\u59cb\u6587\u4ef6\u901a\u5e38\u4ee5\u672a\u538b\u7f29\u7684\u6587\u672c\u683c\u5f0f\u8fdb\u5165 OSS/OSS-HDFS\u3002\u65e0\u8bba\u662f\u5b58\u50a8\u6210\u672c\u8fd8\u662f\u5bf9\u8be5\u6570\u636e\u8fdb\u884c\u5206\u6790\uff0c\u6b64\u683c\u5f0f\u90fd\u4e0d\u751a\u7406\u60f3\u3002JindoDistCp \u53ef\u4ee5\u4f7f\u7528 --codec \u9009\u9879\u5e2e\u52a9\u60a8\u5728\u7ebf\u9ad8\u6548\u5730\u5b58\u50a8\u6570\u636e\u548c\u538b\u7f29\u6587\u4ef6\u3002 --codec \u6307\u5b9a\u6587\u4ef6\u538b\u7f29\u7f16\u89e3\u7801\u5668\u3002 \u652f\u6301 gzip\u3001gz\u3001lzo\u3001lzop \u548c snappy \u5173\u952e\u5b57\u3002 none \u4fdd\u5b58\u4e3a\u672a\u538b\u7f29\u7684\u6587\u4ef6\u3002\u5982\u679c\u6587\u4ef6\u5df2\u538b\u7f29\uff0c\u5219 JindoDistCp \u4f1a\u5c06\u5176\u89e3\u538b\u7f29\u3002 keep(\u9ed8\u8ba4\u503c) \u4e0d\u66f4\u6539\u6587\u4ef6\u538b\u7f29\u5f62\u6001\uff0c\u6309\u539f\u6837\u590d\u5236\u3002 \u547d\u4ee4\u793a\u4f8b\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --codec gz \u6211\u4eec\u68c0\u67e5\u4e00\u4e0b\u76ee\u6807\u6587\u4ef6\u5939\u4e2d\u7684\u6587\u4ef6\uff0c\u8fd9\u4e9b\u6587\u4ef6\u73b0\u5728\u5df2\u7ecf\u7528 gz \u7f16\u89e3\u7801\u5668\u538b\u7f29\u4e86\uff1a [root@emr-header-1 opt]# hdfs dfs -ls oss://example-oss-bucket/hourly_table/2017-02-01/03 Found 6 items -rw-rw-rw- 1 938 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/000151.sst.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/1.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/2.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/OPTIONS-000109.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/emp01.txt.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/emp06.txt.gz \u5982\u60a8\u5728\u5f00\u6e90Hadoop\u96c6\u7fa4\u73af\u5883\u4e2d\u4f7f\u7528lzo\u538b\u7f29\u529f\u80fd\uff0c\u5219\u60a8\u9700\u8981\u5b89\u88c5 gplcompression \u7684 native \u5e93\u548chadoop-lzo\u5305\uff0c\u5982\u60a8\u7f3a\u5c11\u76f8\u5173\u73af\u5883\uff0c\u5efa\u8bae\u4f7f\u7528\u5176\u4ed6\u538b\u7f29\u65b9\u5f0f\u8fdb\u884c\u538b\u7f29\u3002 4\u3001\u4f7f\u7528 --filters \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --filters \u6307\u5b9a\u6709\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --filters filter.txt \u5047\u8bbe filter.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u4e0d\u4f1a\u5c06\u8def\u5f84\u4e2d\u5e26\u6709 test \u5b57\u7b26\u7684\u6587\u4ef6\u62f7\u8d1d\u5230OSS\u3002 5\u3001\u4f7f\u7528 --srcPrefixesFile \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --srcPrefixesFile \u6307\u5b9a\u6709\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --srcPrefixesFile prefixes.txt \u5047\u8bbe prefixes.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u53ea\u6709\u8def\u5f84\u4e2d\u5e26\u6709 test \u5b57\u7b26\u7684\u6587\u4ef6\u624d\u4f1a\u62f7\u8d1d\u5230OSS\u3002 6\u3001\u4f7f\u7528 --parallelism \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --parallelism \u6307\u5b9aMR\u4efb\u52a1\u91cc\u7684 mapreduce.job.reduces \u53c2\u6570\uff0c\u8be5\u53c2\u6570\u5728EMR\u73af\u5883\u4e2d\u9ed8\u8ba4\u4e3a7\uff0c\u60a8\u53ef\u4ee5\u6839\u636e\u96c6\u7fa4\u7684\u8d44\u6e90\u60c5\u51b5\u81ea\u5b9a\u4e49 parallelism \u7684\u5927\u5c0f\u6765\u63a7\u5236 distcp \u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://example-oss-bucket/tmp --parallelism 20 7\u3001\u4f7f\u7528 --taskBatch \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --taskBatch \u6307\u5b9a\u6bcf\u4e2a distcp task \u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --taskBatch 10 8\u3001\u4f7f\u7528 --tmp \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --tmp \u6307\u5b9a\u4e00\u4e2a\u4e34\u65f6\u76ee\u5f55\u5728hdfs\u4e0a\u4f20\u5b58\u653e\u4e34\u65f6\u6570\u636e\uff0c\u9ed8\u8ba4\u503c\u4e3a/tmp\uff0c\u5373hdfs:///tmp/\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --tmp /tmp 9\u3001\u914d\u7f6e\u8bbf\u95ee OSS/OSS-HDFS \u7684 AK \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 \u5728 EMR \u5916\u6216\u514d\u5bc6\u670d\u52a1\u51fa\u73b0\u95ee\u9898\u7684\u60c5\u51b5\u4e0b\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9aAK\u6765\u83b7\u5f97\u8bbf\u95ee OSS/OSS-HDFS \u670d\u52a1\u7684\u6743\u9650\u3002\u60a8\u53ef\u4ee5\u5728\u547d\u4ee4\u4e2d\u4f7f\u7528 --hadoopConf \u9009\u9879\u6765\u6307\u5b9a AK\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret \u60a8\u4e5f\u53ef\u4ee5\u5c06 OSS/OSS-HDFS \u670d\u52a1\u7684 ak\u3001secret \u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u907f\u514d\u6bcf\u6b21\u4f7f\u7528\u65f6\u4e34\u65f6\u586b\u5199 ak\u3002\u60a8\u53ef\u4ee5\u5728 E-MapReduce \u63a7\u5236\u53f0 Hadoop-Common \u670d\u52a1\u7684 core-site.xml \u9875\u9762\u914d\u7f6e\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> 10\u3001\u4f7f\u7528 --disableChecksum \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --disableChecksum \u5173\u95ed\u68c0\u67e5\u6587\u4ef6 checksum\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --disableChecksum 11\u3001\u4f7f\u7528 --deleteOnSuccess \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --deleteOnSuccess \u6307\u5b9a\u79fb\u52a8\u6570\u636e\u800c\u4e0d\u662f\u590d\u5236\u6570\u636e\u3002\u6b64\u9009\u9879\u7c7b\u4f3c\u4e8e mv \u64cd\u4f5c\uff0c\u9996\u5148\u590d\u5236\u6587\u4ef6\uff0c\u7136\u540e\u4ece\u6e90\u4f4d\u7f6e\u5220\u9664\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --deleteOnSuccess 12\u3001\u4f7f\u7528 --enableTransaction \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u4e0d\u652f\u6301 \u652f\u6301 --enableTransaction JindoDistCp \u9ed8\u8ba4\u4f7f\u7528 Task \u7ea7\u522b\u5b8c\u6574\u6027\uff0c\u5982\u60a8\u9700\u8981\u4fdd\u8bc1 Job \u7ea7\u522b\u7684\u5b8c\u6574\u6027\u4ee5\u53ca\u4fdd\u8bc1 Job \u4e4b\u95f4\u7684\u4e8b\u52a1\u652f\u6301\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528\u8be5\u53c2\u6570\u6307\u5b9a\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --enableTransaction 13\u3001\u4f7f\u7528 --ignore \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --ignore \u5ffd\u7565\u6570\u636e\u8fc1\u79fb\u671f\u95f4\u53d1\u751f\u7684\u5f02\u5e38\uff0c\u76f8\u5173\u62a5\u9519\u4e0d\u4f1a\u4e2d\u65ad\u4efb\u52a1\uff0c\u5e76\u6700\u7ec8\u4ee5 JindoCounter \u7684\u5f62\u5f0f\u900f\u51fa\u3002\uff08\u5982\u679c\u5f00\u542fCMS\u7684\u8bdd\uff0c\u4e5f\u4f1a\u4ee5\u6307\u5b9a\u65b9\u5f0f\u8fdb\u884c\u901a\u77e5\u3002\uff09 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --ignore 14\u3001\u4f7f\u7528 --diff \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --diff \u67e5\u770b src \u548c dest \u7684\u6587\u4ef6\u5dee\u5f02\u3002\u5982\u679c src \u7684\u6587\u4ef6\u672a\u80fd\u540c\u6b65\u5230 dest \u4e0a\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\u3002\u5982\u60a8\u7684 JindoDistCp \u4efb\u52a1\u5305\u542b\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u5219 --diff \u4e0d\u80fd\u663e\u793a\u6b63\u786e\u7684\u6587\u4ef6\u5dee\u5f02\uff0c\u56e0\u4e3a\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u4f1a\u6539\u53d8\u6587\u4ef6\u7684\u5927\u5c0f\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --diff \u5982\u679c\u6587\u4ef6\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter DIFF_FILES=1 \u5982\u679c\u60a8\u7684--dest\u4e3aHDFS\u8def\u5f84\uff0c\u73b0\u652f\u6301 /path \uff0c hdfs://hostname:ip/path \uff0c hdfs://headerIp:ip/path \u7684\u5199\u6cd5\uff0c\u6682\u4e0d\u652f\u6301 hdfs:///path \uff0c hdfs:/path \u548c\u5176\u4ed6\u81ea\u5b9a\u4e49\u5199\u6cd5\u3002 \u5982\u679c\u60f3\u67e5\u770b\u6587\u4ef6\u5143\u6570\u636e\u7684\u5dee\u5f02\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 --diff --preserveMeta \u547d\u4ee4\uff1a \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --diff --preserveMeta \u5982\u679c\u6587\u4ef6\u5143\u6570\u636e\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5143\u6570\u636e\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter META_FILE_DIFF=1 15\u3001\u4f7f\u7528 --update \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --update \u6307\u5b9a\u589e\u91cf\u540c\u6b65\u529f\u80fd\uff0c\u8df3\u8fc7\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u548c\u76ee\u5f55\uff0c\u76f4\u63a5\u5c06 src \u4e2d\u65b0\u589e\u6216\u53d1\u751f\u6539\u53d8\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u540c\u6b65\u5230 dest \u4e0a\u3002 \u5982\u679c JindoDistCp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53ea Copy \u5269\u4e0b\u672a Copy \u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 JindoDistCp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u6307\u5b9a\u8be5\u53c2\u6570\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --update 16\u3001\u4f7f\u7528\u51b7\u5f52\u6863/\u5f52\u6863/\u4f4e\u9891\u5199\u5165OSS \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u4e0d\u652f\u6301 --update \u6307\u5b9a\u4ee5\u51b7\u5f52\u6863\u3001\u5f52\u6863\u548c\u4f4e\u9891\u7684\u6a21\u5f0f\u5199\u5165 OSS\uff0c\u8fdb\u884c\u6570\u636e\u5b58\u50a8\u3002 1\uff09\u4f7f\u7528\u51b7\u5f52\u6863\u5199\u5165OSS\uff08coldArchive\uff09 \u76ee\u524d\u53ea\u5728\u90e8\u5206region\u53ef\u7528\uff0c\u5177\u4f53\u53c2\u89c1 OSS\u5b58\u50a8\u7c7b\u578b\u4ecb\u7ecd \uff0c\u4f7f\u7528\u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-bucket/hourly_table --policy coldArchive --parallelism 20 2\uff09\u4f7f\u7528\u5f52\u6863\u5199\u5165OSS\uff08archive\uff09 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-bucket/hourly_table --policy archive --parallelism 20 3\uff09\u4f7f\u7528\u4f4e\u9891\u5199\u5165OSS\uff08ia\uff09 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-bucket/hourly_table --policy ia --parallelism 20 \u5982\u60a8\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4\u4ee5 standard \u6807\u51c6\u6a21\u5f0f\u5199\u5165\uff0c\u4e0d\u8fdb\u884c\u51b7\u5f52\u6863\u3001\u5f52\u6863\u548c\u4f4e\u9891\u64cd\u4f5c\u3002 17\u3001\u4f7f\u7528 --preserveMeta \u00a7 \u7248\u672c OSS OSS-HDFS 4.4.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u4e0d\u652f\u6301 \u652f\u6301 --preserveMeta \u6307\u5b9a\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --preserveMeta 18\u3001\u4f7f\u7528 --jobBatch \u00a7 \u7248\u672c OSS OSS-HDFS 4.5.1 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --jobBatch \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--jobBatch\u6765\u6307\u5b9a\u6bcf\u4e2adistcp job\u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10000\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --jobBatch 50000 19\u3001 \u4f7f\u7528 --enableCMS \u00a7 \u7248\u672c OSS OSS-HDFS 4.5.1 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --enableCMS \u5f00\u542f\u4e91\u76d1\u63a7\u544a\u8b66\u529f\u80fd\uff0c\u5177\u4f53\u53c2\u89c1 \u300aJindoDistcp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66\u300b \u3002 JindoDistCp Counters \u8bf4\u660e \u00a7 \u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf BYTES_EXPECTED \u9884\u671f\u7684copy\u5b57\u8282\u6570 FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 BYTES_COPIED copy\u6210\u529f\u7684\u5b57\u8282\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"JindoDistcp \u5feb\u901f\u5165\u95e8"},{"location":"jindotools/jindodistcp_quickstart/#jindodistcp","text":"","title":"JindoDistCp \u4f7f\u7528\u8bf4\u660e"},{"location":"jindotools/jindodistcp_quickstart/#jindodistcp_1","text":"JindoDistCp \u662f\u963f\u91cc\u4e91\u6570\u636e\u6e56\u5b58\u50a8\u56e2\u961f\u5f00\u53d1\u7684\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u7684\u5de5\u5177\u3002\u5176\u4f7f\u7528 MapReduce \u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3a MapReduce \u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002\u76ee\u524d\u5168\u91cf\u652f\u6301 HDFS/OSS-HDFS/OSS/S3 \u4e4b\u95f4\u7684\u7684\u6570\u636e\u62f7\u8d1d\u573a\u666f\uff0c\u63d0\u4f9b\u591a\u79cd\u4e2a\u6027\u5316\u62f7\u8d1d\u53c2\u6570\u548c\u591a\u79cd\u62f7\u8d1d\u7b56\u7565\u3002\u91cd\u70b9\u4f18\u5316 HDFS \u5230 OSS-HDFS \u7684\u6570\u636e\u62f7\u8d1d\uff0c\u901a\u8fc7\u5b9a\u5236\u5316 CopyCommitter\uff0c\u5b9e\u73b0 No-Rename \u62f7\u8d1d\uff0c\u5e76\u4fdd\u8bc1\u6570\u636e\u62f7\u8d1d\u843d\u5730\u7684\u4e00\u81f4\u6027\u3002\u529f\u80fd\u5168\u91cf\u5bf9\u9f50 S3 DistCp \u548c HDFS DistCp\uff0c\u6027\u80fd\u8f83 HDFS DistCp \u6709\u8f83\u5927\u63d0\u5347\uff0c\u81f4\u529b\u4e8e\u63d0\u4f9b\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177\u3002","title":"JindoDistCp \u4ecb\u7ecd"},{"location":"jindotools/jindodistcp_quickstart/#_1","text":"JDK 1.8\u53ca\u4ee5\u4e0a Hadoop 2.3+ \u7248\u672c\uff0c\u8bf7\u4e0b\u8f7d\u6700\u65b0\u7248\u7684 jindo-distcp-tool-x.x.x.jar\uff0c\u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz \u5185\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 tools/ \u76ee\u5f55\u4e0b\u627e\u5230\uff0c \u4e0b\u8f7d\u9875\u9762 \u3002 \u6ce8\uff1aE-MapReduce EMR-5.6.0/EMR-3.40.0 \u6216\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u5728\u96c6\u7fa4\u4e2d\u5df2\u90e8\u7f72 JindoDistCp\uff0c\u53ef\u5728 /opt/apps/JINDOSDK/jindosdk-current/tools \u76ee\u5f55\u4e0b\u627e\u5230 jindo-distcp-tool-x.x.x.jar\u3002","title":"\u73af\u5883\u8981\u6c42"},{"location":"jindotools/jindodistcp_quickstart/#_2","text":"JindoDistCp \u63d0\u4f9b jar \u5305\u5f62\u5f0f\u4f7f\u7528\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 hadoop jar \u547d\u4ee4\u914d\u5408\u4e00\u7cfb\u5217\u53c2\u6570\u6765\u5b8c\u6210\u8fc1\u79fb\u64cd\u4f5c\u3002 \u53c2\u6570 \u53c2\u6570\u7c7b\u578b \u8bf4\u660e \u9ed8\u8ba4\u503c \u7248\u672c OSS OSS-HDFS --src \u5fc5\u9009 \u8bbe\u7f6e\u6e90\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ \u652f\u6301 \u652f\u6301 --dest \u5fc5\u9009 \u8bbe\u7f6e\u76ee\u6807\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ \u652f\u6301 \u652f\u6301 --bandWidth \u53ef\u9009 \u8bbe\u7f6e\u5355\u4e2a\u8282\u70b9\u7684\u5e26\u5bbd\u9650\u5236\uff0c\u5355\u4f4d M -1 4.3.0+ \u652f\u6301 \u652f\u6301 --codec \u53ef\u9009 \u8bbe\u7f6e\u538b\u7f29\u7c7b\u578b\uff0c\u652f\u6301\u7f16\u89e3\u7801\u5668\u6709 gzip\u3001gz\u3001lzo\u3001lzop\u3001lzop\u3001snappy keep\uff08\u4e0d\u66f4\u6539\u538b\u7f29\u7c7b\u578b\uff09 4.3.0+ \u652f\u6301 \u652f\u6301 --policy \u53ef\u9009 \u8bbe\u7f6e\u76ee\u6807\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301 Standard\u3001IA\u3001Archive\u3001ColdArchive Standard 4.3.0+ \u652f\u6301 \u4e0d\u652f\u6301 --filters \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ \u652f\u6301 \u652f\u6301 --srcPrefixesFile \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ \u652f\u6301 \u652f\u6301 --parallelism \u53ef\u9009 \u8bbe\u7f6eDistCp\u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\uff0c\u5bf9\u5e94MR\u4efb\u52a1\u4e2d\u7684 mapreduce.job.maps 10 4.3.0+ \u652f\u6301 \u652f\u6301 --jobBatch \u53ef\u9009 \u8bbe\u7f6e\u6bcf\u4e2a distcp job \u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf 10000 4.5.1+ \u652f\u6301 \u652f\u6301 --taskBatch \u53ef\u9009 \u8bbe\u7f6e\u6bcf\u4e2a distcp task \u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf 1 4.3.0+ \u652f\u6301 \u652f\u6301 --tmp \u53ef\u9009 \u8bbe\u7f6e\u4e34\u65f6\u76ee\u5f55 /tmp 4.3.0+ \u652f\u6301 \u652f\u6301 --hadoopConf \u53ef\u9009 \u8bbe\u7f6e Configuration - 4.3.0+ \u652f\u6301 \u652f\u6301 --disableChecksum \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5173\u95ed Checksum \u68c0\u67e5 false 4.3.0+ \u652f\u6301 \u652f\u6301 --deleteOnSuccess \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5220\u9664\u6e90\u6587\u4ef6\uff0c\u7528\u4e8e\u79fb\u52a8\u6570\u636e false 4.3.0+ \u652f\u6301 \u652f\u6301 --enableTransaction \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4e8b\u52a1\uff0c\u4ee5\u4fdd\u8bc1 Job \u7ea7\u522b\u7684\u539f\u5b50\u6027 false 4.3.0+ \u652f\u6301 \u652f\u6301 --ignore \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5ffd\u7565\u62f7\u8d1d\u4efb\u52a1\u4e2d\u629b\u51fa\u7684\u5f02\u5e38\uff0c\u907f\u514d\u4e2d\u65ad\u4efb\u52a1 false 4.3.0+ \u652f\u6301 \u652f\u6301 --enableCMS \u53ef\u9009 \u662f\u5426\u5f00\u542f\u76d1\u63a7\u544a\u8b66 false 4.5.1+ \u652f\u6301 \u652f\u6301 --diff \u53ef\u9009 \u8bbe\u7f6e DistCp \u6a21\u5f0f\u4e3a DIFF, \u67e5\u770b src \u548c dest \u7684\u6587\u4ef6\u5dee\u5f02 DistCpMode.COPY 4.3.0+ \u652f\u6301 \u652f\u6301 --update \u53ef\u9009 \u8bbe\u7f6e DistCp \u6a21\u5f0f\u4e3a UPDATE, \u6307\u5b9a\u589e\u91cf\u540c\u6b65\u529f\u80fd\uff0c\u8df3\u8fc7\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u548c\u76ee\u5f55\uff0c\u76f4\u63a5\u5c06 src \u4e2d\u65b0\u589e\u6216\u53d1\u751f\u6539\u53d8\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u540c\u6b65\u5230 dest \u4e0a DistCpMode.COPY 4.3.0+ \u652f\u6301 \u652f\u6301 --preserveMeta \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f false 4.4.0+ \u4e0d\u652f\u6301 \u652f\u6301","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"jindotools/jindodistcp_quickstart/#1-src-dest","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --src \u8868\u793a\u6307\u5b9a\u6e90\u6587\u4ef6\u7684\u8def\u5f84\u3002 --dest \u8868\u793a\u76ee\u6807\u6587\u4ef6\u7684\u8def\u5f84\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table \u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9a dest \u8def\u5f84\u6765\u786e\u5b9a\u62f7\u8d1d\u540e\u7684\u6587\u4ef6\u5c42\u6b21\uff0c\u5982\u60a8\u9700\u8981\u5c06 /data/hourly_table \u4e0b\u7684\u6587\u4ef6\u62f7\u8d1d\u5230 example-oss-bucket \u8fd9\u4e2a bucket \u4e0b\u7684 hourly_table \u76ee\u5f55\u4e0b\uff0c\u5219\u53ef\u4ee5\u4f7f\u7528\u4e0a\u8ff0\u8bed\u53e5\u6765\u5b8c\u6210\u3002\u6b64\u5904\u548c Hadoop \u7684 DistCp \u884c\u4e3a\u6709\u6240\u4e0d\u540c\uff0cJindoDistCp \u4f1a\u9ed8\u8ba4\u5c06 src \u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6587\u4ef6\u62f7\u8d1d\u5230\u60a8\u6307\u5b9a dest \u8def\u5f84\u4e0b\uff0c\u5e76\u4e0d\u5305\u62ec\u5f53\u524d\u7684\u6839\u76ee\u5f55\u540d\u79f0\uff0c\u60a8\u53ef\u4ee5\u5728 dest \u4e2d\u6307\u5b9a\u62f7\u8d1d\u8def\u5f84\u7684\u6839\u76ee\u5f55\uff0c\u5982\u679c\u4e0d\u5b58\u5728\u4f1a\u81ea\u52a8\u521b\u5efa\u3002 \u6ce8\u610f\uff1a\u5982\u9700\u62f7\u8d1d\u5355\u4e2a\u6587\u4ef6, dest \u9700\u6307\u5b9a\u4e3a\u76ee\u5f55\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /test.txt --dest oss://example-oss-bucket/tmp","title":"1\u3001\u4f7f\u7528 --src \u548c --dest (\u5fc5\u9009)"},{"location":"jindotools/jindodistcp_quickstart/#2-bandwidth","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --bandWidth \u8868\u793a\u672c\u6b21distcp\u4efb\u52a1\u6240\u7528\u7684\u5355\u673a\u5e26\u5bbd(\u4ee5MB\u4e3a\u5355\u4f4d)\uff0c\u907f\u514d\u5355\u673a\u5360\u7528\u8fc7\u5927\u5e26\u5bbd\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --bandWidth 6","title":"2\u3001\u4f7f\u7528 --bandWidth"},{"location":"jindotools/jindodistcp_quickstart/#3-codec","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 \u539f\u59cb\u6587\u4ef6\u901a\u5e38\u4ee5\u672a\u538b\u7f29\u7684\u6587\u672c\u683c\u5f0f\u8fdb\u5165 OSS/OSS-HDFS\u3002\u65e0\u8bba\u662f\u5b58\u50a8\u6210\u672c\u8fd8\u662f\u5bf9\u8be5\u6570\u636e\u8fdb\u884c\u5206\u6790\uff0c\u6b64\u683c\u5f0f\u90fd\u4e0d\u751a\u7406\u60f3\u3002JindoDistCp \u53ef\u4ee5\u4f7f\u7528 --codec \u9009\u9879\u5e2e\u52a9\u60a8\u5728\u7ebf\u9ad8\u6548\u5730\u5b58\u50a8\u6570\u636e\u548c\u538b\u7f29\u6587\u4ef6\u3002 --codec \u6307\u5b9a\u6587\u4ef6\u538b\u7f29\u7f16\u89e3\u7801\u5668\u3002 \u652f\u6301 gzip\u3001gz\u3001lzo\u3001lzop \u548c snappy \u5173\u952e\u5b57\u3002 none \u4fdd\u5b58\u4e3a\u672a\u538b\u7f29\u7684\u6587\u4ef6\u3002\u5982\u679c\u6587\u4ef6\u5df2\u538b\u7f29\uff0c\u5219 JindoDistCp \u4f1a\u5c06\u5176\u89e3\u538b\u7f29\u3002 keep(\u9ed8\u8ba4\u503c) \u4e0d\u66f4\u6539\u6587\u4ef6\u538b\u7f29\u5f62\u6001\uff0c\u6309\u539f\u6837\u590d\u5236\u3002 \u547d\u4ee4\u793a\u4f8b\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --codec gz \u6211\u4eec\u68c0\u67e5\u4e00\u4e0b\u76ee\u6807\u6587\u4ef6\u5939\u4e2d\u7684\u6587\u4ef6\uff0c\u8fd9\u4e9b\u6587\u4ef6\u73b0\u5728\u5df2\u7ecf\u7528 gz \u7f16\u89e3\u7801\u5668\u538b\u7f29\u4e86\uff1a [root@emr-header-1 opt]# hdfs dfs -ls oss://example-oss-bucket/hourly_table/2017-02-01/03 Found 6 items -rw-rw-rw- 1 938 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/000151.sst.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/1.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/2.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/OPTIONS-000109.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/emp01.txt.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/emp06.txt.gz \u5982\u60a8\u5728\u5f00\u6e90Hadoop\u96c6\u7fa4\u73af\u5883\u4e2d\u4f7f\u7528lzo\u538b\u7f29\u529f\u80fd\uff0c\u5219\u60a8\u9700\u8981\u5b89\u88c5 gplcompression \u7684 native \u5e93\u548chadoop-lzo\u5305\uff0c\u5982\u60a8\u7f3a\u5c11\u76f8\u5173\u73af\u5883\uff0c\u5efa\u8bae\u4f7f\u7528\u5176\u4ed6\u538b\u7f29\u65b9\u5f0f\u8fdb\u884c\u538b\u7f29\u3002","title":"3\u3001\u4f7f\u7528 --codec"},{"location":"jindotools/jindodistcp_quickstart/#4-filters","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --filters \u6307\u5b9a\u6709\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --filters filter.txt \u5047\u8bbe filter.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u4e0d\u4f1a\u5c06\u8def\u5f84\u4e2d\u5e26\u6709 test \u5b57\u7b26\u7684\u6587\u4ef6\u62f7\u8d1d\u5230OSS\u3002","title":"4\u3001\u4f7f\u7528 --filters"},{"location":"jindotools/jindodistcp_quickstart/#5-srcprefixesfile","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --srcPrefixesFile \u6307\u5b9a\u6709\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --srcPrefixesFile prefixes.txt \u5047\u8bbe prefixes.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u53ea\u6709\u8def\u5f84\u4e2d\u5e26\u6709 test \u5b57\u7b26\u7684\u6587\u4ef6\u624d\u4f1a\u62f7\u8d1d\u5230OSS\u3002","title":"5\u3001\u4f7f\u7528 --srcPrefixesFile"},{"location":"jindotools/jindodistcp_quickstart/#6-parallelism","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --parallelism \u6307\u5b9aMR\u4efb\u52a1\u91cc\u7684 mapreduce.job.reduces \u53c2\u6570\uff0c\u8be5\u53c2\u6570\u5728EMR\u73af\u5883\u4e2d\u9ed8\u8ba4\u4e3a7\uff0c\u60a8\u53ef\u4ee5\u6839\u636e\u96c6\u7fa4\u7684\u8d44\u6e90\u60c5\u51b5\u81ea\u5b9a\u4e49 parallelism \u7684\u5927\u5c0f\u6765\u63a7\u5236 distcp \u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://example-oss-bucket/tmp --parallelism 20","title":"6\u3001\u4f7f\u7528 --parallelism"},{"location":"jindotools/jindodistcp_quickstart/#7-taskbatch","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --taskBatch \u6307\u5b9a\u6bcf\u4e2a distcp task \u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --taskBatch 10","title":"7\u3001\u4f7f\u7528 --taskBatch"},{"location":"jindotools/jindodistcp_quickstart/#8-tmp","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --tmp \u6307\u5b9a\u4e00\u4e2a\u4e34\u65f6\u76ee\u5f55\u5728hdfs\u4e0a\u4f20\u5b58\u653e\u4e34\u65f6\u6570\u636e\uff0c\u9ed8\u8ba4\u503c\u4e3a/tmp\uff0c\u5373hdfs:///tmp/\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --tmp /tmp","title":"8\u3001\u4f7f\u7528 --tmp"},{"location":"jindotools/jindodistcp_quickstart/#9-ossoss-hdfs-ak","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 \u5728 EMR \u5916\u6216\u514d\u5bc6\u670d\u52a1\u51fa\u73b0\u95ee\u9898\u7684\u60c5\u51b5\u4e0b\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9aAK\u6765\u83b7\u5f97\u8bbf\u95ee OSS/OSS-HDFS \u670d\u52a1\u7684\u6743\u9650\u3002\u60a8\u53ef\u4ee5\u5728\u547d\u4ee4\u4e2d\u4f7f\u7528 --hadoopConf \u9009\u9879\u6765\u6307\u5b9a AK\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret \u60a8\u4e5f\u53ef\u4ee5\u5c06 OSS/OSS-HDFS \u670d\u52a1\u7684 ak\u3001secret \u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u907f\u514d\u6bcf\u6b21\u4f7f\u7528\u65f6\u4e34\u65f6\u586b\u5199 ak\u3002\u60a8\u53ef\u4ee5\u5728 E-MapReduce \u63a7\u5236\u53f0 Hadoop-Common \u670d\u52a1\u7684 core-site.xml \u9875\u9762\u914d\u7f6e\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration>","title":"9\u3001\u914d\u7f6e\u8bbf\u95ee OSS/OSS-HDFS \u7684 AK"},{"location":"jindotools/jindodistcp_quickstart/#10-disablechecksum","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --disableChecksum \u5173\u95ed\u68c0\u67e5\u6587\u4ef6 checksum\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --disableChecksum","title":"10\u3001\u4f7f\u7528 --disableChecksum"},{"location":"jindotools/jindodistcp_quickstart/#11-deleteonsuccess","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --deleteOnSuccess \u6307\u5b9a\u79fb\u52a8\u6570\u636e\u800c\u4e0d\u662f\u590d\u5236\u6570\u636e\u3002\u6b64\u9009\u9879\u7c7b\u4f3c\u4e8e mv \u64cd\u4f5c\uff0c\u9996\u5148\u590d\u5236\u6587\u4ef6\uff0c\u7136\u540e\u4ece\u6e90\u4f4d\u7f6e\u5220\u9664\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --deleteOnSuccess","title":"11\u3001\u4f7f\u7528 --deleteOnSuccess"},{"location":"jindotools/jindodistcp_quickstart/#12-enabletransaction","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u4e0d\u652f\u6301 \u652f\u6301 --enableTransaction JindoDistCp \u9ed8\u8ba4\u4f7f\u7528 Task \u7ea7\u522b\u5b8c\u6574\u6027\uff0c\u5982\u60a8\u9700\u8981\u4fdd\u8bc1 Job \u7ea7\u522b\u7684\u5b8c\u6574\u6027\u4ee5\u53ca\u4fdd\u8bc1 Job \u4e4b\u95f4\u7684\u4e8b\u52a1\u652f\u6301\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528\u8be5\u53c2\u6570\u6307\u5b9a\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --enableTransaction","title":"12\u3001\u4f7f\u7528 --enableTransaction"},{"location":"jindotools/jindodistcp_quickstart/#13-ignore","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --ignore \u5ffd\u7565\u6570\u636e\u8fc1\u79fb\u671f\u95f4\u53d1\u751f\u7684\u5f02\u5e38\uff0c\u76f8\u5173\u62a5\u9519\u4e0d\u4f1a\u4e2d\u65ad\u4efb\u52a1\uff0c\u5e76\u6700\u7ec8\u4ee5 JindoCounter \u7684\u5f62\u5f0f\u900f\u51fa\u3002\uff08\u5982\u679c\u5f00\u542fCMS\u7684\u8bdd\uff0c\u4e5f\u4f1a\u4ee5\u6307\u5b9a\u65b9\u5f0f\u8fdb\u884c\u901a\u77e5\u3002\uff09 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --ignore","title":"13\u3001\u4f7f\u7528 --ignore"},{"location":"jindotools/jindodistcp_quickstart/#14-diff","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --diff \u67e5\u770b src \u548c dest \u7684\u6587\u4ef6\u5dee\u5f02\u3002\u5982\u679c src \u7684\u6587\u4ef6\u672a\u80fd\u540c\u6b65\u5230 dest \u4e0a\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\u3002\u5982\u60a8\u7684 JindoDistCp \u4efb\u52a1\u5305\u542b\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u5219 --diff \u4e0d\u80fd\u663e\u793a\u6b63\u786e\u7684\u6587\u4ef6\u5dee\u5f02\uff0c\u56e0\u4e3a\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u4f1a\u6539\u53d8\u6587\u4ef6\u7684\u5927\u5c0f\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --diff \u5982\u679c\u6587\u4ef6\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter DIFF_FILES=1 \u5982\u679c\u60a8\u7684--dest\u4e3aHDFS\u8def\u5f84\uff0c\u73b0\u652f\u6301 /path \uff0c hdfs://hostname:ip/path \uff0c hdfs://headerIp:ip/path \u7684\u5199\u6cd5\uff0c\u6682\u4e0d\u652f\u6301 hdfs:///path \uff0c hdfs:/path \u548c\u5176\u4ed6\u81ea\u5b9a\u4e49\u5199\u6cd5\u3002 \u5982\u679c\u60f3\u67e5\u770b\u6587\u4ef6\u5143\u6570\u636e\u7684\u5dee\u5f02\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 --diff --preserveMeta \u547d\u4ee4\uff1a \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --diff --preserveMeta \u5982\u679c\u6587\u4ef6\u5143\u6570\u636e\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5143\u6570\u636e\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter META_FILE_DIFF=1","title":"14\u3001\u4f7f\u7528 --diff"},{"location":"jindotools/jindodistcp_quickstart/#15-update","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --update \u6307\u5b9a\u589e\u91cf\u540c\u6b65\u529f\u80fd\uff0c\u8df3\u8fc7\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u548c\u76ee\u5f55\uff0c\u76f4\u63a5\u5c06 src \u4e2d\u65b0\u589e\u6216\u53d1\u751f\u6539\u53d8\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u540c\u6b65\u5230 dest \u4e0a\u3002 \u5982\u679c JindoDistCp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53ea Copy \u5269\u4e0b\u672a Copy \u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 JindoDistCp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u6307\u5b9a\u8be5\u53c2\u6570\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --update","title":"15\u3001\u4f7f\u7528 --update"},{"location":"jindotools/jindodistcp_quickstart/#16oss","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u4e0d\u652f\u6301 --update \u6307\u5b9a\u4ee5\u51b7\u5f52\u6863\u3001\u5f52\u6863\u548c\u4f4e\u9891\u7684\u6a21\u5f0f\u5199\u5165 OSS\uff0c\u8fdb\u884c\u6570\u636e\u5b58\u50a8\u3002 1\uff09\u4f7f\u7528\u51b7\u5f52\u6863\u5199\u5165OSS\uff08coldArchive\uff09 \u76ee\u524d\u53ea\u5728\u90e8\u5206region\u53ef\u7528\uff0c\u5177\u4f53\u53c2\u89c1 OSS\u5b58\u50a8\u7c7b\u578b\u4ecb\u7ecd \uff0c\u4f7f\u7528\u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-bucket/hourly_table --policy coldArchive --parallelism 20 2\uff09\u4f7f\u7528\u5f52\u6863\u5199\u5165OSS\uff08archive\uff09 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-bucket/hourly_table --policy archive --parallelism 20 3\uff09\u4f7f\u7528\u4f4e\u9891\u5199\u5165OSS\uff08ia\uff09 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-bucket/hourly_table --policy ia --parallelism 20 \u5982\u60a8\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4\u4ee5 standard \u6807\u51c6\u6a21\u5f0f\u5199\u5165\uff0c\u4e0d\u8fdb\u884c\u51b7\u5f52\u6863\u3001\u5f52\u6863\u548c\u4f4e\u9891\u64cd\u4f5c\u3002","title":"16\u3001\u4f7f\u7528\u51b7\u5f52\u6863/\u5f52\u6863/\u4f4e\u9891\u5199\u5165OSS"},{"location":"jindotools/jindodistcp_quickstart/#17-preservemeta","text":"\u7248\u672c OSS OSS-HDFS 4.4.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u4e0d\u652f\u6301 \u652f\u6301 --preserveMeta \u6307\u5b9a\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --preserveMeta","title":"17\u3001\u4f7f\u7528 --preserveMeta"},{"location":"jindotools/jindodistcp_quickstart/#18-jobbatch","text":"\u7248\u672c OSS OSS-HDFS 4.5.1 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --jobBatch \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--jobBatch\u6765\u6307\u5b9a\u6bcf\u4e2adistcp job\u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10000\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --jobBatch 50000","title":"18\u3001\u4f7f\u7528 --jobBatch"},{"location":"jindotools/jindodistcp_quickstart/#19-enablecms","text":"\u7248\u672c OSS OSS-HDFS 4.5.1 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --enableCMS \u5f00\u542f\u4e91\u76d1\u63a7\u544a\u8b66\u529f\u80fd\uff0c\u5177\u4f53\u53c2\u89c1 \u300aJindoDistcp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66\u300b \u3002","title":"19\u3001 \u4f7f\u7528 --enableCMS"},{"location":"jindotools/jindodistcp_quickstart/#jindodistcp-counters","text":"\u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf BYTES_EXPECTED \u9884\u671f\u7684copy\u5b57\u8282\u6570 FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 BYTES_COPIED copy\u6210\u529f\u7684\u5b57\u8282\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"JindoDistCp Counters \u8bf4\u660e"},{"location":"jindotools/jindodistcp_s3_to_oss/","text":"\u6570\u636e\u4ece\u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u8fc1\u79fb\u5230 OSS/OSS-HDFS \u00a7 \u4f7f\u7528\u524d\u987b\u77e5 \u00a7 \u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d\u3002 \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\u3002 1\u3001\u62f7\u8d1d\u6570\u636e\u5230\u963f\u91cc\u4e91 OSS/OSS-HDFS \u4e0a \u00a7 \u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06\u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS/OSS-HDFS \u4e0a hadoop jar jindo-distcp-tool-${version}.jar s3://example-s3-bucket/data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src \u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u8def\u5f84\u3002 s3://example-s3-bucket/data/ --dest OSS/OSS-HDFS \u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/ --hadoopConf \u6307\u5b9a OSS/OSS-HDFS/S3/COS/OBS \u7684 Access Key ID , Access Key Secret * \u914d\u7f6e OSS/OSS-HDFS \u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS/OSS-HDFS \u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10 2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6 \u00a7 \u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53eaCopy\u5269\u4e0b\u672aCopy\u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a \u4f7f\u7528 --update \u547d\u4ee4\uff0c\u83b7\u5f97\u589e\u91cf\u7684\u6587\u4ef6\u5217\u8868 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src s3://example-s3-bucket/data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --update --parallelism 20 3\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9 \u00a7 \u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src s3://example-s3-bucket/data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB 4\u3001\u914d\u7f6e\u8bbf\u95ee\u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u7684 AK \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9a AK \u6765\u83b7\u5f97\u8bbf\u95ee\u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u7684\u6743\u9650\u3002\u60a8\u53ef\u4ee5\u5728\u547d\u4ee4\u4e2d\u4f7f\u7528 --hadoopConf \u9009\u9879\u6765\u6307\u5b9a AK\u3002 S3 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src s3://example-s3-bucket/data/hourly_table --dest oss://example-oss-bucket/hourly_table --hadoopConf fs.s3.accessKeyId=yourkey --hadoopConf fs.s3.accessKeySecret=yoursecret COS \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src cos://example-cos-bucket/data/hourly_table --dest oss://example-oss-bucket/hourly_table --hadoopConf fs.cos.accessKeyId=yourkey --hadoopConf fs.cos.accessKeySecret=yoursecret OBS \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src obs://example-obs-bucket/data/hourly_table --dest oss://example-oss-bucket/hourly_table --hadoopConf fs.obs.accessKeyId=yourkey --hadoopConf fs.obs.accessKeySecret=yoursecret \u60a8\u4e5f\u53ef\u4ee5\u5c06\u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u7684 AK \u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u907f\u514d\u6bcf\u6b21\u4f7f\u7528\u65f6\u4e34\u65f6\u586b\u5199 AK, \u4ee5 S3 \u4e3a\u4f8b\u3002 <configuration> <property> <name>fs.s3.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.s3.accessKeySecret</name> <value>xxx</value> </property> </configuration> 5\u3001\u914d\u7f6e OSS/OSS-HDFS \u7684 AK \u00a7 \u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS/OSS-HDFS AccessKey/AccessSecret \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662f JindoDistCp \u53ef\u4ee5\u5c06\u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> 6\u3001\u914d\u7f6e OSS-HDFS Endpoint \u00a7 \u8bbf\u95ee OSS-HDFS \u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS/OSS-HDFS Endpoint \u914d\u7f6e \u3002 \u6ce8\u610f \uff1a JindoSDK 4.4.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u9ed8\u8ba4\u4f7f\u7528\u57df\u540d\u5206\u79bb\u7684\u65b9\u5f0f\uff0c\u6570\u636e\u8bfb\u5199\u57df\u540d\u9ed8\u8ba4\u4f7f\u7528\u5185\u7f51\u6807\u51c6 OSS \u57df\u540d\uff0c\u5982\u679c\u5728\u975e\u963f\u91cc\u4e91\u5185\u7f51\u73af\u5883\u6267\u884c distcp \u547d\u4ee4\uff0c\u5219\u9700\u8981\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u914d\u7f6e\u6570\u636e\u6d41\u8bbf\u95ee\u7684\u516c\u7f51 OSS endpoint\u3002 <configuration> <property> <name>fs.oss.data.endpoint</name> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> 7\u3001\u5176\u4ed6\u529f\u80fd \u00a7 \u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003: * JindoDistCp \u4f7f\u7528\u8bf4\u660e","title":"\u4ece S3 \u8fc1\u79fb\u5230 OSS \u4e0a"},{"location":"jindotools/jindodistcp_s3_to_oss/#ossoss-hdfs","text":"","title":"\u6570\u636e\u4ece\u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u8fc1\u79fb\u5230 OSS/OSS-HDFS"},{"location":"jindotools/jindodistcp_s3_to_oss/#_1","text":"\u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d\u3002 \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\u3002","title":"\u4f7f\u7528\u524d\u987b\u77e5"},{"location":"jindotools/jindodistcp_s3_to_oss/#1-ossoss-hdfs","text":"\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06\u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS/OSS-HDFS \u4e0a hadoop jar jindo-distcp-tool-${version}.jar s3://example-s3-bucket/data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src \u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u8def\u5f84\u3002 s3://example-s3-bucket/data/ --dest OSS/OSS-HDFS \u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/ --hadoopConf \u6307\u5b9a OSS/OSS-HDFS/S3/COS/OBS \u7684 Access Key ID , Access Key Secret * \u914d\u7f6e OSS/OSS-HDFS \u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS/OSS-HDFS \u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10","title":"1\u3001\u62f7\u8d1d\u6570\u636e\u5230\u963f\u91cc\u4e91 OSS/OSS-HDFS \u4e0a"},{"location":"jindotools/jindodistcp_s3_to_oss/#2","text":"\u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53eaCopy\u5269\u4e0b\u672aCopy\u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a","title":"2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6"},{"location":"jindotools/jindodistcp_s3_to_oss/#-update","text":"hadoop jar jindo-distcp-tool-${version}.jar --src s3://example-s3-bucket/data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --update --parallelism 20","title":"\u4f7f\u7528 --update \u547d\u4ee4\uff0c\u83b7\u5f97\u589e\u91cf\u7684\u6587\u4ef6\u5217\u8868"},{"location":"jindotools/jindodistcp_s3_to_oss/#3yarn","text":"\u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src s3://example-s3-bucket/data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB","title":"3\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9"},{"location":"jindotools/jindodistcp_s3_to_oss/#4-ak","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9a AK \u6765\u83b7\u5f97\u8bbf\u95ee\u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u7684\u6743\u9650\u3002\u60a8\u53ef\u4ee5\u5728\u547d\u4ee4\u4e2d\u4f7f\u7528 --hadoopConf \u9009\u9879\u6765\u6307\u5b9a AK\u3002 S3 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src s3://example-s3-bucket/data/hourly_table --dest oss://example-oss-bucket/hourly_table --hadoopConf fs.s3.accessKeyId=yourkey --hadoopConf fs.s3.accessKeySecret=yoursecret COS \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src cos://example-cos-bucket/data/hourly_table --dest oss://example-oss-bucket/hourly_table --hadoopConf fs.cos.accessKeyId=yourkey --hadoopConf fs.cos.accessKeySecret=yoursecret OBS \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src obs://example-obs-bucket/data/hourly_table --dest oss://example-oss-bucket/hourly_table --hadoopConf fs.obs.accessKeyId=yourkey --hadoopConf fs.obs.accessKeySecret=yoursecret \u60a8\u4e5f\u53ef\u4ee5\u5c06\u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u7684 AK \u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u907f\u514d\u6bcf\u6b21\u4f7f\u7528\u65f6\u4e34\u65f6\u586b\u5199 AK, \u4ee5 S3 \u4e3a\u4f8b\u3002 <configuration> <property> <name>fs.s3.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.s3.accessKeySecret</name> <value>xxx</value> </property> </configuration>","title":"4\u3001\u914d\u7f6e\u8bbf\u95ee\u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u7684 AK"},{"location":"jindotools/jindodistcp_s3_to_oss/#5-ossoss-hdfs-ak","text":"\u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS/OSS-HDFS AccessKey/AccessSecret \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662f JindoDistCp \u53ef\u4ee5\u5c06\u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration>","title":"5\u3001\u914d\u7f6e OSS/OSS-HDFS \u7684 AK"},{"location":"jindotools/jindodistcp_s3_to_oss/#6-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS-HDFS \u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS/OSS-HDFS Endpoint \u914d\u7f6e \u3002 \u6ce8\u610f \uff1a JindoSDK 4.4.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u9ed8\u8ba4\u4f7f\u7528\u57df\u540d\u5206\u79bb\u7684\u65b9\u5f0f\uff0c\u6570\u636e\u8bfb\u5199\u57df\u540d\u9ed8\u8ba4\u4f7f\u7528\u5185\u7f51\u6807\u51c6 OSS \u57df\u540d\uff0c\u5982\u679c\u5728\u975e\u963f\u91cc\u4e91\u5185\u7f51\u73af\u5883\u6267\u884c distcp \u547d\u4ee4\uff0c\u5219\u9700\u8981\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u914d\u7f6e\u6570\u636e\u6d41\u8bbf\u95ee\u7684\u516c\u7f51 OSS endpoint\u3002 <configuration> <property> <name>fs.oss.data.endpoint</name> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration>","title":"6\u3001\u914d\u7f6e OSS-HDFS Endpoint"},{"location":"jindotools/jindodistcp_s3_to_oss/#7","text":"\u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003: * JindoDistCp \u4f7f\u7528\u8bf4\u660e","title":"7\u3001\u5176\u4ed6\u529f\u80fd"},{"location":"jindotools/jindotable_moveto/","text":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS/OSS-HDFS \u00a7 HDFS \u53d7\u9650\u4e8e\u96c6\u7fa4\u89c4\u6a21\u548c\u6210\u672c\u56e0\u7d20\uff0c\u65e0\u6cd5\u65e0\u9650\u6269\u5c55\uff0c\u5bb9\u91cf\u5b58\u5728\u74f6\u9888\u3002\u4e91\u5382\u5546\u63d0\u4f9b\u7684 OSS/OSS-HDFS \u53ef\u4ee5\u4f5c\u4e3a HDFS \u7684\u66ff\u4ee3\u6216\u8865\u5145\uff0c\u6269\u5c55\u4e91\u4e0a Hadoop \u5e73\u53f0\u7684\u5b58\u50a8\u80fd\u529b\u3002 \u672c\u5de5\u5177\u7528\u4e8e\u628a Hive \u6570\u636e\u6839\u636e\u5206\u533a\u952e\u89c4\u5219\u7b5b\u9009\uff0c\u5728 HDFS \u548c OSS/OSS-HDFS \u4e4b\u95f4\u8f6c\u79fb\u5206\u533a\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u90e8\u7f72 JindoSDK \u00a7 EMR \u73af\u5883\u4e2d\uff0c\u9ed8\u8ba4\u5df2\u5b89\u88c5 JindoSDK\uff0c\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528\u3002\u6ce8\u610f\uff1a \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u521b\u5efaEMR-3.42.0\u53ca\u4ee5\u4e0a\u7248\u672c\u6216EMR-5.8.0\u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u96c6\u7fa4\u3002 \u975e EMR \u73af\u5883\uff0c\u8bf7\u5148\u5b89\u88c5\u90e8\u7f72 JindoSDK\u3002\u90e8\u7f72\u65b9\u5f0f\u8bf7\u53c2\u8003 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u90e8\u7f72 JindoSDK 4.x \u53ca\u4ee5\u4e0a\u7248\u672c\u3002 \u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883 \u00a7 \u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e \u914d\u7f6e MoveTo \u5de5\u5177\u5728 HDFS \u4e0b\u7684\u9501\u76ee\u5f55\uff1a \u00a7 \u5728 Hadoop \u914d\u7f6e\u6587\u4ef6 core-site.xml \u6216 hdfs-site.xml \uff08\u4efb\u4e00\u5373\u53ef\uff0c\u5728 $HADOOP_CONF_DIR \u76ee\u5f55\u4e0b\uff09\u65b0\u589e\u914d\u7f6e\u9879\uff1a jindotable.moveto.tablelock.base.dir \u8be5\u914d\u7f6e\u7684\u503c\u5e94\u6307\u5411\u4e00\u4e2a HDFS \u76ee\u5f55\uff0c\u76ee\u7684\u662f\u5b58\u653e MoveTo \u5de5\u5177\u5728\u8fd0\u884c\u65f6\u81ea\u52a8\u521b\u5efa\u7684\u9501\u6587\u4ef6\u3002\u9700\u786e\u4fdd\u8be5\u76ee\u5f55\u53ea\u4f1a\u88ab MoveTo \u5de5\u5177\u8bbf\u95ee\uff0c\u5e76\u4e14\u6709\u8bbf\u95ee\u6743\u9650\u3002\u5982\u679c\u4e0d\u914d\u7f6e\uff0c\u5219\u4f7f\u7528\u7f3a\u7701\u503c hdfs:///tmp/jindotable-lock/ \uff0c\u65e0\u6743\u9650\u5219\u62a5\u9519\u3002 \u4f7f\u7528\u8bf4\u660e \u00a7 \u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f \u00a7 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help moveTo \u53c2\u6570\u8bf4\u660e \u00a7 jindotable -moveTo -t <dbName.tableName> -d <destination path> [-c \"<condition>\" | -fullTable] [-b/-before <before days>] [-p/-parallel <parallelism>] [-s/-storagePolicy <OSS storage policy>] [-o/-overWrite] [-r/-removeSource] [-skipTrash] [-e/-explain] [-q/-queue <yarn queue>] [-w/-workingDir <working directory>][-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u8981\u79fb\u52a8\u7684\u8868\u3002 \u662f -d <destination path> \u76ee\u6807\u8def\u5f84\uff0c\u4e3a\u8868\u7ea7\u522b\u7684\u8def\u5f84\uff0c\u5206\u533a\u8def\u5f84\u4f1a\u5728\u8fd9\u4e2a\u8def\u5f84\u4e0b\u81ea\u52a8\u521b\u5efa\u3002 \u662f -c \"<condition>\" / -fullTable \u5206\u533a\u8fc7\u6ee4\u6761\u4ef6\u8868\u8fbe\u5f0f\uff0c\u652f\u6301\u57fa\u672c\u8fd0\u7b97\u7b26\uff0c\u4e0d\u652f\u6301udf\u3002 \u5426 -b/before <before days> \u6839\u636e\u5206\u533a\u521b\u5efa\u65f6\u95f4\uff0c\u521b\u5efa\u65f6\u95f4\u8d85\u8fc7\u7ed9\u5b9a\u5929\u6570\u7684\u5206\u533a\u624d\u8fdb\u884c\u79fb\u52a8\u3002 \u5426 -p/-parallel <parallelism> \u6574\u4e2amoveTo\u4efb\u52a1\u7684\u6700\u5927task\u5e76\u53d1\u5ea6\uff0c\u9ed8\u8ba4\u4e3a1\u3002 \u5426 -s/-storagePolicy <OSS storage policy> \u62f7\u8d1d\u81f3 OSS \u65f6\u6570\u636e\u6587\u4ef6\u7684\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301 Standard\uff08\u9ed8\u8ba4\uff09\u3001IA\u3001Archive\u3001ColdArchive\u3002 \u5426 -o/-overWrite \u662f\u5426\u8986\u76d6\u6700\u7ec8\u76ee\u5f55\u3002\u5206\u533a\u7ea7\u522b\u8986\u76d6\uff0c\u4e0d\u4f1a\u8986\u76d6\u672c\u6b21\u79fb\u52a8\u4e0d\u6d89\u53ca\u7684\u5206\u533a\u3002 \u5426 -r/-removeSource \u662f\u5426\u5728\u79fb\u52a8\u5b8c\u6210\u540e\u5220\u9664\u6e90\u8def\u5f84\u3002 \u5426 -skipTrash \u5982\u679c\u5220\u9664\u6e90\u8def\u5f84\uff0c\u662f\u5426\u8df3\u8fc7Trash\u3002 \u5426 -e/-explain \u5982\u679c\u6307\u5b9aexplain\u6a21\u5f0f\uff0c\u4e0d\u4f1a\u89e6\u53d1\u5b9e\u9645\u64cd\u4f5c\uff0c\u4ec5\u6253\u5370\u4f1a\u540c\u6b65\u7684\u5206\u533a\u3002 \u5426 -q/-queue <yarn queue \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684yarn\u961f\u5217\u3002 \u5426 -w/-workingDir \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684\u5de5\u4f5c\u4e34\u65f6\u76ee\u5f55\u3002 \u5426 -iceberg \u6307\u5b9a\u6240\u8fc1\u79fb\u8868\u4e3aIceberg\u3002\u76ee\u524d\u4ec5\u652f\u6301Iceberg\u6574\u8868\u8fc1\u79fb\uff0c\u9700\u914d\u5408 -fullTable \u4f7f\u7528\u3002(6.3.0\u4ee5\u4e0a\u7248\u672c) \u5426 -l/-logDir <log directory> \u672c\u5730\u65e5\u5fd7\u76ee\u5f55\uff0c\u9ed8\u8ba4\u4e3a /tmp/<current user>/ \u5426 \u4f7f\u7528\u793a\u4f8b \u00a7 \u6709\u4e00\u4e2a HDFS \u4e0a\u7684 Hive \u5206\u533a\u8868\uff0c\u5982\u4e0b\u56fe\u6240\u793a\uff1a \u60f3\u628a bbb \u548c ccc \u5206\u533a\u79fb\u52a8\u5230 OSS\uff0c\u5148\u7528 explain \u6a21\u5f0f\u770b\u770b\u4f1a\u79fb\u52a8\u7684\u5206\u533a\u662f\u5426\u7b26\u5408\u9884\u671f\uff0c\u53c2\u6570\u4e3a -e \u6216 -explain\uff1a \u53bb\u6389\u53c2\u6570 -e\uff0c\u771f\u6b63\u79fb\u52a8\u5206\u533a\uff1a \u6267\u884c\u5b8c\u6210\u540e\uff0c\u68c0\u67e5\u6570\u636e\uff0c\u6570\u636e\u5df2\u7ecf\u5728 OSS\uff1a \u518d\u79fb\u56de HDFS\uff0c\u7ed3\u679c\u5931\u8d25\u4e86\uff0c\u4f1a\u663e\u793a\u5931\u8d25\u539f\u56e0\uff0c\u539f\u6765\u662f\u76ee\u6807\u76ee\u5f55\u975e\u7a7a\uff0c\u63d0\u793a\u53ef\u4ee5\u4f7f\u7528 -overWrite \u6e05\u7a7a\u76ee\u6807\u76ee\u5f55\uff1a \u4f7f\u7528 -overWrite \u5f3a\u5236\u8986\u76d6\uff1a \u5f02\u5e38\u72b6\u6001\u5904\u7406 \u00a7 \u4e3a\u4e86\u5c3d\u53ef\u80fd\u4fdd\u8bc1\u6570\u636e\u5b89\u5168\uff0c\u9632\u6b62\u6c61\u67d3\uff0c\u8be5\u547d\u4ee4\u4f1a\u81ea\u52a8\u68c0\u67e5\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u786e\u4fdd\u4e0d\u5b58\u5728\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u76ee\u5f55\u62f7\u8d1d\u3002 \u5982\u679c\u65e0\u6cd5\u6392\u9664\uff0c\u5219\u8be5\u8868\u6216\u5206\u533a\u7684\u62f7\u8d1d\u547d\u4ee4\u5c06\u6267\u884c\u5931\u8d25\u3002 \u6b64\u65f6\uff0c\u9700\u8981\u4e3b\u52a8\u5e72\u9884\uff0c\u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u6b63\u5728\u6267\u884c\u62f7\u8d1d\u4e4b\u540e\uff0c\u6e05\u7406\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u5219\u53ef\u4ee5\u91cd\u65b0\u6267\u884c\u8be5\u547d\u4ee4\u3002 \u5bf9\u4e8e\u975e\u5206\u533a\u8868\u7684\u62f7\u8d1d\u6216\u8fc1\u79fb\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u5373\u4e3a\u8868\u7ea7\u522b\u76ee\u5f55\uff1b\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u4e3a\u5f85\u62f7\u8d1d\u6216\u79fb\u52a8\u5206\u533a\u7684\u5206\u533a\u7ea7\u76ee\u5f55\uff0c \u53ea\u9700\u5bf9\u5f85\u62f7\u8d1d\u7684\u5206\u533a\u8fdb\u884c\u6e05\u7406\u3002 \u51fa\u73b0\u539f\u56e0 \u00a7 \u5982\u679c\u8be5\u547d\u4ee4\u88ab\u5f02\u5e38\u4e2d\u6b62\uff0c\u5219\u53ef\u80fd\u51fa\u73b0\u9700\u8981\u4e3b\u52a8\u5e72\u9884\u7684\u60c5\u51b5\u3002\u6b64\u65f6\u62f7\u8d1d\u8fd8\u672a\u5b8c\u6210\uff0c\u6e90\u6570\u636e\u4e0e\u8868\u7684\u5143\u4fe1\u606f\u5747\u672a\u6539\u53d8\uff0c \u6570\u636e\u4ecd\u5904\u4e8e\u5b89\u5168\u72b6\u6001\u3002\u5e38\u89c1\u7684\u5f02\u5e38\u4e2d\u6b62\u6709\u4e24\u79cd\u60c5\u51b5\uff1a * \u7528\u6237\u5728\u547d\u4ee4\u5c1a\u672a\u7ed3\u675f\u65f6\uff0c\u4e3b\u52a8\u6740\u6389\u547d\u4ee4\u8fdb\u7a0b * \u7531\u4e8e\u5185\u5b58\u6ea2\u51fa\u7b49\u5f02\u5e38\uff0c\u8fdb\u7a0b\u5f02\u5e38\u4e2d\u6b62 \u5982\u679c\u62f7\u8d1d\u8868\u6216\u5206\u533a\u65f6\u62f7\u8d1d\u5931\u8d25\u5e76\u663e\u793a Conflicts found \uff0c\u5219\u5c5e\u4e8e\u5f02\u5e38\u72b6\u6001\uff0c\u9700\u4e3b\u52a8\u5e72\u9884\u3002 \u5904\u7406\u65b9\u6cd5 \u00a7 \u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u7684\u76ee\u6807\u8def\u5f84\u62f7\u8d1d\u6570\u636e\u3002\u7279\u522b\u662f\uff0c\u786e\u4fdd\u6ca1\u6709 DistCp/JindoDistCp \u7b49\u5206\u5e03\u5f0f\u62f7\u8d1d\u547d\u4ee4\u3002 \u5220\u9664\u76ee\u7684\u5730\u76ee\u5f55\u3002\u5bf9\u4e8e\u975e\u5206\u533a\u8868\uff0c\u5220\u9664\u8868\u4e00\u7ea7\u76ee\u5f55\u3002\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u5220\u9664\u5b58\u5728\u51b2\u7a81\u7684\u5206\u533a\u7ea7\u76ee\u5f55\u3002\u5207\u52ff\u5220\u9664\u6e90\u76ee\u5f55\u3002","title":"\u4f7f\u7528 JindoTable \u5c06 Hive \u6570\u636e\u5206\u533a\u5f52\u6863\u548c\u5206\u5c42\u5230OSS"},{"location":"jindotools/jindotable_moveto/#jindotable-hive-ossoss-hdfs","text":"HDFS \u53d7\u9650\u4e8e\u96c6\u7fa4\u89c4\u6a21\u548c\u6210\u672c\u56e0\u7d20\uff0c\u65e0\u6cd5\u65e0\u9650\u6269\u5c55\uff0c\u5bb9\u91cf\u5b58\u5728\u74f6\u9888\u3002\u4e91\u5382\u5546\u63d0\u4f9b\u7684 OSS/OSS-HDFS \u53ef\u4ee5\u4f5c\u4e3a HDFS \u7684\u66ff\u4ee3\u6216\u8865\u5145\uff0c\u6269\u5c55\u4e91\u4e0a Hadoop \u5e73\u53f0\u7684\u5b58\u50a8\u80fd\u529b\u3002 \u672c\u5de5\u5177\u7528\u4e8e\u628a Hive \u6570\u636e\u6839\u636e\u5206\u533a\u952e\u89c4\u5219\u7b5b\u9009\uff0c\u5728 HDFS \u548c OSS/OSS-HDFS \u4e4b\u95f4\u8f6c\u79fb\u5206\u533a\u3002","title":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS/OSS-HDFS"},{"location":"jindotools/jindotable_moveto/#_1","text":"","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"jindotools/jindotable_moveto/#jindosdk","text":"EMR \u73af\u5883\u4e2d\uff0c\u9ed8\u8ba4\u5df2\u5b89\u88c5 JindoSDK\uff0c\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528\u3002\u6ce8\u610f\uff1a \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u521b\u5efaEMR-3.42.0\u53ca\u4ee5\u4e0a\u7248\u672c\u6216EMR-5.8.0\u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u96c6\u7fa4\u3002 \u975e EMR \u73af\u5883\uff0c\u8bf7\u5148\u5b89\u88c5\u90e8\u7f72 JindoSDK\u3002\u90e8\u7f72\u65b9\u5f0f\u8bf7\u53c2\u8003 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u90e8\u7f72 JindoSDK 4.x \u53ca\u4ee5\u4e0a\u7248\u672c\u3002","title":"\u5df2\u90e8\u7f72 JindoSDK"},{"location":"jindotools/jindotable_moveto/#hadoop-hive","text":"\u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e","title":"\u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883"},{"location":"jindotools/jindotable_moveto/#moveto-hdfs","text":"\u5728 Hadoop \u914d\u7f6e\u6587\u4ef6 core-site.xml \u6216 hdfs-site.xml \uff08\u4efb\u4e00\u5373\u53ef\uff0c\u5728 $HADOOP_CONF_DIR \u76ee\u5f55\u4e0b\uff09\u65b0\u589e\u914d\u7f6e\u9879\uff1a jindotable.moveto.tablelock.base.dir \u8be5\u914d\u7f6e\u7684\u503c\u5e94\u6307\u5411\u4e00\u4e2a HDFS \u76ee\u5f55\uff0c\u76ee\u7684\u662f\u5b58\u653e MoveTo \u5de5\u5177\u5728\u8fd0\u884c\u65f6\u81ea\u52a8\u521b\u5efa\u7684\u9501\u6587\u4ef6\u3002\u9700\u786e\u4fdd\u8be5\u76ee\u5f55\u53ea\u4f1a\u88ab MoveTo \u5de5\u5177\u8bbf\u95ee\uff0c\u5e76\u4e14\u6709\u8bbf\u95ee\u6743\u9650\u3002\u5982\u679c\u4e0d\u914d\u7f6e\uff0c\u5219\u4f7f\u7528\u7f3a\u7701\u503c hdfs:///tmp/jindotable-lock/ \uff0c\u65e0\u6743\u9650\u5219\u62a5\u9519\u3002","title":"\u914d\u7f6e MoveTo \u5de5\u5177\u5728 HDFS \u4e0b\u7684\u9501\u76ee\u5f55\uff1a"},{"location":"jindotools/jindotable_moveto/#_2","text":"","title":"\u4f7f\u7528\u8bf4\u660e"},{"location":"jindotools/jindotable_moveto/#_3","text":"\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help moveTo","title":"\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f"},{"location":"jindotools/jindotable_moveto/#_4","text":"jindotable -moveTo -t <dbName.tableName> -d <destination path> [-c \"<condition>\" | -fullTable] [-b/-before <before days>] [-p/-parallel <parallelism>] [-s/-storagePolicy <OSS storage policy>] [-o/-overWrite] [-r/-removeSource] [-skipTrash] [-e/-explain] [-q/-queue <yarn queue>] [-w/-workingDir <working directory>][-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u8981\u79fb\u52a8\u7684\u8868\u3002 \u662f -d <destination path> \u76ee\u6807\u8def\u5f84\uff0c\u4e3a\u8868\u7ea7\u522b\u7684\u8def\u5f84\uff0c\u5206\u533a\u8def\u5f84\u4f1a\u5728\u8fd9\u4e2a\u8def\u5f84\u4e0b\u81ea\u52a8\u521b\u5efa\u3002 \u662f -c \"<condition>\" / -fullTable \u5206\u533a\u8fc7\u6ee4\u6761\u4ef6\u8868\u8fbe\u5f0f\uff0c\u652f\u6301\u57fa\u672c\u8fd0\u7b97\u7b26\uff0c\u4e0d\u652f\u6301udf\u3002 \u5426 -b/before <before days> \u6839\u636e\u5206\u533a\u521b\u5efa\u65f6\u95f4\uff0c\u521b\u5efa\u65f6\u95f4\u8d85\u8fc7\u7ed9\u5b9a\u5929\u6570\u7684\u5206\u533a\u624d\u8fdb\u884c\u79fb\u52a8\u3002 \u5426 -p/-parallel <parallelism> \u6574\u4e2amoveTo\u4efb\u52a1\u7684\u6700\u5927task\u5e76\u53d1\u5ea6\uff0c\u9ed8\u8ba4\u4e3a1\u3002 \u5426 -s/-storagePolicy <OSS storage policy> \u62f7\u8d1d\u81f3 OSS \u65f6\u6570\u636e\u6587\u4ef6\u7684\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301 Standard\uff08\u9ed8\u8ba4\uff09\u3001IA\u3001Archive\u3001ColdArchive\u3002 \u5426 -o/-overWrite \u662f\u5426\u8986\u76d6\u6700\u7ec8\u76ee\u5f55\u3002\u5206\u533a\u7ea7\u522b\u8986\u76d6\uff0c\u4e0d\u4f1a\u8986\u76d6\u672c\u6b21\u79fb\u52a8\u4e0d\u6d89\u53ca\u7684\u5206\u533a\u3002 \u5426 -r/-removeSource \u662f\u5426\u5728\u79fb\u52a8\u5b8c\u6210\u540e\u5220\u9664\u6e90\u8def\u5f84\u3002 \u5426 -skipTrash \u5982\u679c\u5220\u9664\u6e90\u8def\u5f84\uff0c\u662f\u5426\u8df3\u8fc7Trash\u3002 \u5426 -e/-explain \u5982\u679c\u6307\u5b9aexplain\u6a21\u5f0f\uff0c\u4e0d\u4f1a\u89e6\u53d1\u5b9e\u9645\u64cd\u4f5c\uff0c\u4ec5\u6253\u5370\u4f1a\u540c\u6b65\u7684\u5206\u533a\u3002 \u5426 -q/-queue <yarn queue \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684yarn\u961f\u5217\u3002 \u5426 -w/-workingDir \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684\u5de5\u4f5c\u4e34\u65f6\u76ee\u5f55\u3002 \u5426 -iceberg \u6307\u5b9a\u6240\u8fc1\u79fb\u8868\u4e3aIceberg\u3002\u76ee\u524d\u4ec5\u652f\u6301Iceberg\u6574\u8868\u8fc1\u79fb\uff0c\u9700\u914d\u5408 -fullTable \u4f7f\u7528\u3002(6.3.0\u4ee5\u4e0a\u7248\u672c) \u5426 -l/-logDir <log directory> \u672c\u5730\u65e5\u5fd7\u76ee\u5f55\uff0c\u9ed8\u8ba4\u4e3a /tmp/<current user>/ \u5426","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"jindotools/jindotable_moveto/#_5","text":"\u6709\u4e00\u4e2a HDFS \u4e0a\u7684 Hive \u5206\u533a\u8868\uff0c\u5982\u4e0b\u56fe\u6240\u793a\uff1a \u60f3\u628a bbb \u548c ccc \u5206\u533a\u79fb\u52a8\u5230 OSS\uff0c\u5148\u7528 explain \u6a21\u5f0f\u770b\u770b\u4f1a\u79fb\u52a8\u7684\u5206\u533a\u662f\u5426\u7b26\u5408\u9884\u671f\uff0c\u53c2\u6570\u4e3a -e \u6216 -explain\uff1a \u53bb\u6389\u53c2\u6570 -e\uff0c\u771f\u6b63\u79fb\u52a8\u5206\u533a\uff1a \u6267\u884c\u5b8c\u6210\u540e\uff0c\u68c0\u67e5\u6570\u636e\uff0c\u6570\u636e\u5df2\u7ecf\u5728 OSS\uff1a \u518d\u79fb\u56de HDFS\uff0c\u7ed3\u679c\u5931\u8d25\u4e86\uff0c\u4f1a\u663e\u793a\u5931\u8d25\u539f\u56e0\uff0c\u539f\u6765\u662f\u76ee\u6807\u76ee\u5f55\u975e\u7a7a\uff0c\u63d0\u793a\u53ef\u4ee5\u4f7f\u7528 -overWrite \u6e05\u7a7a\u76ee\u6807\u76ee\u5f55\uff1a \u4f7f\u7528 -overWrite \u5f3a\u5236\u8986\u76d6\uff1a","title":"\u4f7f\u7528\u793a\u4f8b"},{"location":"jindotools/jindotable_moveto/#_6","text":"\u4e3a\u4e86\u5c3d\u53ef\u80fd\u4fdd\u8bc1\u6570\u636e\u5b89\u5168\uff0c\u9632\u6b62\u6c61\u67d3\uff0c\u8be5\u547d\u4ee4\u4f1a\u81ea\u52a8\u68c0\u67e5\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u786e\u4fdd\u4e0d\u5b58\u5728\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u76ee\u5f55\u62f7\u8d1d\u3002 \u5982\u679c\u65e0\u6cd5\u6392\u9664\uff0c\u5219\u8be5\u8868\u6216\u5206\u533a\u7684\u62f7\u8d1d\u547d\u4ee4\u5c06\u6267\u884c\u5931\u8d25\u3002 \u6b64\u65f6\uff0c\u9700\u8981\u4e3b\u52a8\u5e72\u9884\uff0c\u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u6b63\u5728\u6267\u884c\u62f7\u8d1d\u4e4b\u540e\uff0c\u6e05\u7406\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u5219\u53ef\u4ee5\u91cd\u65b0\u6267\u884c\u8be5\u547d\u4ee4\u3002 \u5bf9\u4e8e\u975e\u5206\u533a\u8868\u7684\u62f7\u8d1d\u6216\u8fc1\u79fb\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u5373\u4e3a\u8868\u7ea7\u522b\u76ee\u5f55\uff1b\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u4e3a\u5f85\u62f7\u8d1d\u6216\u79fb\u52a8\u5206\u533a\u7684\u5206\u533a\u7ea7\u76ee\u5f55\uff0c \u53ea\u9700\u5bf9\u5f85\u62f7\u8d1d\u7684\u5206\u533a\u8fdb\u884c\u6e05\u7406\u3002","title":"\u5f02\u5e38\u72b6\u6001\u5904\u7406"},{"location":"jindotools/jindotable_moveto/#_7","text":"\u5982\u679c\u8be5\u547d\u4ee4\u88ab\u5f02\u5e38\u4e2d\u6b62\uff0c\u5219\u53ef\u80fd\u51fa\u73b0\u9700\u8981\u4e3b\u52a8\u5e72\u9884\u7684\u60c5\u51b5\u3002\u6b64\u65f6\u62f7\u8d1d\u8fd8\u672a\u5b8c\u6210\uff0c\u6e90\u6570\u636e\u4e0e\u8868\u7684\u5143\u4fe1\u606f\u5747\u672a\u6539\u53d8\uff0c \u6570\u636e\u4ecd\u5904\u4e8e\u5b89\u5168\u72b6\u6001\u3002\u5e38\u89c1\u7684\u5f02\u5e38\u4e2d\u6b62\u6709\u4e24\u79cd\u60c5\u51b5\uff1a * \u7528\u6237\u5728\u547d\u4ee4\u5c1a\u672a\u7ed3\u675f\u65f6\uff0c\u4e3b\u52a8\u6740\u6389\u547d\u4ee4\u8fdb\u7a0b * \u7531\u4e8e\u5185\u5b58\u6ea2\u51fa\u7b49\u5f02\u5e38\uff0c\u8fdb\u7a0b\u5f02\u5e38\u4e2d\u6b62 \u5982\u679c\u62f7\u8d1d\u8868\u6216\u5206\u533a\u65f6\u62f7\u8d1d\u5931\u8d25\u5e76\u663e\u793a Conflicts found \uff0c\u5219\u5c5e\u4e8e\u5f02\u5e38\u72b6\u6001\uff0c\u9700\u4e3b\u52a8\u5e72\u9884\u3002","title":"\u51fa\u73b0\u539f\u56e0"},{"location":"jindotools/jindotable_moveto/#_8","text":"\u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u7684\u76ee\u6807\u8def\u5f84\u62f7\u8d1d\u6570\u636e\u3002\u7279\u522b\u662f\uff0c\u786e\u4fdd\u6ca1\u6709 DistCp/JindoDistCp \u7b49\u5206\u5e03\u5f0f\u62f7\u8d1d\u547d\u4ee4\u3002 \u5220\u9664\u76ee\u7684\u5730\u76ee\u5f55\u3002\u5bf9\u4e8e\u975e\u5206\u533a\u8868\uff0c\u5220\u9664\u8868\u4e00\u7ea7\u76ee\u5f55\u3002\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u5220\u9664\u5b58\u5728\u51b2\u7a81\u7684\u5206\u533a\u7ea7\u76ee\u5f55\u3002\u5207\u52ff\u5220\u9664\u6e90\u76ee\u5f55\u3002","title":"\u5904\u7406\u65b9\u6cd5"},{"location":"jindotools/jindotable_set_storage_class/","text":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fdb\u884c\u5f52\u6863 \u00a7 \u4f7f\u7528\u8bf4\u660e \u00a7 \u8be5\u547d\u4ee4\u53ea\u652f\u6301\u6807\u51c6 OSS \u5b58\u50a8\u7c7b\u578b\u7684\u8f6c\u6362\uff0c \u4e0d\u652f\u6301OSS-HDFS \u5b58\u50a8\u7c7b\u578b\u8f6c\u6362\uff0cOSS-HDFS \u5b58\u50a8\u7c7b\u578b\u8f6c\u6362\u53c2\u8003 \u94fe\u63a5 \u914d\u7f6eOSS \u751f\u547d\u5468\u671f \u00a7 \u914d\u7f6e\u4f4e\u9891\u751f\u547d\u5468\u671f\u89c4\u5219 \u00a7 \u914d\u7f6e\u5f52\u6863\u751f\u547d\u5468\u671f\u89c4\u5219 \u00a7 \u914d\u7f6e\u51b7\u5f52\u6863\u751f\u547d\u5468\u671f\u89c4\u5219 \u00a7 \u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f \u00a7 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable help -setStorageClass \u53c2\u6570\u8bf4\u660e \u00a7 [-setStorageClass -t <dbName.tableName> -i/-a/-ca[-c \"<condition>\" | -fullTable] [-b/-before <before days>] [-p/-parallel <parallelism>] [-mr/-mapReduce] [-e/-explain] [-w/-workingDir <working directory>][-l/-logDir <log directory>]] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u8981\u79fb\u52a8\u7684\u8868\u3002 \u662f -i/-a/-ca \u6307\u5b9a\u5b58\u50a8\u7c7b\u578b\uff0c-i\u8868\u793a\u4f4e\u9891\uff0c-a\u8868\u793a\u5f52\u6863 -ca\u8868\u793a\u51b7\u5f52\u6863\u3002 \u662f -c \"<condition>\" / -fullTable \u5206\u533a\u8fc7\u6ee4\u6761\u4ef6\u8868\u8fbe\u5f0f\uff0c\u652f\u6301\u57fa\u672c\u8fd0\u7b97\u7b26\uff0c\u4e0d\u652f\u6301udf\u3002 \u5426 -b/before <before days> \u6839\u636e\u5206\u533a\u521b\u5efa\u65f6\u95f4\uff0c\u521b\u5efa\u65f6\u95f4\u8d85\u8fc7\u7ed9\u5b9a\u5929\u6570\u7684\u5206\u533a\u624d\u8fdb\u884c\u79fb\u52a8\u3002 \u5426 -p/-parallel <parallelism> \u6574\u4e2amoveTo\u4efb\u52a1\u7684\u6700\u5927task\u5e76\u53d1\u5ea6\uff0c\u9ed8\u8ba4\u4e3a1\u3002 \u5426 -e/-explain \u5982\u679c\u6307\u5b9aexplain\u6a21\u5f0f\uff0c\u4e0d\u4f1a\u89e6\u53d1\u5b9e\u9645\u64cd\u4f5c\uff0c\u4ec5\u6253\u5370\u4f1a\u540c\u6b65\u7684\u5206\u533a\u3002 \u5426 -w/-workingDir \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684\u5de5\u4f5c\u4e34\u65f6\u76ee\u5f55\u3002 \u5426 -l/-logDir <log directory> \u672c\u5730\u65e5\u5fd7\u76ee\u5f55\uff0c\u9ed8\u8ba4\u4e3a /tmp/<current user>/ \u5426 \u4f7f\u7528\u793a\u4f8b \u00a7 jindotable -setStorageClass -t data_center.test_partition -ca -c \"date_id > '20240110'\"","title":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fdb\u884c\u5f52\u6863"},{"location":"jindotools/jindotable_set_storage_class/#jindotable-hive","text":"","title":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fdb\u884c\u5f52\u6863"},{"location":"jindotools/jindotable_set_storage_class/#_1","text":"\u8be5\u547d\u4ee4\u53ea\u652f\u6301\u6807\u51c6 OSS \u5b58\u50a8\u7c7b\u578b\u7684\u8f6c\u6362\uff0c \u4e0d\u652f\u6301OSS-HDFS \u5b58\u50a8\u7c7b\u578b\u8f6c\u6362\uff0cOSS-HDFS \u5b58\u50a8\u7c7b\u578b\u8f6c\u6362\u53c2\u8003 \u94fe\u63a5","title":"\u4f7f\u7528\u8bf4\u660e"},{"location":"jindotools/jindotable_set_storage_class/#oss","text":"","title":"\u914d\u7f6eOSS \u751f\u547d\u5468\u671f"},{"location":"jindotools/jindotable_set_storage_class/#_2","text":"","title":"\u914d\u7f6e\u4f4e\u9891\u751f\u547d\u5468\u671f\u89c4\u5219"},{"location":"jindotools/jindotable_set_storage_class/#_3","text":"","title":"\u914d\u7f6e\u5f52\u6863\u751f\u547d\u5468\u671f\u89c4\u5219"},{"location":"jindotools/jindotable_set_storage_class/#_4","text":"","title":"\u914d\u7f6e\u51b7\u5f52\u6863\u751f\u547d\u5468\u671f\u89c4\u5219"},{"location":"jindotools/jindotable_set_storage_class/#_5","text":"\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable help -setStorageClass","title":"\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f"},{"location":"jindotools/jindotable_set_storage_class/#_6","text":"[-setStorageClass -t <dbName.tableName> -i/-a/-ca[-c \"<condition>\" | -fullTable] [-b/-before <before days>] [-p/-parallel <parallelism>] [-mr/-mapReduce] [-e/-explain] [-w/-workingDir <working directory>][-l/-logDir <log directory>]] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u8981\u79fb\u52a8\u7684\u8868\u3002 \u662f -i/-a/-ca \u6307\u5b9a\u5b58\u50a8\u7c7b\u578b\uff0c-i\u8868\u793a\u4f4e\u9891\uff0c-a\u8868\u793a\u5f52\u6863 -ca\u8868\u793a\u51b7\u5f52\u6863\u3002 \u662f -c \"<condition>\" / -fullTable \u5206\u533a\u8fc7\u6ee4\u6761\u4ef6\u8868\u8fbe\u5f0f\uff0c\u652f\u6301\u57fa\u672c\u8fd0\u7b97\u7b26\uff0c\u4e0d\u652f\u6301udf\u3002 \u5426 -b/before <before days> \u6839\u636e\u5206\u533a\u521b\u5efa\u65f6\u95f4\uff0c\u521b\u5efa\u65f6\u95f4\u8d85\u8fc7\u7ed9\u5b9a\u5929\u6570\u7684\u5206\u533a\u624d\u8fdb\u884c\u79fb\u52a8\u3002 \u5426 -p/-parallel <parallelism> \u6574\u4e2amoveTo\u4efb\u52a1\u7684\u6700\u5927task\u5e76\u53d1\u5ea6\uff0c\u9ed8\u8ba4\u4e3a1\u3002 \u5426 -e/-explain \u5982\u679c\u6307\u5b9aexplain\u6a21\u5f0f\uff0c\u4e0d\u4f1a\u89e6\u53d1\u5b9e\u9645\u64cd\u4f5c\uff0c\u4ec5\u6253\u5370\u4f1a\u540c\u6b65\u7684\u5206\u533a\u3002 \u5426 -w/-workingDir \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684\u5de5\u4f5c\u4e34\u65f6\u76ee\u5f55\u3002 \u5426 -l/-logDir <log directory> \u672c\u5730\u65e5\u5fd7\u76ee\u5f55\uff0c\u9ed8\u8ba4\u4e3a /tmp/<current user>/ \u5426","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"jindotools/jindotable_set_storage_class/#_7","text":"jindotable -setStorageClass -t data_center.test_partition -ca -c \"date_id > '20240110'\"","title":"\u4f7f\u7528\u793a\u4f8b"},{"location":"jindotools/jindotable_storagepolicy_restore/","text":"\u4f7f\u7528 JindoTable \u5de5\u5177\u5b9e\u73b0 OSS-HDFS \u7ed3\u6784\u5316\u6570\u636e\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8 \u00a7 \u6982\u8ff0 \u00a7 JindoTable \u662f\u57fa\u4e8e JindoSDK \u7684\u7ed3\u6784\u5316\u6570\u636e\u6cbb\u7406\u5de5\u5177\uff0c\u76ee\u6807\u662f\u5728 \u201c\u8868\u201d \u6216\u8005 \u201c\u5206\u533a\u201d \u5c42\u9762\u63d0\u4f9b\u4fbf\u6377\u7684\u6570\u636e\u8bbf\u95ee\u548c\u64cd\u4f5c\u3002\u4f7f\u7528 JindoTable \u5de5\u5177\u53ef\u4ee5\u65b9\u4fbf\u5730\u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u548c\u5206\u5c42\u5b58\u50a8\u3002 \u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u4f7f\u7528 JindoTable \u5de5\u5177\u5bf9\u5b58\u50a8\u5728 OSS-HDFS \u4e0a\u7684\u7ed3\u6784\u5316\u6570\u636e\u8fdb\u884c\u51b7\u70ed\u5206\u5c42\u5b58\u50a8\u3002 \u83b7\u53d6 \u00a7 \u53ef\u4ee5\u4ece JindoData \u4e0b\u8f7d\u9875\u9762 \u83b7\u53d6 JindoTable \u5de5\u5177\uff1a 1. \u4e0b\u8f7d jindosdk-<version>-<platform>.tar.gz \uff0c\u6839\u636e\u7cfb\u7edf\u9009\u62e9\u5408\u9002\u7684 <platform> \u3002 2. \u672c\u6587\u4ecb\u7ecd\u7684\u529f\u80fd\u6700\u4f4e\u7248\u672c\u8981\u6c42\u4e3a <version> 6.1.2\u3002 \u6ce8\uff1a - JindoTable \u5de5\u5177\u5bf9 OSS-HDFS \u7684\u4e34\u65f6\u89e3\u51bb\uff08 -once \uff09\u652f\u6301\u8bbe\u7f6e\u89e3\u51bb\u53c2\u6570\uff08 -d/-restoreDays <restore days> \uff09\uff0c\u7248\u672c\u8981\u6c42\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0\u3002\u82e5\u5bf9\u8be5\u529f\u80fd\u6709\u9700\u8981\uff0c\u8bf7\u4f7f\u7528\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0 \u7684\u7248\u672c\u3002 \u57fa\u672c\u7528\u6cd5 \u00a7 \u6309\u7167\u4e0a\u9762\u7684\u6b65\u9aa4\u83b7\u53d6\u538b\u7f29\u5305\u540e\uff0c\u89e3\u538b\u7f29\uff0c\u4fbf\u53ef\u5728 bin/ \u76ee\u5f55\u4e0b\u627e\u5230\u540d\u4e3a jindotable \u7684\u811a\u672c\u3002\u6267\u884c\u811a\u672c\uff0c\u5373\u53ef\u4f7f\u7528 JindoTable \u63d0\u4f9b\u7684\u5404\u79cd\u547d\u4ee4\u3002 \u6267\u884c jindotable -usage \u53ef\u4ee5\u663e\u793a\u5de5\u5177\u652f\u6301\u7684\u5404\u79cd\u547d\u4ee4\u3002 \u4e00\u4e9b\u6ce8\u610f\u4e8b\u9879\uff1a - \u6267\u884c\u65f6\uff0c\u987b\u4fdd\u6301\u89e3\u538b\u7f29\u540e\u7684\u76ee\u5f55\u7ed3\u6784\u3002 - \u9700\u8981\u96c6\u7fa4\u4e0a\u6709 Hadoop \u548c Hive \u73af\u5883\uff0c\u76f8\u5173\u7684\u73af\u5883\u53d8\u91cf\u5df2\u7ecf\u88ab\u6b63\u786e\u914d\u7f6e\u3002 \u5206\u5c42\u5b58\u50a8\u547d\u4ee4 \u00a7 \u652f\u6301 OSS-HDFS \u7ed3\u6784\u5316\u6570\u636e\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8\u529f\u80fd\uff0c\u547d\u4ee4\u4e3a -setStoragePolicy \uff0c\u6267\u884c jindotable -help setStoragePolicy \u53ef\u4ee5\u663e\u793a\u5e2e\u52a9\u3002 \u53c2\u6570\u8bf4\u660e \u00a7 jindotable -setStoragePolicy -t <dbName.tableName> -std/-ia/-archive/-coldArchive/-once [-c \"<condition>\" | -fullTable] [-d/-restoreDays <restore days>] [-b/-before <before days>] [-p/-parallel <parallelism>] [-mr/-mapReduce] [-e/-explain] [-w/-workingDir <working directory>] [-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u76ee\u6807\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a\u6570\u636e\u5e93\u540d.\u8868\u540d\u3002\u6570\u636e\u5e93\u548c\u8868\u540d\u4e4b\u95f4\u4ee5\u534a\u89d2\u53e5\u53f7\uff08.\uff09\u5206\u9694\u3002\u8868\u53ef\u4ee5\u662f\u5206\u533a\u8868\u6216\u975e\u5206\u533a\u8868\u3002 \u662f -std/-ia/-archive/-coldArchive/-once \u76ee\u6807\u5b58\u50a8\u65b9\u5f0f\u3002\u652f\u6301\u5982\u4e0b\u7c7b\u578b\uff1a-std\uff1a\u6807\u51c6\u5b58\u50a8\uff0c-ia\uff1a\u4f4e\u9891\u5b58\u50a8\uff0c-archive\uff1a\u5f52\u6863\u5b58\u50a8\uff0c-coldArchive\uff1a\u51b7\u5f52\u6863\u5b58\u50a8\uff0c-once\uff1a\u4e34\u65f6\u89e3\u51bb\u3002 \u662f -c \"<condition>\" / -fullTable -fullTable \u548c -c \"<condition>\" \u53ea\u9700\u63d0\u4f9b\u4e00\u4e2a\uff0c\u5373\u8981\u4e48\u6307\u5b9a -c \"<condition>\" \uff0c\u8981\u4e48\u6307\u5b9a -fullTable \u3002\u6307\u5b9a -fullTable \u65f6\uff0c\u5219\u4e3a\u79fb\u52a8\u6574\u8868\uff0c\u65e2\u53ef\u4ee5\u662f\u975e\u5206\u533a\u8868\u4e5f\u53ef\u4ee5\u662f\u5206\u533a\u8868\u3002\u6307\u5b9a -c \"<condition>\" \u65f6\uff0c\u5219\u63d0\u4f9b\u4e86\u4e00\u4e2a\u8fc7\u6ee4\u6761\u4ef6\uff0c\u7528\u6765\u9009\u62e9\u5e0c\u671b\u79fb\u52a8\u7684\u5206\u533a\uff0c\u652f\u6301\u5e38\u89c1\u8fd0\u7b97\u7b26\uff0c\u4f8b\u5982\u5927\u4e8e\u53f7\uff08>\uff09\u3002\u4f8b\u5982\uff0c\u6570\u636e\u7c7b\u578b\u4e3aString\u7684\u5206\u533ads\uff0c\u5e0c\u671b\u5206\u533a\u540d\u5927\u4e8e 'd'\uff0c\u5219\u4ee3\u7801\u4e3a-c \" ds > 'd' \"\u3002 \u662f -d/-restoreDays <restore days> \u6307\u5b9a\u4e34\u65f6\u89e3\u51bb\u7684\u5929\u6570\uff0c\u4ec5\u5bf9\u4e34\u65f6\u89e3\u51bb -once \u751f\u6548\u3002 \u5426 -b/before <before days> \u53ea\u6709\u521b\u5efa\u65f6\u95f4\u8ddd\u79bb\u73b0\u5728\u8d85\u8fc7\u4e00\u5b9a\u5929\u6570\u7684\u8868\u6216\u5206\u533a\u624d\u4f1a\u88ab\u6267\u884c\u3002 \u5426 -p/-parallel <parallelism> \u547d\u4ee4\u6267\u884c\u7684\u5e76\u884c\u5ea6\u3002 \u5426 -mr/-mapReduce \u4f7f\u7528Hadoop MapReduce\u800c\u975e\u672c\u5730\u591a\u7ebf\u7a0b\u6765\u6267\u884c\u547d\u4ee4\u3002 \u5426 -e/-explain \u5982\u679c\u51fa\u73b0\u8be5\u9009\u9879\uff0c\u5219\u4e3a\u89e3\u91ca\uff08explain\uff09\u6a21\u5f0f\uff0c\u53ea\u4f1a\u663e\u793a\u5f85\u64cd\u4f5c\u7684\u5206\u533a\u5217\u8868\uff0c\u800c\u4e0d\u4f1a\u771f\u6b63\u6267\u884c\u547d\u4ee4\u3002 \u5426 -w/-workingDir <working directory> \u53ea\u5728MapReduce\u4f5c\u4e1a\u65f6\u4f7f\u7528\uff0c\u4e3aMapReduce\u4f5c\u4e1a\u7684\u5de5\u4f5c\u76ee\u5f55\u3002\u5fc5\u987b\u6709\u8bfb\u5199\u6743\u9650\uff0c\u5de5\u4f5c\u76ee\u5f55\u53ef\u4ee5\u975e\u7a7a\uff08\u4f5c\u4e1a\u6267\u884c\u8fc7\u7a0b\u4e2d\u4f1a\u521b\u5efa\u4e34\u65f6\u6587\u4ef6\uff0c\u6267\u884c\u5b8c\u6bd5\u4f1a\u6e05\u7406\u4e34\u65f6\u6587\u4ef6\uff09\u3002 \u5426 -l/-logDir <log directory> \u6307\u5b9aLog\u6587\u4ef6\u7684\u76ee\u5f55\u3002 \u5426 \u6ce8\u610f\u4e8b\u9879 \u00a7 JindoTable \u63d0\u4f9b\u7684\u4e0a\u8ff0\u529f\u80fd\u57fa\u672c\u5bf9\u5e94\u4e8e OSS-HDFS \u7ebf\u4e0a\u670d\u52a1\u7684\u5206\u5c42\u5b58\u50a8\u80fd\u529b\uff0c\u6709\u4e0b\u5217\u4e8b\u9879\u9700\u8981\u6ce8\u610f\uff1a 1. \u4ec5\u652f\u6301 OSS-HDFS \u5b58\u50a8\u7684\u8868\u6216\u8005\u5206\u533a\u3002\u5bf9\u666e\u901a OSS \u6570\u636e\u7684\u5206\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoTable \u63d0\u4f9b\u7684 archiveTable \u4e0e unarchiveTable \u547d\u4ee4\u3002 2. \u6210\u529f\u6267\u884c -std/-ia/-archive/-coldArchive/-once \u6307\u5b9a\u7684\u8f6c\u6362\u5b58\u50a8\u7c7b\u578b\u547d\u4ee4\u540e\uff0c\u5fc5\u987b\u7b49\u5f85 2 \u5929\u624d\u53ef\u518d\u6b21\u8f6c\u6362\u5b58\u50a8\u7c7b\u578b\u3002 3. \u6807\u51c6\u548c\u4f4e\u9891\u7c7b\u578b\u65e0\u6cd5\u4e34\u65f6\u89e3\u51bb\uff08 -once \uff09\u3002\u5f52\u6863\u3001\u51b7\u5f52\u6863\uff0c\u6216\u8005\u5df2\u7ecf\u5904\u4e8e\u89e3\u51bb\u72b6\u6001\u65f6\uff0c\u53ef\u4ee5\u89e3\u51bb\uff0c\u4f46\u5982\u4e0a\u6240\u8ff0\uff0c\u95f4\u9694\u987b\u8d85\u8fc7 2 \u5929\u3002 4. \u4e34\u65f6\u89e3\u51bb\u547d\u4ee4\u8fd4\u56de\u540e\uff0c\u6570\u636e\u4ecd\u4e0d\u80fd\u7acb\u5373\u53ef\u8bfb\uff0c\u9700\u8981\u7b49\u5f85\u8fdc\u7aef\u670d\u52a1\u5b8c\u6210\u6570\u636e\u7684\u89e3\u51bb\u3002\u5bf9\u4e8e\u5f52\u6863\u6570\u636e\uff0c\u901a\u5e38\u9700\u8981\u6570\u5206\u949f\uff1b\u5bf9\u4e8e\u51b7\u5f52\u6863\u6570\u636e\uff0c\u901a\u5e38\u9700\u8981\u6570\u5c0f\u65f6\u3002 5. \u4e34\u65f6\u89e3\u51bb\u6709\u671f\u9650\u9650\u5236\uff0c\u901a\u8fc7 -d/restoreDays <restore days> \u6307\u5b9a\uff0c\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4 1 \u5929\uff0c\u5230\u671f\u540e\u6570\u636e\u5c06\u56de\u5230\u4e0d\u53ef\u8bfb\u72b6\u6001\u3002\u8be5\u529f\u80fd\u8981\u6c42\u7248\u672c\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0\u3002","title":"\u4f7f\u7528 JindoTable \u5de5\u5177\u5b9e\u73b0 OSS-HDFS \u7ed3\u6784\u5316\u6570\u636e\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8"},{"location":"jindotools/jindotable_storagepolicy_restore/#jindotable-oss-hdfs","text":"","title":"\u4f7f\u7528 JindoTable \u5de5\u5177\u5b9e\u73b0 OSS-HDFS \u7ed3\u6784\u5316\u6570\u636e\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8"},{"location":"jindotools/jindotable_storagepolicy_restore/#_1","text":"JindoTable \u662f\u57fa\u4e8e JindoSDK \u7684\u7ed3\u6784\u5316\u6570\u636e\u6cbb\u7406\u5de5\u5177\uff0c\u76ee\u6807\u662f\u5728 \u201c\u8868\u201d \u6216\u8005 \u201c\u5206\u533a\u201d \u5c42\u9762\u63d0\u4f9b\u4fbf\u6377\u7684\u6570\u636e\u8bbf\u95ee\u548c\u64cd\u4f5c\u3002\u4f7f\u7528 JindoTable \u5de5\u5177\u53ef\u4ee5\u65b9\u4fbf\u5730\u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u548c\u5206\u5c42\u5b58\u50a8\u3002 \u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u4f7f\u7528 JindoTable \u5de5\u5177\u5bf9\u5b58\u50a8\u5728 OSS-HDFS \u4e0a\u7684\u7ed3\u6784\u5316\u6570\u636e\u8fdb\u884c\u51b7\u70ed\u5206\u5c42\u5b58\u50a8\u3002","title":"\u6982\u8ff0"},{"location":"jindotools/jindotable_storagepolicy_restore/#_2","text":"\u53ef\u4ee5\u4ece JindoData \u4e0b\u8f7d\u9875\u9762 \u83b7\u53d6 JindoTable \u5de5\u5177\uff1a 1. \u4e0b\u8f7d jindosdk-<version>-<platform>.tar.gz \uff0c\u6839\u636e\u7cfb\u7edf\u9009\u62e9\u5408\u9002\u7684 <platform> \u3002 2. \u672c\u6587\u4ecb\u7ecd\u7684\u529f\u80fd\u6700\u4f4e\u7248\u672c\u8981\u6c42\u4e3a <version> 6.1.2\u3002 \u6ce8\uff1a - JindoTable \u5de5\u5177\u5bf9 OSS-HDFS \u7684\u4e34\u65f6\u89e3\u51bb\uff08 -once \uff09\u652f\u6301\u8bbe\u7f6e\u89e3\u51bb\u53c2\u6570\uff08 -d/-restoreDays <restore days> \uff09\uff0c\u7248\u672c\u8981\u6c42\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0\u3002\u82e5\u5bf9\u8be5\u529f\u80fd\u6709\u9700\u8981\uff0c\u8bf7\u4f7f\u7528\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0 \u7684\u7248\u672c\u3002","title":"\u83b7\u53d6"},{"location":"jindotools/jindotable_storagepolicy_restore/#_3","text":"\u6309\u7167\u4e0a\u9762\u7684\u6b65\u9aa4\u83b7\u53d6\u538b\u7f29\u5305\u540e\uff0c\u89e3\u538b\u7f29\uff0c\u4fbf\u53ef\u5728 bin/ \u76ee\u5f55\u4e0b\u627e\u5230\u540d\u4e3a jindotable \u7684\u811a\u672c\u3002\u6267\u884c\u811a\u672c\uff0c\u5373\u53ef\u4f7f\u7528 JindoTable \u63d0\u4f9b\u7684\u5404\u79cd\u547d\u4ee4\u3002 \u6267\u884c jindotable -usage \u53ef\u4ee5\u663e\u793a\u5de5\u5177\u652f\u6301\u7684\u5404\u79cd\u547d\u4ee4\u3002 \u4e00\u4e9b\u6ce8\u610f\u4e8b\u9879\uff1a - \u6267\u884c\u65f6\uff0c\u987b\u4fdd\u6301\u89e3\u538b\u7f29\u540e\u7684\u76ee\u5f55\u7ed3\u6784\u3002 - \u9700\u8981\u96c6\u7fa4\u4e0a\u6709 Hadoop \u548c Hive \u73af\u5883\uff0c\u76f8\u5173\u7684\u73af\u5883\u53d8\u91cf\u5df2\u7ecf\u88ab\u6b63\u786e\u914d\u7f6e\u3002","title":"\u57fa\u672c\u7528\u6cd5"},{"location":"jindotools/jindotable_storagepolicy_restore/#_4","text":"\u652f\u6301 OSS-HDFS \u7ed3\u6784\u5316\u6570\u636e\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8\u529f\u80fd\uff0c\u547d\u4ee4\u4e3a -setStoragePolicy \uff0c\u6267\u884c jindotable -help setStoragePolicy \u53ef\u4ee5\u663e\u793a\u5e2e\u52a9\u3002","title":"\u5206\u5c42\u5b58\u50a8\u547d\u4ee4"},{"location":"jindotools/jindotable_storagepolicy_restore/#_5","text":"jindotable -setStoragePolicy -t <dbName.tableName> -std/-ia/-archive/-coldArchive/-once [-c \"<condition>\" | -fullTable] [-d/-restoreDays <restore days>] [-b/-before <before days>] [-p/-parallel <parallelism>] [-mr/-mapReduce] [-e/-explain] [-w/-workingDir <working directory>] [-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u76ee\u6807\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a\u6570\u636e\u5e93\u540d.\u8868\u540d\u3002\u6570\u636e\u5e93\u548c\u8868\u540d\u4e4b\u95f4\u4ee5\u534a\u89d2\u53e5\u53f7\uff08.\uff09\u5206\u9694\u3002\u8868\u53ef\u4ee5\u662f\u5206\u533a\u8868\u6216\u975e\u5206\u533a\u8868\u3002 \u662f -std/-ia/-archive/-coldArchive/-once \u76ee\u6807\u5b58\u50a8\u65b9\u5f0f\u3002\u652f\u6301\u5982\u4e0b\u7c7b\u578b\uff1a-std\uff1a\u6807\u51c6\u5b58\u50a8\uff0c-ia\uff1a\u4f4e\u9891\u5b58\u50a8\uff0c-archive\uff1a\u5f52\u6863\u5b58\u50a8\uff0c-coldArchive\uff1a\u51b7\u5f52\u6863\u5b58\u50a8\uff0c-once\uff1a\u4e34\u65f6\u89e3\u51bb\u3002 \u662f -c \"<condition>\" / -fullTable -fullTable \u548c -c \"<condition>\" \u53ea\u9700\u63d0\u4f9b\u4e00\u4e2a\uff0c\u5373\u8981\u4e48\u6307\u5b9a -c \"<condition>\" \uff0c\u8981\u4e48\u6307\u5b9a -fullTable \u3002\u6307\u5b9a -fullTable \u65f6\uff0c\u5219\u4e3a\u79fb\u52a8\u6574\u8868\uff0c\u65e2\u53ef\u4ee5\u662f\u975e\u5206\u533a\u8868\u4e5f\u53ef\u4ee5\u662f\u5206\u533a\u8868\u3002\u6307\u5b9a -c \"<condition>\" \u65f6\uff0c\u5219\u63d0\u4f9b\u4e86\u4e00\u4e2a\u8fc7\u6ee4\u6761\u4ef6\uff0c\u7528\u6765\u9009\u62e9\u5e0c\u671b\u79fb\u52a8\u7684\u5206\u533a\uff0c\u652f\u6301\u5e38\u89c1\u8fd0\u7b97\u7b26\uff0c\u4f8b\u5982\u5927\u4e8e\u53f7\uff08>\uff09\u3002\u4f8b\u5982\uff0c\u6570\u636e\u7c7b\u578b\u4e3aString\u7684\u5206\u533ads\uff0c\u5e0c\u671b\u5206\u533a\u540d\u5927\u4e8e 'd'\uff0c\u5219\u4ee3\u7801\u4e3a-c \" ds > 'd' \"\u3002 \u662f -d/-restoreDays <restore days> \u6307\u5b9a\u4e34\u65f6\u89e3\u51bb\u7684\u5929\u6570\uff0c\u4ec5\u5bf9\u4e34\u65f6\u89e3\u51bb -once \u751f\u6548\u3002 \u5426 -b/before <before days> \u53ea\u6709\u521b\u5efa\u65f6\u95f4\u8ddd\u79bb\u73b0\u5728\u8d85\u8fc7\u4e00\u5b9a\u5929\u6570\u7684\u8868\u6216\u5206\u533a\u624d\u4f1a\u88ab\u6267\u884c\u3002 \u5426 -p/-parallel <parallelism> \u547d\u4ee4\u6267\u884c\u7684\u5e76\u884c\u5ea6\u3002 \u5426 -mr/-mapReduce \u4f7f\u7528Hadoop MapReduce\u800c\u975e\u672c\u5730\u591a\u7ebf\u7a0b\u6765\u6267\u884c\u547d\u4ee4\u3002 \u5426 -e/-explain \u5982\u679c\u51fa\u73b0\u8be5\u9009\u9879\uff0c\u5219\u4e3a\u89e3\u91ca\uff08explain\uff09\u6a21\u5f0f\uff0c\u53ea\u4f1a\u663e\u793a\u5f85\u64cd\u4f5c\u7684\u5206\u533a\u5217\u8868\uff0c\u800c\u4e0d\u4f1a\u771f\u6b63\u6267\u884c\u547d\u4ee4\u3002 \u5426 -w/-workingDir <working directory> \u53ea\u5728MapReduce\u4f5c\u4e1a\u65f6\u4f7f\u7528\uff0c\u4e3aMapReduce\u4f5c\u4e1a\u7684\u5de5\u4f5c\u76ee\u5f55\u3002\u5fc5\u987b\u6709\u8bfb\u5199\u6743\u9650\uff0c\u5de5\u4f5c\u76ee\u5f55\u53ef\u4ee5\u975e\u7a7a\uff08\u4f5c\u4e1a\u6267\u884c\u8fc7\u7a0b\u4e2d\u4f1a\u521b\u5efa\u4e34\u65f6\u6587\u4ef6\uff0c\u6267\u884c\u5b8c\u6bd5\u4f1a\u6e05\u7406\u4e34\u65f6\u6587\u4ef6\uff09\u3002 \u5426 -l/-logDir <log directory> \u6307\u5b9aLog\u6587\u4ef6\u7684\u76ee\u5f55\u3002 \u5426","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"jindotools/jindotable_storagepolicy_restore/#_6","text":"JindoTable \u63d0\u4f9b\u7684\u4e0a\u8ff0\u529f\u80fd\u57fa\u672c\u5bf9\u5e94\u4e8e OSS-HDFS \u7ebf\u4e0a\u670d\u52a1\u7684\u5206\u5c42\u5b58\u50a8\u80fd\u529b\uff0c\u6709\u4e0b\u5217\u4e8b\u9879\u9700\u8981\u6ce8\u610f\uff1a 1. \u4ec5\u652f\u6301 OSS-HDFS \u5b58\u50a8\u7684\u8868\u6216\u8005\u5206\u533a\u3002\u5bf9\u666e\u901a OSS \u6570\u636e\u7684\u5206\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoTable \u63d0\u4f9b\u7684 archiveTable \u4e0e unarchiveTable \u547d\u4ee4\u3002 2. \u6210\u529f\u6267\u884c -std/-ia/-archive/-coldArchive/-once \u6307\u5b9a\u7684\u8f6c\u6362\u5b58\u50a8\u7c7b\u578b\u547d\u4ee4\u540e\uff0c\u5fc5\u987b\u7b49\u5f85 2 \u5929\u624d\u53ef\u518d\u6b21\u8f6c\u6362\u5b58\u50a8\u7c7b\u578b\u3002 3. \u6807\u51c6\u548c\u4f4e\u9891\u7c7b\u578b\u65e0\u6cd5\u4e34\u65f6\u89e3\u51bb\uff08 -once \uff09\u3002\u5f52\u6863\u3001\u51b7\u5f52\u6863\uff0c\u6216\u8005\u5df2\u7ecf\u5904\u4e8e\u89e3\u51bb\u72b6\u6001\u65f6\uff0c\u53ef\u4ee5\u89e3\u51bb\uff0c\u4f46\u5982\u4e0a\u6240\u8ff0\uff0c\u95f4\u9694\u987b\u8d85\u8fc7 2 \u5929\u3002 4. \u4e34\u65f6\u89e3\u51bb\u547d\u4ee4\u8fd4\u56de\u540e\uff0c\u6570\u636e\u4ecd\u4e0d\u80fd\u7acb\u5373\u53ef\u8bfb\uff0c\u9700\u8981\u7b49\u5f85\u8fdc\u7aef\u670d\u52a1\u5b8c\u6210\u6570\u636e\u7684\u89e3\u51bb\u3002\u5bf9\u4e8e\u5f52\u6863\u6570\u636e\uff0c\u901a\u5e38\u9700\u8981\u6570\u5206\u949f\uff1b\u5bf9\u4e8e\u51b7\u5f52\u6863\u6570\u636e\uff0c\u901a\u5e38\u9700\u8981\u6570\u5c0f\u65f6\u3002 5. \u4e34\u65f6\u89e3\u51bb\u6709\u671f\u9650\u9650\u5236\uff0c\u901a\u8fc7 -d/restoreDays <restore days> \u6307\u5b9a\uff0c\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4 1 \u5929\uff0c\u5230\u671f\u540e\u6570\u636e\u5c06\u56de\u5230\u4e0d\u53ef\u8bfb\u72b6\u6001\u3002\u8be5\u529f\u80fd\u8981\u6c42\u7248\u672c\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0\u3002","title":"\u6ce8\u610f\u4e8b\u9879"},{"location":"oss/dls_atime/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09AccessTime\u4f7f\u7528\u8bf4\u660e \u00a7 \u4ecb\u7ecd \u00a7 OSS-HDFS \u652f\u6301\u6587\u4ef6 AccessTime\uff0c \u6587\u4ef6 AccessTime \u7cbe\u5ea6\u9ed8\u8ba4\u914d\u7f6e\u4e3a1\u5c0f\u65f6\uff0c\u5177\u4f53\u914d\u7f6e\u9879\u4e3a namespace.file.accesstime.precision.millisecond = 3600000 AccessTime \u7cbe\u5ea6\u8868\u793a\u5982\u679c\u5f53\u524d\u65f6\u95f4\u548c\u6587\u4ef6\u5f53\u524d\u7684 AccessTime \u4e4b\u524d\u7684\u5dee\u503c\u8d85\u8fc7\u914d\u7f6e\u7684\u7cbe\u5ea6\u503c\uff0c\u5219\u66f4\u65b0\u6587\u4ef6\u7684AccessTime\uff0c\u5426\u5219\u4fdd\u6301\u6587\u4ef6AccessTime\u4e0d\u53d8\u3002AccessTime\u7cbe\u5ea6\u8fc7\u5c0f\u4f1a\u5bfc\u81f4\u9891\u7e41\u66f4\u65b0\u6587\u4ef6\u7684AccessTime\uff0c\u5f71\u54cd\u6587\u4ef6\u7684\u8bfb\u53d6\u6027\u80fd\u3002 \u83b7\u53d6 AccessTime \u00a7 OSS-HDFS \u6587\u4ef6\u7684AccessTime \u53ea\u80fd\u901a\u8fc7\u5143\u6570\u636e\u5bfc\u51fa\u67e5\u770b\uff0c\u5143\u6570\u636e\u5bfc\u51fa\u529f\u80fd\u53c2\u8003 \u5143\u6570\u636e\u5bfc\u51fa .","title":"OSS-HDFS \u6587\u4ef6\u8bbf\u95ee\u65f6\u95f4\u6233"},{"location":"oss/dls_atime/#oss-hdfs-jindofs-accesstime","text":"","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09AccessTime\u4f7f\u7528\u8bf4\u660e"},{"location":"oss/dls_atime/#_1","text":"OSS-HDFS \u652f\u6301\u6587\u4ef6 AccessTime\uff0c \u6587\u4ef6 AccessTime \u7cbe\u5ea6\u9ed8\u8ba4\u914d\u7f6e\u4e3a1\u5c0f\u65f6\uff0c\u5177\u4f53\u914d\u7f6e\u9879\u4e3a namespace.file.accesstime.precision.millisecond = 3600000 AccessTime \u7cbe\u5ea6\u8868\u793a\u5982\u679c\u5f53\u524d\u65f6\u95f4\u548c\u6587\u4ef6\u5f53\u524d\u7684 AccessTime \u4e4b\u524d\u7684\u5dee\u503c\u8d85\u8fc7\u914d\u7f6e\u7684\u7cbe\u5ea6\u503c\uff0c\u5219\u66f4\u65b0\u6587\u4ef6\u7684AccessTime\uff0c\u5426\u5219\u4fdd\u6301\u6587\u4ef6AccessTime\u4e0d\u53d8\u3002AccessTime\u7cbe\u5ea6\u8fc7\u5c0f\u4f1a\u5bfc\u81f4\u9891\u7e41\u66f4\u65b0\u6587\u4ef6\u7684AccessTime\uff0c\u5f71\u54cd\u6587\u4ef6\u7684\u8bfb\u53d6\u6027\u80fd\u3002","title":"\u4ecb\u7ecd"},{"location":"oss/dls_atime/#accesstime","text":"OSS-HDFS \u6587\u4ef6\u7684AccessTime \u53ea\u80fd\u901a\u8fc7\u5143\u6570\u636e\u5bfc\u51fa\u67e5\u770b\uff0c\u5143\u6570\u636e\u5bfc\u51fa\u529f\u80fd\u53c2\u8003 \u5143\u6570\u636e\u5bfc\u51fa .","title":"\u83b7\u53d6 AccessTime"},{"location":"oss/dls_checksum/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c \u00a7 (\u4ece 4.6.2 \u5f00\u59cb\u652f\u6301) \u901a\u8fc7 JindoDistCp \u4ece HDFS \u8fc1\u79fb\u5230 OSS-HDFS \u7684\u6587\u4ef6\uff0c\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u6821\u9a8c\u6587\u4ef6\u5185\u5bb9\u3002 \u6b65\u9aa41. \u00a7 \u8ba1\u7b97\u8f93\u51fa\u901a\u8fc7 JindoDistCp \u8fc1\u79fb\u7684\u76ee\u6807\u76ee\u5f55\u7684\u6587\u4ef6 checksum\u3002 \u547d\u4ee4\u683c\u5f0f \u00a7 jindo distjob -checksum --src <src> --dest <dest> --blockSize <blockSize> --recalculate \u53c2\u6570\u8bf4\u660e\uff1a --src <src> \uff1a\u5f85\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c\u7684 OSS-HDFS \u7684\u8def\u5f84\uff08\u5fc5\u586b\uff09\u3002\u6ce8\uff1asrc \u53ea\u80fd\u4e3a OSS-HDFS \u6216 HDFS \u8def\u5f84\u3002 --dest <dest> \uff1a\u8f93\u51fa checksum \u6587\u4ef6\u7684\u76ee\u6807\u8def\u5f84\uff08\u5fc5\u586b\uff09\u3002 --blockSize <blockSize> \uff1a\u6e90\u6587\u4ef6\u5199\u5165\u7684 blockSize\uff08\u53ef\u9009)\uff0c\u9ed8\u8ba4\u4e3a 134217728\u3002 --recalculate \uff1a\u662f\u5426\u8bfb\u53d6\u6587\u4ef6\u91cd\u65b0\u8ba1\u7b97 checksum\uff08\u53ef\u9009)\uff0c\u9ed8\u8ba4\u4e0d\u4f1a\u91cd\u65b0\u8ba1\u7b97\uff0c\u4ece OSS-HDFS \u8bfb\u53d6\u5199\u5165\u65f6\u8ba1\u7b97\u7684 checksum\u3002\u6ce8\uff1a\u8be5\u53c2\u6570\u53ea\u652f\u6301 --src \u4e3a OSS-HDFS\u3002 \u793a\u4f8b \u00a7 \u5047\u8bbe\u5728 OSS-HDFS \u7684 bucket \u540d\u4e3a dlsbucket\uff0c\u9700\u8981\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c\u7684\u76ee\u5f55\u4e3a oss://dlsbucket/test\uff0c\u8f93\u51fa checksum \u6587\u4ef6\u5230 OSS \u76ee\u5f55 oss://ossbucket/test-dls\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u547d\u4ee4\u4e3a\uff1a jindo distjob -checksum --src oss://dlsbucket/test --dest oss://ossbucket/test-dls --recalculate \u6b65\u9aa42. \u00a7 \u8ba1\u7b97\u8f93\u51fa JindoDistCp \u8fc1\u79fb\u7684\u6e90\u76ee\u5f55\u7684\u6587\u4ef6 checksum\u3002 \u793a\u4f8b \u00a7 \u5047\u8bbe JindoDistCp \u7684\u6e90\u76ee\u5f55\u4e3a hdfs:///test\uff0c\u8f93\u51fa checksum \u6587\u4ef6\u5230 OSS \u76ee\u5f55 oss://ossbucket/test-hdfs\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u547d\u4ee4\u4e3a\uff1a jindo distjob -checksum --src hdfs:///test --dest oss://ossbucket/test-hdfs \u6b65\u9aa43. \u00a7 \u901a\u8fc7 JindoDistCp --diff \u5bf9\u6bd4\u6b65\u9aa41\u548c\u6b65\u9aa42\u8f93\u51fa\u7684\u6587\u4ef6 checksum \u76ee\u5f55 \u793a\u4f8b \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src oss://ossbucket/test-hdfs --dest oss://ossbucket/test-dls --diff","title":"OSS-HDFS \u6587\u4ef6\u6821\u9a8c"},{"location":"oss/dls_checksum/#oss-hdfs-jindofs","text":"(\u4ece 4.6.2 \u5f00\u59cb\u652f\u6301) \u901a\u8fc7 JindoDistCp \u4ece HDFS \u8fc1\u79fb\u5230 OSS-HDFS \u7684\u6587\u4ef6\uff0c\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u6821\u9a8c\u6587\u4ef6\u5185\u5bb9\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c"},{"location":"oss/dls_checksum/#1","text":"\u8ba1\u7b97\u8f93\u51fa\u901a\u8fc7 JindoDistCp \u8fc1\u79fb\u7684\u76ee\u6807\u76ee\u5f55\u7684\u6587\u4ef6 checksum\u3002","title":"\u6b65\u9aa41."},{"location":"oss/dls_checksum/#_1","text":"jindo distjob -checksum --src <src> --dest <dest> --blockSize <blockSize> --recalculate \u53c2\u6570\u8bf4\u660e\uff1a --src <src> \uff1a\u5f85\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c\u7684 OSS-HDFS \u7684\u8def\u5f84\uff08\u5fc5\u586b\uff09\u3002\u6ce8\uff1asrc \u53ea\u80fd\u4e3a OSS-HDFS \u6216 HDFS \u8def\u5f84\u3002 --dest <dest> \uff1a\u8f93\u51fa checksum \u6587\u4ef6\u7684\u76ee\u6807\u8def\u5f84\uff08\u5fc5\u586b\uff09\u3002 --blockSize <blockSize> \uff1a\u6e90\u6587\u4ef6\u5199\u5165\u7684 blockSize\uff08\u53ef\u9009)\uff0c\u9ed8\u8ba4\u4e3a 134217728\u3002 --recalculate \uff1a\u662f\u5426\u8bfb\u53d6\u6587\u4ef6\u91cd\u65b0\u8ba1\u7b97 checksum\uff08\u53ef\u9009)\uff0c\u9ed8\u8ba4\u4e0d\u4f1a\u91cd\u65b0\u8ba1\u7b97\uff0c\u4ece OSS-HDFS \u8bfb\u53d6\u5199\u5165\u65f6\u8ba1\u7b97\u7684 checksum\u3002\u6ce8\uff1a\u8be5\u53c2\u6570\u53ea\u652f\u6301 --src \u4e3a OSS-HDFS\u3002","title":"\u547d\u4ee4\u683c\u5f0f"},{"location":"oss/dls_checksum/#_2","text":"\u5047\u8bbe\u5728 OSS-HDFS \u7684 bucket \u540d\u4e3a dlsbucket\uff0c\u9700\u8981\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c\u7684\u76ee\u5f55\u4e3a oss://dlsbucket/test\uff0c\u8f93\u51fa checksum \u6587\u4ef6\u5230 OSS \u76ee\u5f55 oss://ossbucket/test-dls\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u547d\u4ee4\u4e3a\uff1a jindo distjob -checksum --src oss://dlsbucket/test --dest oss://ossbucket/test-dls --recalculate","title":"\u793a\u4f8b"},{"location":"oss/dls_checksum/#2","text":"\u8ba1\u7b97\u8f93\u51fa JindoDistCp \u8fc1\u79fb\u7684\u6e90\u76ee\u5f55\u7684\u6587\u4ef6 checksum\u3002","title":"\u6b65\u9aa42."},{"location":"oss/dls_checksum/#_3","text":"\u5047\u8bbe JindoDistCp \u7684\u6e90\u76ee\u5f55\u4e3a hdfs:///test\uff0c\u8f93\u51fa checksum \u6587\u4ef6\u5230 OSS \u76ee\u5f55 oss://ossbucket/test-hdfs\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u547d\u4ee4\u4e3a\uff1a jindo distjob -checksum --src hdfs:///test --dest oss://ossbucket/test-hdfs","title":"\u793a\u4f8b"},{"location":"oss/dls_checksum/#3","text":"\u901a\u8fc7 JindoDistCp --diff \u5bf9\u6bd4\u6b65\u9aa41\u548c\u6b65\u9aa42\u8f93\u51fa\u7684\u6587\u4ef6 checksum \u76ee\u5f55","title":"\u6b65\u9aa43."},{"location":"oss/dls_checksum/#_4","text":"hadoop jar jindo-distcp-tool-${version}.jar --src oss://ossbucket/test-hdfs --dest oss://ossbucket/test-dls --diff","title":"\u793a\u4f8b"},{"location":"oss/dls_dump_inventory_howto/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5143\u6570\u636e\u5bfc\u51fa\u4f7f\u7528\u8bf4\u660e \u00a7 (\u4ece 4.6.0 \u5f00\u59cb\u652f\u6301) \u4ecb\u7ecd \u00a7 \u4f7f\u7528\u5143\u6570\u636e\u5bfc\u51fa\u529f\u80fd\uff0c\u53ef\u4ee5\u5c06\u5f53\u524d OSS-HDFS bucket \u4e0b\u7684\u6587\u4ef6\u5143\u6570\u636e\u6e05\u5355\u5bfc\u51fa\u5230 OSS \u4e0a\u7684\uff0c\u683c\u5f0f\u4e3a json \u6587\u4ef6\uff0c\u65b9\u4fbf\u7528\u6237\u5bf9\u5143\u6570\u636e\u8fdb\u884c\u7edf\u8ba1\u5206\u6790 \u914d\u7f6e JindoFS \u547d\u4ee4\u884c\u5de5\u5177\uff0c\u914d\u7f6e\u5bf9\u5e94 OSS-HDFS bucket \u7684\u8bbf\u95ee\u5bc6\u94a5\uff0c\u53c2\u8003 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u8bf4\u660e \u6267\u884c\u5bfc\u51fa\u547d\u4ee4 ./jindofs admin -dumpInventory oss://<hdfs_bucket>/ \u6b64\u65f6\u53ef\u4ee5\u89c2\u5bdf\u5230\u8f93\u51fa\u8def\u5f84 ============Dump Inventory============= Job Id: 2ce40fba-5704-45c4-8720-d92a891d5cfd Data Location: oss://<oss_bucket>/.dlsdata/.sysinfo/meta_analyze/inventory/1666584461201.2ce40fba-5704-45c4-8720-d92a891d5cfd ..................................................................................................................... FINISHED. \u8be5\u547d\u4ee4\u4e3a\u963b\u585e\u547d\u4ee4\uff0c\u8bf7\u8010\u5fc3\u7b49\u5f8510\u79d2\u949f~10\u5206\u949f\uff08\u6839\u636e\u5143\u6570\u636e\u91cf\u5927\u5c0f\uff09\uff0c\u77e5\u9053\u6700\u540e\u8f93\u51fa FINISHED \u8868\u793a\u5bfc\u51fa\u6210\u529f\u3002 \u4e0b\u8f7d\u7ed3\u679c\u6587\u4ef6 \u4f7f\u7528 jindo \u547d\u4ee4\u884c\u5de5\u5177 \uff08\u914d\u7f6e\u666e\u901aOSS\u7684key\u975eOSS-HDFS\uff09 \u3001\u6216 ossutil \u3001\u6216\u4f7f\u7528Hadoop fs \u547d\u4ee4\u3001\u6216\u5728OSS\u63a7\u5236\u53f0\u9875\u9762\uff0c\u4e0b\u8f7d\u7ed3\u679c\u6587\u4ef6\u3002 ossutil cp oss://<oss_bucket>/.dlsdata/.sysinfo/meta_analyze/inventory/1666584461201.2ce40fba-5704-45c4-8720-d92a891d5cfd ./ \u4e0b\u8f7d\u5230\u672c\u5730\uff0c\u4f7f\u7528vi/vim\u6253\u5f00\u5373\u53ef\u3002 \u793a\u4f8b\u7ed3\u679c\u53c2\u8003 {\"id\":16385,\"path\":\"/\",\"type\":\"directory\",\"size\":0,\"user\":\"admin\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1666581702933,\"permission\":511,\"state\":1} {\"id\":6246684106789500068,\"path\":\"/dls-1000326249\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500069,\"path\":\"/dls-1000326249/benchmark\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500070,\"path\":\"/dls-1000326249/benchmark/n1\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500071,\"path\":\"/dls-1000326249/benchmark/n1/490747449\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660895613953,\"permission\":511,\"state\":0}","title":"OSS-HDFS \u6e05\u5355\u5bfc\u51fa"},{"location":"oss/dls_dump_inventory_howto/#oss-hdfs-jindofs","text":"(\u4ece 4.6.0 \u5f00\u59cb\u652f\u6301)","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5143\u6570\u636e\u5bfc\u51fa\u4f7f\u7528\u8bf4\u660e"},{"location":"oss/dls_dump_inventory_howto/#_1","text":"\u4f7f\u7528\u5143\u6570\u636e\u5bfc\u51fa\u529f\u80fd\uff0c\u53ef\u4ee5\u5c06\u5f53\u524d OSS-HDFS bucket \u4e0b\u7684\u6587\u4ef6\u5143\u6570\u636e\u6e05\u5355\u5bfc\u51fa\u5230 OSS \u4e0a\u7684\uff0c\u683c\u5f0f\u4e3a json \u6587\u4ef6\uff0c\u65b9\u4fbf\u7528\u6237\u5bf9\u5143\u6570\u636e\u8fdb\u884c\u7edf\u8ba1\u5206\u6790 \u914d\u7f6e JindoFS \u547d\u4ee4\u884c\u5de5\u5177\uff0c\u914d\u7f6e\u5bf9\u5e94 OSS-HDFS bucket \u7684\u8bbf\u95ee\u5bc6\u94a5\uff0c\u53c2\u8003 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u8bf4\u660e \u6267\u884c\u5bfc\u51fa\u547d\u4ee4 ./jindofs admin -dumpInventory oss://<hdfs_bucket>/ \u6b64\u65f6\u53ef\u4ee5\u89c2\u5bdf\u5230\u8f93\u51fa\u8def\u5f84 ============Dump Inventory============= Job Id: 2ce40fba-5704-45c4-8720-d92a891d5cfd Data Location: oss://<oss_bucket>/.dlsdata/.sysinfo/meta_analyze/inventory/1666584461201.2ce40fba-5704-45c4-8720-d92a891d5cfd ..................................................................................................................... FINISHED. \u8be5\u547d\u4ee4\u4e3a\u963b\u585e\u547d\u4ee4\uff0c\u8bf7\u8010\u5fc3\u7b49\u5f8510\u79d2\u949f~10\u5206\u949f\uff08\u6839\u636e\u5143\u6570\u636e\u91cf\u5927\u5c0f\uff09\uff0c\u77e5\u9053\u6700\u540e\u8f93\u51fa FINISHED \u8868\u793a\u5bfc\u51fa\u6210\u529f\u3002 \u4e0b\u8f7d\u7ed3\u679c\u6587\u4ef6 \u4f7f\u7528 jindo \u547d\u4ee4\u884c\u5de5\u5177 \uff08\u914d\u7f6e\u666e\u901aOSS\u7684key\u975eOSS-HDFS\uff09 \u3001\u6216 ossutil \u3001\u6216\u4f7f\u7528Hadoop fs \u547d\u4ee4\u3001\u6216\u5728OSS\u63a7\u5236\u53f0\u9875\u9762\uff0c\u4e0b\u8f7d\u7ed3\u679c\u6587\u4ef6\u3002 ossutil cp oss://<oss_bucket>/.dlsdata/.sysinfo/meta_analyze/inventory/1666584461201.2ce40fba-5704-45c4-8720-d92a891d5cfd ./ \u4e0b\u8f7d\u5230\u672c\u5730\uff0c\u4f7f\u7528vi/vim\u6253\u5f00\u5373\u53ef\u3002 \u793a\u4f8b\u7ed3\u679c\u53c2\u8003 {\"id\":16385,\"path\":\"/\",\"type\":\"directory\",\"size\":0,\"user\":\"admin\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1666581702933,\"permission\":511,\"state\":1} {\"id\":6246684106789500068,\"path\":\"/dls-1000326249\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500069,\"path\":\"/dls-1000326249/benchmark\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500070,\"path\":\"/dls-1000326249/benchmark/n1\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500071,\"path\":\"/dls-1000326249/benchmark/n1/490747449\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660895613953,\"permission\":511,\"state\":0}","title":"\u4ecb\u7ecd"},{"location":"oss/dls_protected_directories/","text":"\u80cc\u666f \u00a7 JindoFS \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u5f53\u524d\u53ef\u4ee5\u901a\u8fc7\u547d\u4ee4\u884c\u547d\u4ee4\u7684\u65b9\u5f0f\u8fdb\u884c\u64cd\u4f5c\uff0c\u53ef\u4ee5\u663e\u793a\u6b63\u5728\u4fdd\u62a4\u7684\u76ee\u5f55\u5217\u8868\uff0c\u6216\u8005\u4fee\u6539\u4fdd\u62a4\u7684\u76ee\u5f55\u3002\u5c06\u6765\u8fd8\u4f1a\u901a\u8fc7\u4ea7\u54c1\u754c\u9762\u7684\u65b9\u5f0f\uff0c\u63d0\u4f9b\u66f4\u52a0\u4fbf\u6377\u6613\u7528\u7684\u67e5\u770b\u548c\u8bbe\u7f6e\u65b9\u6cd5\u3002 \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u63cf\u8ff0 \u00a7 JindoFS \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\uff0c\u4e3b\u8981\u9632\u6b62\u7528\u6237\u8bef\u5220\u76ee\u5f55\uff0c\u88ab\u4fdd\u62a4\u7684\u76ee\u5f55\u53ea\u8981\u8be5\u76ee\u5f55\u4e0b\u5b58\u5728\u76ee\u5f55\u6216\u8005\u6587\u4ef6\uff0c\u4fdd\u62a4\u76ee\u5f55\u7684\u5220\u9664\u4ee5\u53ca\u91cd\u547d\u540d\u64cd\u4f5c\u4f1a\u62a5\u9519\uff0c\u53ea\u6709\u5f53\u88ab\u4fdd\u62a4\u7684\u76ee\u5f55\u4e0b\u4e0d\u5b58\u5728\u4efb\u4f55\u76ee\u5f55\u6216\u8005\u6587\u4ef6\uff0c\u4fdd\u62a4\u76ee\u5f55\u624d\u4f1a\u88ab\u5141\u8bb8\u5220\u9664\u6216\u8005\u91cd\u547d\u540d\u3002 JindoFS \u547d\u4ee4\u884c\u7b80\u4ecb \u00a7 \u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u9700\u8981\u7528\u5230\u7684\u662f -putConfig \u4e0e -getConfig \u547d\u4ee4\uff0c\u5177\u4f53\u7528\u6cd5\u5c06\u5728\u4e0b\u6587\u4ecb\u7ecd\u3002 \u914d\u7f6e\u8bbf\u95ee AK \u00a7 \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u9700\u8981\u914d\u7f6e\u8fde\u63a5 OSS-HDFS \u6240\u9700\u7684 AK \u4fe1\u606f\u3002\u6709\u4e24\u79cd\u65b9\u6cd5\u53ef\u4ee5\u8fdb\u884c\u914d\u7f6e\uff1a \u914d\u7f6e\u6587\u4ef6\uff08\u63a8\u8350\uff09 \u00a7 \u8bbe\u7f6e\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6307\u5411\u67d0\u4e2a\u76ee\u5f55\uff0c\u7136\u540e\u5728\u76ee\u5f55\u4e0b\u521b\u5efa\u540d\u4e3a jindofs.cfg \u7684\u6587\u4ef6\uff0c\u7136\u540e\u5728\u6587\u4ef6\u91cc\u5199\u5165\u5982\u4e0b\u5185\u5bb9\uff1a [client] fs.oss.accessKeyId = <key> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK key fs.oss.accessKeySecret = <secret> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK secret fs.oss.bucket.<bucket>.accessKeyId = <key> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK key\uff0c\u4f18\u5148\u7ea7\u9ad8\u4e8e\u9ed8\u8ba4 fs.oss.bucket.<bucket>.accessKeySecret = <secret> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK secret --extraConf \u00a7 \u547d\u4ee4\u884c\u5de5\u5177\u7684\u6240\u6709\u6307\u4ee4\u5747\u53ef\u901a\u8fc7 --extraConf \u9009\u9879\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u914d\u7f6e\u6587\u4ef6\uff0c\u4f8b\u5982\uff1a jindofs admin -getConfig -dlsUri <path> -name <keys> \\ --extraConf fs.oss.accessKeyId=<AK key> \\ --extraConf fs.oss.accessKeySecret=<AK secret> \u5c06\u7ed9\u547d\u4ee4\u63d0\u4f9b\u989d\u5916\u7684 AK key \u4e0e AK secret \u914d\u7f6e\u3002 \u8bbe\u7f6e\u4fdd\u62a4\u76ee\u5f55 \u00a7 \u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf fs.protected.directories=/path/to/dir1,/path/to/dir2 \u4e0a\u8ff0\u547d\u4ee4\u5bf9\u540d\u4e3a \uff08endpoint \u4e3a \uff09\u7684\u6876\u914d\u7f6e\u4e86\u4e24\u4e2a\u4fdd\u62a4\u76ee\u5f55\uff1a /path/to/dir1 /path/to/dir2 \u5982\u793a\u4f8b\u4e2d\u7684\u683c\u5f0f\uff0c\u8fd9\u4e9b\u4fdd\u62a4\u76ee\u5f55\u7684\u8def\u5f84\u9700\u8981\u5199\u5728\u4e00\u8d77\uff0c\u9017\u53f7\u9694\u5f00\u3002\u8def\u5f84\u4e0d\u4ee5 oss:// \u5f00\u5934\uff0c\u800c\u662f\u4f7f\u7528\u6876\u5185\u4ee5 / \u5f00\u5934\u7684\u7edd\u5bf9\u8def\u5f84\u3002\u5982\u679c\u547d\u4ee4\u5931\u8d25\uff0c\u5c06\u663e\u793a\u5931\u8d25\u4fe1\u606f\u3002\u5982\u679c\u6ca1\u6709\u4efb\u4f55\u4fe1\u606f\u663e\u793a\uff0c\u8868\u660e\u6267\u884c\u6210\u529f\u3002\u547d\u4ee4\u6267\u884c\u6210\u529f\u4e4b\u540e\uff0c\u5bf9\u8fd9\u4e9b\u76ee\u5f55\u7684\u4fdd\u62a4\u529f\u80fd\u5c06\u5728 30 \u79d2\u5185\u6b63\u5f0f\u751f\u6548\u3002 \u67e5\u770b\u4fdd\u62a4\u76ee\u5f55 \u00a7 \u547d\u4ee4\uff1a jindofs admin -getConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -name fs.protected.directories \u4e0a\u8ff0\u547d\u4ee4\u8fd4\u56de\u4e86\u540d\u4e3a \uff08endpoint \u4e3a \uff09\u7684\u6876\u7684\u4fdd\u62a4\u76ee\u5f55\u5217\u8868\uff0c\u8fd4\u56de\u503c\u793a\u4f8b\uff1a fs.protected.directories: /path/to/dir1,/path/to/dir2 \u8868\u660e\u8fd9\u4e9b\u76ee\u5f55\u76ee\u524d\u6b63\u5728\u4fdd\u62a4\u5f53\u4e2d\uff1a /path/to/dir1 /path/to/dir2 \u5220\u9664\u6240\u6709\u4fdd\u62a4\u76ee\u5f55 \u00a7 \u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf fs.protected.directories= \u4e0a\u8ff0\u547d\u4ee4\u5220\u9664\u4e86\u540d\u4e3a \uff08endpoint \u4e3a \uff09\u7684\u6876\u7684\u6240\u6709\u4fdd\u62a4\u76ee\u5f55\uff0c\u6b63\u5f0f\u751f\u6548\u540e\uff08\u547d\u4ee4\u8fd4\u56de\u540e 30 \u79d2\u5185\uff09\uff0c\u8be5\u6876\u5c06\u4e0d\u518d\u6709\u4efb\u4f55\u76ee\u5f55\u53d7\u5230\u4fdd\u62a4\u3002","title":"OSS-HDFS \u53d7\u4fdd\u62a4\u76ee\u5f55"},{"location":"oss/dls_protected_directories/#_1","text":"JindoFS \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u5f53\u524d\u53ef\u4ee5\u901a\u8fc7\u547d\u4ee4\u884c\u547d\u4ee4\u7684\u65b9\u5f0f\u8fdb\u884c\u64cd\u4f5c\uff0c\u53ef\u4ee5\u663e\u793a\u6b63\u5728\u4fdd\u62a4\u7684\u76ee\u5f55\u5217\u8868\uff0c\u6216\u8005\u4fee\u6539\u4fdd\u62a4\u7684\u76ee\u5f55\u3002\u5c06\u6765\u8fd8\u4f1a\u901a\u8fc7\u4ea7\u54c1\u754c\u9762\u7684\u65b9\u5f0f\uff0c\u63d0\u4f9b\u66f4\u52a0\u4fbf\u6377\u6613\u7528\u7684\u67e5\u770b\u548c\u8bbe\u7f6e\u65b9\u6cd5\u3002","title":"\u80cc\u666f"},{"location":"oss/dls_protected_directories/#_2","text":"JindoFS \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\uff0c\u4e3b\u8981\u9632\u6b62\u7528\u6237\u8bef\u5220\u76ee\u5f55\uff0c\u88ab\u4fdd\u62a4\u7684\u76ee\u5f55\u53ea\u8981\u8be5\u76ee\u5f55\u4e0b\u5b58\u5728\u76ee\u5f55\u6216\u8005\u6587\u4ef6\uff0c\u4fdd\u62a4\u76ee\u5f55\u7684\u5220\u9664\u4ee5\u53ca\u91cd\u547d\u540d\u64cd\u4f5c\u4f1a\u62a5\u9519\uff0c\u53ea\u6709\u5f53\u88ab\u4fdd\u62a4\u7684\u76ee\u5f55\u4e0b\u4e0d\u5b58\u5728\u4efb\u4f55\u76ee\u5f55\u6216\u8005\u6587\u4ef6\uff0c\u4fdd\u62a4\u76ee\u5f55\u624d\u4f1a\u88ab\u5141\u8bb8\u5220\u9664\u6216\u8005\u91cd\u547d\u540d\u3002","title":"\u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u63cf\u8ff0"},{"location":"oss/dls_protected_directories/#jindofs","text":"\u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u9700\u8981\u7528\u5230\u7684\u662f -putConfig \u4e0e -getConfig \u547d\u4ee4\uff0c\u5177\u4f53\u7528\u6cd5\u5c06\u5728\u4e0b\u6587\u4ecb\u7ecd\u3002","title":"JindoFS \u547d\u4ee4\u884c\u7b80\u4ecb"},{"location":"oss/dls_protected_directories/#ak","text":"\u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u9700\u8981\u914d\u7f6e\u8fde\u63a5 OSS-HDFS \u6240\u9700\u7684 AK \u4fe1\u606f\u3002\u6709\u4e24\u79cd\u65b9\u6cd5\u53ef\u4ee5\u8fdb\u884c\u914d\u7f6e\uff1a","title":"\u914d\u7f6e\u8bbf\u95ee AK"},{"location":"oss/dls_protected_directories/#_3","text":"\u8bbe\u7f6e\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6307\u5411\u67d0\u4e2a\u76ee\u5f55\uff0c\u7136\u540e\u5728\u76ee\u5f55\u4e0b\u521b\u5efa\u540d\u4e3a jindofs.cfg \u7684\u6587\u4ef6\uff0c\u7136\u540e\u5728\u6587\u4ef6\u91cc\u5199\u5165\u5982\u4e0b\u5185\u5bb9\uff1a [client] fs.oss.accessKeyId = <key> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK key fs.oss.accessKeySecret = <secret> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK secret fs.oss.bucket.<bucket>.accessKeyId = <key> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK key\uff0c\u4f18\u5148\u7ea7\u9ad8\u4e8e\u9ed8\u8ba4 fs.oss.bucket.<bucket>.accessKeySecret = <secret> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK secret","title":"\u914d\u7f6e\u6587\u4ef6\uff08\u63a8\u8350\uff09"},{"location":"oss/dls_protected_directories/#-extraconf","text":"\u547d\u4ee4\u884c\u5de5\u5177\u7684\u6240\u6709\u6307\u4ee4\u5747\u53ef\u901a\u8fc7 --extraConf \u9009\u9879\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u914d\u7f6e\u6587\u4ef6\uff0c\u4f8b\u5982\uff1a jindofs admin -getConfig -dlsUri <path> -name <keys> \\ --extraConf fs.oss.accessKeyId=<AK key> \\ --extraConf fs.oss.accessKeySecret=<AK secret> \u5c06\u7ed9\u547d\u4ee4\u63d0\u4f9b\u989d\u5916\u7684 AK key \u4e0e AK secret \u914d\u7f6e\u3002","title":"--extraConf"},{"location":"oss/dls_protected_directories/#_4","text":"\u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf fs.protected.directories=/path/to/dir1,/path/to/dir2 \u4e0a\u8ff0\u547d\u4ee4\u5bf9\u540d\u4e3a \uff08endpoint \u4e3a \uff09\u7684\u6876\u914d\u7f6e\u4e86\u4e24\u4e2a\u4fdd\u62a4\u76ee\u5f55\uff1a /path/to/dir1 /path/to/dir2 \u5982\u793a\u4f8b\u4e2d\u7684\u683c\u5f0f\uff0c\u8fd9\u4e9b\u4fdd\u62a4\u76ee\u5f55\u7684\u8def\u5f84\u9700\u8981\u5199\u5728\u4e00\u8d77\uff0c\u9017\u53f7\u9694\u5f00\u3002\u8def\u5f84\u4e0d\u4ee5 oss:// \u5f00\u5934\uff0c\u800c\u662f\u4f7f\u7528\u6876\u5185\u4ee5 / \u5f00\u5934\u7684\u7edd\u5bf9\u8def\u5f84\u3002\u5982\u679c\u547d\u4ee4\u5931\u8d25\uff0c\u5c06\u663e\u793a\u5931\u8d25\u4fe1\u606f\u3002\u5982\u679c\u6ca1\u6709\u4efb\u4f55\u4fe1\u606f\u663e\u793a\uff0c\u8868\u660e\u6267\u884c\u6210\u529f\u3002\u547d\u4ee4\u6267\u884c\u6210\u529f\u4e4b\u540e\uff0c\u5bf9\u8fd9\u4e9b\u76ee\u5f55\u7684\u4fdd\u62a4\u529f\u80fd\u5c06\u5728 30 \u79d2\u5185\u6b63\u5f0f\u751f\u6548\u3002","title":"\u8bbe\u7f6e\u4fdd\u62a4\u76ee\u5f55"},{"location":"oss/dls_protected_directories/#_5","text":"\u547d\u4ee4\uff1a jindofs admin -getConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -name fs.protected.directories \u4e0a\u8ff0\u547d\u4ee4\u8fd4\u56de\u4e86\u540d\u4e3a \uff08endpoint \u4e3a \uff09\u7684\u6876\u7684\u4fdd\u62a4\u76ee\u5f55\u5217\u8868\uff0c\u8fd4\u56de\u503c\u793a\u4f8b\uff1a fs.protected.directories: /path/to/dir1,/path/to/dir2 \u8868\u660e\u8fd9\u4e9b\u76ee\u5f55\u76ee\u524d\u6b63\u5728\u4fdd\u62a4\u5f53\u4e2d\uff1a /path/to/dir1 /path/to/dir2","title":"\u67e5\u770b\u4fdd\u62a4\u76ee\u5f55"},{"location":"oss/dls_protected_directories/#_6","text":"\u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf fs.protected.directories= \u4e0a\u8ff0\u547d\u4ee4\u5220\u9664\u4e86\u540d\u4e3a \uff08endpoint \u4e3a \uff09\u7684\u6876\u7684\u6240\u6709\u4fdd\u62a4\u76ee\u5f55\uff0c\u6b63\u5f0f\u751f\u6548\u540e\uff08\u547d\u4ee4\u8fd4\u56de\u540e 30 \u79d2\u5185\uff09\uff0c\u8be5\u6876\u5c06\u4e0d\u518d\u6709\u4efb\u4f55\u76ee\u5f55\u53d7\u5230\u4fdd\u62a4\u3002","title":"\u5220\u9664\u6240\u6709\u4fdd\u62a4\u76ee\u5f55"},{"location":"oss/dls_root_policy/","text":"\u4f7f\u7528 RootPolicy \u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 (\u4ece 4.6.0 \u5f00\u59cb\u652f\u6301) \u80cc\u666f \u00a7 JindoFS \u652f\u6301 RootPolicy \u89c4\u5219\uff0c\u901a\u8fc7 RootPolicy \u53ef\u4ee5\u4e3a OSS-HDFS\uff08JindoFS \u670d\u52a1\uff09\u8bbe\u7f6e\u81ea\u5b9a\u4e49\u524d\u7f00\uff0c\u8fd9\u6837\u7528\u6237\u539f\u6709\u7684\u8bbf\u95ee hdfs:// \u524d\u7f00\u7684\u4f5c\u4e1a\uff0c\u53ef\u4ee5\u4e0d\u7ecf\u4fee\u6539\u76f4\u63a5\u8fd0\u884c\u5728 JindoFS \u670d\u52a1\u4e0a\u3002 1\u3001 \u8bbe\u7f6e RootPolicy \u00a7 \u6211\u4eec\u63d0\u4f9b\u4e86\u4e00\u7ec4 Shell \u547d\u4ee4\uff0c\u5e2e\u52a9\u7528\u6237\u5411\u7279\u5b9a\u7684bucket\u6ce8\u518c\u81ea\u5b9a\u4e49\u5730\u5740\u3002\u901a\u8fc7 SetRootPolicy\u547d\u4ee4 \uff0c\u53ef\u4ee5\u4e3a\u7279\u5b9a bucket \u6ce8\u518c\u81ea\u5b9a\u4e49\u524d\u7f00\u7684\u8bbf\u95ee\u5730\u5740\u3002\u5982\uff1a ./jindofs admin -setRootPolicy oss://<bucket_name>.<dls_endpoint>/ hdfs://<your_ns_name>/ : OSS-HDFS \u670d\u52a1 bucket \u7684\u540d\u5b57\uff0c\u5f53\u524d\u7248\u672c\u4ec5\u652f\u6301\u6839\u76ee\u5f55\u3002 : OSS-HDFS \u670d\u52a1\u7684 endpoint\uff0c\u5982\u676d\u5dderegion\u4e3a cn-hangzhou.oss-dls.aliyuncs.com \u3002\u5982\u679c Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u6709\u5982\u4e0b\u914d\u7f6e\u9879 <configuration> <property> <name>fs.oss.endpoint</name> <value><dls_endpoint></value> </property> \u6216 <property> <name>fs.oss.bucket.<bucket_name>.endpoint</name> <value><dls_endpoint></value> </property> </configuration> \u4e0a\u9762\u4f8b\u5b50\u4e2d\u7684 oss://<bucket_name>.<dls_endpoint>/ \u53ef\u4ee5\u5199\u6210 oss://<bucket_name>/ \u7684\u5f62\u5f0f\uff0c\u4e0b\u6587\u4e2d\u9700\u8981 endpoint \u7684\u5730\u65b9\u90fd\u53ef\u7701\u7565\u4e0d\u586b\u3002 * : \u81ea\u5b9a\u4e49\u8bbf\u95ee hdfs \u670d\u52a1\u7684 nsname\uff0c\u4e3a\u4efb\u610f\u975e\u7a7a\u5b57\u7b26\u4e32\uff0c\u5982 test \uff0c\u5f53\u524d\u7248\u672c\u4ec5\u652f\u6301\u6839\u76ee\u5f55\u3002 2\u3001 \u914d\u7f6e access policy \u53d1\u73b0\u5730\u5740 \u548c scheme \u5b9e\u73b0\u7c7b \u00a7 \u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u6dfb\u52a0\u5982\u4e0b\u914d\u7f6e\u9879 <configuration> <property> <name>fs.accessPolicies.discovery</name> <value>oss://<bucket_name>.<dls_endpoint>/</value> </property> <property> <name>fs.AbstractFileSystem.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.HDFS</value> </property> <property> <name>fs.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.JindoHdfsFileSystem</value> </property> </configuration> \u5982\u679c\u662f\u591a\u4e2abucket\uff0c\u7528 , \u5206\u9694\u3002 3\u3001 \u9a8c\u8bc1\u548c\u4f7f\u7528 \u00a7 \u914d\u7f6e\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u5bf9\u914d\u7f6e\u8fdb\u884c\u7b80\u5355\u9a8c\u8bc1\u3002 hadoop fs -ls hdfs://<your_ns_name>/ \u9a8c\u8bc1\u901a\u8fc7\u540e\uff0c\u91cd\u542f\u76f8\u5173\u670d\u52a1\uff0c\u5982 Hive/Spark \u7b49\uff0c\u5373\u53ef\u4f7f\u7528\u81ea\u5b9a\u4e49\u524d\u7f00\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u3002 4\u3001\u5176\u4ed6\u529f\u80fd \u00a7 4.1 \u5220\u9664 RootPolicy \u89c4\u5219 \u00a7 \u901a\u8fc7 UnsetRootPolicy\u547d\u4ee4 \u53ef\u4ee5\u5220\u9664\u76f8\u5e94\u5730\u5740\u3002 ./jindofs admin -unsetRootPolicy oss://<bucket_name>.<dls_endpoint>/ hdfs://<your_ns_name>/ 4.2 \u5217\u4e3e\u6240\u6709 RootPolicy \u89c4\u5219 \u00a7 \u4f7f\u7528 ListAccessPolicies\u547d\u4ee4 \uff0c\u53ef\u4ee5\u5217\u51fa\u7279\u5b9a bucket \u5f53\u524d\u6ce8\u518c\u7684\u6240\u6709\u524d\u7f00\u5730\u5740\u3002 ./jindofs admin -listAccessPolicies oss://<bucket_name>.<dls_endpoint>/","title":"OSS-HDFS \u6839\u8def\u5f84\u6539\u5199"},{"location":"oss/dls_root_policy/#rootpolicy-oss-hdfs-jindofs","text":"(\u4ece 4.6.0 \u5f00\u59cb\u652f\u6301)","title":"\u4f7f\u7528 RootPolicy \u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"oss/dls_root_policy/#_1","text":"JindoFS \u652f\u6301 RootPolicy \u89c4\u5219\uff0c\u901a\u8fc7 RootPolicy \u53ef\u4ee5\u4e3a OSS-HDFS\uff08JindoFS \u670d\u52a1\uff09\u8bbe\u7f6e\u81ea\u5b9a\u4e49\u524d\u7f00\uff0c\u8fd9\u6837\u7528\u6237\u539f\u6709\u7684\u8bbf\u95ee hdfs:// \u524d\u7f00\u7684\u4f5c\u4e1a\uff0c\u53ef\u4ee5\u4e0d\u7ecf\u4fee\u6539\u76f4\u63a5\u8fd0\u884c\u5728 JindoFS \u670d\u52a1\u4e0a\u3002","title":"\u80cc\u666f"},{"location":"oss/dls_root_policy/#1-rootpolicy","text":"\u6211\u4eec\u63d0\u4f9b\u4e86\u4e00\u7ec4 Shell \u547d\u4ee4\uff0c\u5e2e\u52a9\u7528\u6237\u5411\u7279\u5b9a\u7684bucket\u6ce8\u518c\u81ea\u5b9a\u4e49\u5730\u5740\u3002\u901a\u8fc7 SetRootPolicy\u547d\u4ee4 \uff0c\u53ef\u4ee5\u4e3a\u7279\u5b9a bucket \u6ce8\u518c\u81ea\u5b9a\u4e49\u524d\u7f00\u7684\u8bbf\u95ee\u5730\u5740\u3002\u5982\uff1a ./jindofs admin -setRootPolicy oss://<bucket_name>.<dls_endpoint>/ hdfs://<your_ns_name>/ : OSS-HDFS \u670d\u52a1 bucket \u7684\u540d\u5b57\uff0c\u5f53\u524d\u7248\u672c\u4ec5\u652f\u6301\u6839\u76ee\u5f55\u3002 : OSS-HDFS \u670d\u52a1\u7684 endpoint\uff0c\u5982\u676d\u5dderegion\u4e3a cn-hangzhou.oss-dls.aliyuncs.com \u3002\u5982\u679c Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u6709\u5982\u4e0b\u914d\u7f6e\u9879 <configuration> <property> <name>fs.oss.endpoint</name> <value><dls_endpoint></value> </property> \u6216 <property> <name>fs.oss.bucket.<bucket_name>.endpoint</name> <value><dls_endpoint></value> </property> </configuration> \u4e0a\u9762\u4f8b\u5b50\u4e2d\u7684 oss://<bucket_name>.<dls_endpoint>/ \u53ef\u4ee5\u5199\u6210 oss://<bucket_name>/ \u7684\u5f62\u5f0f\uff0c\u4e0b\u6587\u4e2d\u9700\u8981 endpoint \u7684\u5730\u65b9\u90fd\u53ef\u7701\u7565\u4e0d\u586b\u3002 * : \u81ea\u5b9a\u4e49\u8bbf\u95ee hdfs \u670d\u52a1\u7684 nsname\uff0c\u4e3a\u4efb\u610f\u975e\u7a7a\u5b57\u7b26\u4e32\uff0c\u5982 test \uff0c\u5f53\u524d\u7248\u672c\u4ec5\u652f\u6301\u6839\u76ee\u5f55\u3002","title":"1\u3001 \u8bbe\u7f6e RootPolicy"},{"location":"oss/dls_root_policy/#2-access-policy-scheme","text":"\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u6dfb\u52a0\u5982\u4e0b\u914d\u7f6e\u9879 <configuration> <property> <name>fs.accessPolicies.discovery</name> <value>oss://<bucket_name>.<dls_endpoint>/</value> </property> <property> <name>fs.AbstractFileSystem.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.HDFS</value> </property> <property> <name>fs.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.JindoHdfsFileSystem</value> </property> </configuration> \u5982\u679c\u662f\u591a\u4e2abucket\uff0c\u7528 , \u5206\u9694\u3002","title":"2\u3001 \u914d\u7f6e access policy \u53d1\u73b0\u5730\u5740 \u548c scheme \u5b9e\u73b0\u7c7b"},{"location":"oss/dls_root_policy/#3","text":"\u914d\u7f6e\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u5bf9\u914d\u7f6e\u8fdb\u884c\u7b80\u5355\u9a8c\u8bc1\u3002 hadoop fs -ls hdfs://<your_ns_name>/ \u9a8c\u8bc1\u901a\u8fc7\u540e\uff0c\u91cd\u542f\u76f8\u5173\u670d\u52a1\uff0c\u5982 Hive/Spark \u7b49\uff0c\u5373\u53ef\u4f7f\u7528\u81ea\u5b9a\u4e49\u524d\u7f00\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u3002","title":"3\u3001 \u9a8c\u8bc1\u548c\u4f7f\u7528"},{"location":"oss/dls_root_policy/#4","text":"","title":"4\u3001\u5176\u4ed6\u529f\u80fd"},{"location":"oss/dls_root_policy/#41-rootpolicy","text":"\u901a\u8fc7 UnsetRootPolicy\u547d\u4ee4 \u53ef\u4ee5\u5220\u9664\u76f8\u5e94\u5730\u5740\u3002 ./jindofs admin -unsetRootPolicy oss://<bucket_name>.<dls_endpoint>/ hdfs://<your_ns_name>/","title":"4.1 \u5220\u9664 RootPolicy \u89c4\u5219"},{"location":"oss/dls_root_policy/#42-rootpolicy","text":"\u4f7f\u7528 ListAccessPolicies\u547d\u4ee4 \uff0c\u53ef\u4ee5\u5217\u51fa\u7279\u5b9a bucket \u5f53\u524d\u6ce8\u518c\u7684\u6240\u6709\u524d\u7f00\u5730\u5740\u3002 ./jindofs admin -listAccessPolicies oss://<bucket_name>.<dls_endpoint>/","title":"4.2 \u5217\u4e3e\u6240\u6709 RootPolicy \u89c4\u5219"},{"location":"oss/dls_snapshot_quickstart/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u7167\u4f7f\u7528\u8bf4\u660e \u00a7 (\u4ece JindoData 4.0.0 \u5f00\u59cb\u652f\u6301) \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u662fOSS\u65b0\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9HDFS\u63a5\u53e3, \u652f\u6301\u76ee\u5f55\u4ee5\u53ca\u76ee\u5f55\u5c42\u7ea7\uff0c\u901a\u8fc7Jindo SDK 4.x \u53ef\u4ee5\u517c\u5bb9\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u3002\u5173\u4e8e\u7528\u6237\u5982\u4f55\u521b\u5efa\u548c\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u7684\u57fa\u672c\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 \u94fe\u63a5 \u3002 OSS-HDFS \u670d\u52a1\u7684\u5feb\u7167\u529f\u80fd\u5728\u4f7f\u7528\u4e0a\u4e0e HDFS \u7684\u5feb\u7167\u529f\u80fd\u5b8c\u5168\u517c\u5bb9\u3002\u672c\u6587\u4e3b\u8981\u4ecb\u7ecd\u5728 OSS-HDFS \u670d\u52a1\u4e2d\u4f7f\u7528\u5feb\u7167\u529f\u80fd\u7684\u4e00\u4e9b\u5e38\u89c1\u64cd\u4f5c\u3002 \u4e3a\u4e86\u65b9\u4fbf\u4e0b\u9762\u547d\u4ee4\u7684\u4ecb\u7ecd\uff0c\u6211\u4eec\u5047\u8bbe oss://oss-dfs-test \u8fd9\u4e2aBucket\u5df2\u7ecf\u5f00\u542f OSS-HDFS \u670d\u52a1\u3002\u4e0b\u9762\u6240\u6709\u7684\u4f8b\u5b50\u90fd\u4f1a\u9488\u5bf9\u8fd9\u4e2a Bucket \u8fdb\u884c\u64cd\u4f5c\u3002 \u5f00\u542f\u5feb\u7167\u529f\u80fd \u00a7 \u5047\u8bbe\u6211\u4eec\u5148\u5728 oss://oss-dfs-test \u4e0b\u521b\u5efa\u4e00\u4e2a\u76ee\u5f55TestSnapshot\u3002 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u662f\u5173\u95ed\u7684\u3002\u9700\u8981\u5f00\u542f\u548c\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\uff0c\u9700\u8981\u4f7f\u7528 jindofs \u547d\u4ee4\u884c\u5de5\u5177 \u3002\u5177\u4f53\u7684\u5f00\u542f\u5feb\u7167\u7684\u683c\u5f0f\u4e3a ./jindofs admin -allowSnapshot -dlsUri <path> \u5982\u679c\u6211\u4eec\u8981\u5f00\u542f\u524d\u9762\u521b\u5efa\u7684 TestSnapshot \u7684\u5feb\u7167\u529f\u80fd\u3002\u53ef\u4ee5\u8f93\u5165\u5982\u4e0b\u7684\u547d\u4ee4\uff1a ./jindofs admin -allowSnapshot -dlsUri oss://oss-dfs-test.<Endpoint>/TestSnapshot \u521b\u5efa\u5feb\u7167 \u00a7 \u5728\u4e00\u4e2a\u76ee\u5f55\u5f00\u542f\u5feb\u7167\u529f\u80fd\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7 HDFS \u7684 shell \u547d\u4ee4\u3002\u5177\u4f53\u683c\u5f0f\u4e3a\uff1a hdfs dfs -createSnapshot <path> [<snapshotName>] \u4f8b\u5982\u4e0a\u9762\u4f8b\u5b50\u7684\u4e2d\u7684 TestSnapshot\uff0c\u6211\u4eec\u8fdb\u884c\u4e86\u4e00\u4e9b\u76ee\u5f55\u548c\u6587\u4ef6\u7684\u64cd\u4f5c\u3002\u7136\u540e\u53ef\u4ee5\u521b\u5efa\u4e00\u4e2a\u5feb\u7167\u5bf9\u6b64\u523b\u7684\u5feb\u7167\u7684\u72b6\u6001\u8fdb\u884c\u4fdd\u5b58\u3002\u5047\u8bbe\u6211\u4eec\u547d\u540d\u5feb\u7167\u540d\u4e3a S1\u3002 #\u4e0b\u9762\u8fd9\u4e9b\u4e3a\u521b\u5efa\u6d4b\u8bd5\u7684\u6587\u4ef6\u548c\u76ee\u5f55 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir2 hdfs dfs -touchz oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1/file1 hdfs dfs -touchz oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir3/file2 #\u901a\u8fc7HDFS\u7684shell\u547d\u4ee4\u884c\u5de5\u5177\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u521b\u5efa\u5feb\u7167\uff0c\u5e76\u5c06\u5feb\u7167\u547d\u540d\u4e3aS1\uff1a hdfs dfs -createSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S1 \u8bbf\u95ee\u5feb\u7167\u4e2d\u7684\u76ee\u5f55\u548c\u6587\u4ef6 \u00a7 \u5feb\u7167\u76ee\u5f55\u7684\u683c\u5f0f \u00a7 \u4e3a\u4e86\u533a\u522b\u4e8e\u6587\u4ef6\u7cfb\u7edf\u73b0\u6709\u7684\u76ee\u5f55\u548c\u6587\u4ef6\uff0c\u8bbf\u95ee\u5feb\u7167\u4e2d\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u9700\u8981\u6dfb\u52a0\u5feb\u7167\u7684\u4fe1\u606f\u6765\u6307\u51fa\u81ea\u5df1\u8981\u8bbf\u95ee\u7684\u5feb\u7167\u540d\u3002\u5bf9\u4e8e\u4e00\u4e2a\u5f00\u542f\u4e86\u5feb\u7167\u7684\u76ee\u5f55\uff0c\u6211\u4eec\u8981\u8bbf\u95ee\u5b83\u4e0b\u9762\u67d0\u4e2a\u5feb\u7167\u7684\u76ee\u5f55\u548c\u6587\u4ef6\uff0c\u6211\u4eec\u53ef\u4ee5\u4e0b\u9762\u7684\u683c\u5f0f\u6765\u8bbf\u95ee\uff1a <snapshotRoot>/.snapshot/<snapshotName>/<actual subPath> \u5176\u4e2d snapshotRoot \u4e3a\u5feb\u7167\u7684\u6839\u76ee\u5f55\uff0c\u4e5f\u5c31\u662f\u5f00\u542f\u5feb\u7167\u547d\u4ee4\u4e2ddlsUri\u53c2\u6570\u5236\u5b9a\u7684\u8def\u5f84\u3002snapshotName\u662f\u4e4b\u524d\u521b\u5efa\u7684\u5feb\u7167\u540d\u3002\u5236\u5b9a\u4e86\u5feb\u7167\u540d\u4e4b\u540e\uff0c\u4e4b\u540e\u7684\u8def\u5f84\u5c31\u662f\u539f\u6765\u5feb\u7167\u6839\u76ee\u5f55\uff08snapshotRoot\uff09\u4e0b\u8981\u8bbf\u95ee\u7684\u8def\u5f84\u3002 \u6bd4\u5982\u4e0a\u9762\u7684\u4f8b\u5b50 TestSnapshot \u76ee\u5f55\uff0c\u6211\u4eec\u5982\u679c\u8981\u5217\u51fa oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u4e0b\u7684\u6587\u4ef6\u3002\u53ef\u4ee5\u7528\u5e38\u89c4\u7684 ls \u547d\u4ee4\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u7531\u4e8e\u6211\u4eec\u5728 TestSnapshot \u4e0b\u5f00\u542f\u7684\u5feb\u7167\uff0c\u5e76\u521b\u5efa\u4e86\u5feb\u7167 S1\u3002\u6211\u4eec\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u8def\u5f84\u6765\u5217\u51fa\u5feb\u7167 S1 \u4e0b\u7684\u76ee\u5f55\u548c\u6587\u4ef6\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/.snapshot/S1/dir1 \u5176\u4e2d .snapshot/S1 \u5c31\u662f\u7528\u6765\u8868\u660e\u76ee\u5f55\u7684\u5feb\u7167\u7684\u4fe1\u606f\u3002 \u5229\u7528\u5feb\u7167\u6062\u590d\u6570\u636e \u00a7 \u5feb\u7167\u7684\u4e00\u4e2a\u5e38\u89c1\u4f5c\u7528\u662f\u8fdb\u884c\u6570\u636e\u5907\u4efd\u548c\u6062\u590d\u3002\u5229\u7528\u5feb\u7167\u53ef\u4ee5\u5bf9\u91cd\u8981\u6570\u636e\u8fdb\u884c\u6062\u590d\uff0c\u9632\u6b62\u7528\u6237\u9519\u8bef\u6027\u7684\u64cd\u4f5c\u3002\u53ea\u8981\u6839\u636e\u4e0a\u9762\u8bb2\u7684 \u5feb\u7167\u76ee\u5f55\u7684\u683c\u5f0f \u5c31\u53ef\u4ee5\u8bbf\u95ee\u5230\u53ea\u8bfb\u526f\u672c\u4e2d\u7684\u6570\u636e\uff0c\u7136\u540e\u53ef\u4ee5\u6839\u636e\u9700\u6c42\u8fdb\u884c\u6062\u590d\u64cd\u4f5c\u3002 \u5728\u4e0a\u9762\u7684\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u5047\u8bbe\u5728\u540e\u9762\u7684\u67d0\u6b21\u4efb\u52a1\u4e2d\u8bef\u5220\u4e86 oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 hdfs dfs -rm -r oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u7531\u4e8e\u6211\u4eec\u5df2\u7ecf\u5bf9 /TestSnapshot \u6dfb\u52a0\u4e86\u5feb\u7167 S1\u3002\u6211\u4eec\u53ef\u4ee5\u7528\u5982\u4e0b\u7684\u547d\u4ee4\u6062\u590d\u6570\u636e\u3002 hdfs dfs -cp oss://oss-dfs-test.<Endpoint>/TestSnapshot/.snapshot/S1/dir1 oss://oss-dfs-test.<Endpoint>/TestSnapshot \u8fd9\u4e2a\u65f6\u5019\u6211\u4eec\u518d\u8bbf\u95ee /TestSnapshot/dir1\uff0c\u5c31\u4f1a\u53d1\u73b0\u539f\u6765\u7684\u8bef\u5220\u7684\u6587\u4ef6\u5939\u548c\u6587\u4ef6\u90fd\u6062\u590d\u4e86\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u91cd\u547d\u540d\u5feb\u7167 \u00a7 \u5bf9\u4e8e\u4e00\u4e2a\u5df2\u7ecf\u521b\u5efa\u7684\u5feb\u7167\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u91cd\u547d\u540d hdfs dfs -renameSnapshot <path> <oldName> <newName> \u4f8b\u5982\u4e0a\u9762\u4f8b\u5b50\u7684\u4e2d\u7684 TestSnapshot \u7684\u5feb\u7167 S1\uff0c\u6211\u4eec\u53ef\u4ee5\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u91cd\u547d\u540d\u4e3a S100 hdfs dfs -renameSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S1 S100 \u5220\u9664\u5feb\u7167 \u00a7 \u5f53\u4e0d\u9700\u8981\u5feb\u7167\u7684\u65f6\u5019\uff0c\u53ef\u4ee5\u8fd0\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5220\u9664\u5236\u5b9a\u7684\u5feb\u7167\u3002 hdfs dfs -deleteSnapshot <path> <snapshotName> \u5728\u4e0a\u9762\u7684\u4f8b\u5b50\u4e2d\uff0c\u6211\u4eec\u53ef\u4ee5\u8fd0\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5220\u9664\u4e4b\u524d\u521b\u5efa\u5e76\u91cd\u547d\u540d\u7684\u5feb\u7167 S100 hdfs dfs -deleteSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S100 \u5173\u95ed\u5feb\u7167\u529f\u80fd \u00a7 \u5f53\u6211\u4eec\u9700\u8981\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u7684\u65f6\u5019\uff0c\u6211\u4eec\u540c\u6837\u9700\u8981\u4f7f\u7528 JindoSDK \u7684 shell \u547d\u4ee4\u884c\u3002\u5177\u4f53\u7684\u5173\u95ed\u5feb\u7167\u7684\u683c\u5f0f\u4e3a ./jindofs admin -disallowSnapshot -dlsUri <path> \u67e5\u770b\u4e24\u4e2a\u5feb\u7167\u4e4b\u95f4\u7684\u5dee\u5f02 \u00a7 \u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u5f53\u547d\u4ee4\u67e5\u770b\u4e24\u4e2a\u5feb\u7167\u4e4b\u95f4\u7684\u5dee\u5f02 ./jindofs admin -snapshotDiff -dlsUri <uri> -fromSnapshot <fromSnapshot> -toSnapshot <toSnapshot> \u6ce8\u610f\u4e8b\u9879 \u00a7 \u5982\u679c\u9700\u8981\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\uff0c\u9996\u5148\u8981\u786e\u4fdd\u8be5\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u5feb\u7167\u5df2\u7ecf\u88ab\u5220\u9664\u3002\u5220\u9664\u5feb\u7167\u7684\u547d\u4ee4\u53ef\u4ee5\u53c2\u8003\u524d\u9762\u7684\u5220\u9664\u5feb\u7167\u547d\u4ee4\u3002\u5982\u679c\u8be5\u76ee\u5f55\u4e0b\u8fd8\u5b58\u5728\u5feb\u7167\uff0c\u5173\u95ed\u5feb\u7167\u529f\u80fd\u4f1a\u62a5\u9519\u3002 \u5bf9\u4e8e\u6211\u4eec\u524d\u9762\u7684\u4f8b\u5b50 TestSnapshot\uff0c\u5047\u8bbe\u6211\u4eec\u5df2\u7ecf\u5220\u9664\u4e86\u6240\u6709\u5feb\u7167\u3002\u6211\u4eec\u53ef\u4ee5\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u3002 ./jindofs admin -disallowSnapshot -dlsUri oss://oss-dfs-test.<Endpoint>/TestSnapshot","title":"OSS-HDFS \u5feb\u7167"},{"location":"oss/dls_snapshot_quickstart/#oss-hdfs-jindofs","text":"(\u4ece JindoData 4.0.0 \u5f00\u59cb\u652f\u6301) \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u662fOSS\u65b0\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9HDFS\u63a5\u53e3, \u652f\u6301\u76ee\u5f55\u4ee5\u53ca\u76ee\u5f55\u5c42\u7ea7\uff0c\u901a\u8fc7Jindo SDK 4.x \u53ef\u4ee5\u517c\u5bb9\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u3002\u5173\u4e8e\u7528\u6237\u5982\u4f55\u521b\u5efa\u548c\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u7684\u57fa\u672c\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 \u94fe\u63a5 \u3002 OSS-HDFS \u670d\u52a1\u7684\u5feb\u7167\u529f\u80fd\u5728\u4f7f\u7528\u4e0a\u4e0e HDFS \u7684\u5feb\u7167\u529f\u80fd\u5b8c\u5168\u517c\u5bb9\u3002\u672c\u6587\u4e3b\u8981\u4ecb\u7ecd\u5728 OSS-HDFS \u670d\u52a1\u4e2d\u4f7f\u7528\u5feb\u7167\u529f\u80fd\u7684\u4e00\u4e9b\u5e38\u89c1\u64cd\u4f5c\u3002 \u4e3a\u4e86\u65b9\u4fbf\u4e0b\u9762\u547d\u4ee4\u7684\u4ecb\u7ecd\uff0c\u6211\u4eec\u5047\u8bbe oss://oss-dfs-test \u8fd9\u4e2aBucket\u5df2\u7ecf\u5f00\u542f OSS-HDFS \u670d\u52a1\u3002\u4e0b\u9762\u6240\u6709\u7684\u4f8b\u5b50\u90fd\u4f1a\u9488\u5bf9\u8fd9\u4e2a Bucket \u8fdb\u884c\u64cd\u4f5c\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u7167\u4f7f\u7528\u8bf4\u660e"},{"location":"oss/dls_snapshot_quickstart/#_1","text":"\u5047\u8bbe\u6211\u4eec\u5148\u5728 oss://oss-dfs-test \u4e0b\u521b\u5efa\u4e00\u4e2a\u76ee\u5f55TestSnapshot\u3002 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u662f\u5173\u95ed\u7684\u3002\u9700\u8981\u5f00\u542f\u548c\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\uff0c\u9700\u8981\u4f7f\u7528 jindofs \u547d\u4ee4\u884c\u5de5\u5177 \u3002\u5177\u4f53\u7684\u5f00\u542f\u5feb\u7167\u7684\u683c\u5f0f\u4e3a ./jindofs admin -allowSnapshot -dlsUri <path> \u5982\u679c\u6211\u4eec\u8981\u5f00\u542f\u524d\u9762\u521b\u5efa\u7684 TestSnapshot \u7684\u5feb\u7167\u529f\u80fd\u3002\u53ef\u4ee5\u8f93\u5165\u5982\u4e0b\u7684\u547d\u4ee4\uff1a ./jindofs admin -allowSnapshot -dlsUri oss://oss-dfs-test.<Endpoint>/TestSnapshot","title":"\u5f00\u542f\u5feb\u7167\u529f\u80fd"},{"location":"oss/dls_snapshot_quickstart/#_2","text":"\u5728\u4e00\u4e2a\u76ee\u5f55\u5f00\u542f\u5feb\u7167\u529f\u80fd\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7 HDFS \u7684 shell \u547d\u4ee4\u3002\u5177\u4f53\u683c\u5f0f\u4e3a\uff1a hdfs dfs -createSnapshot <path> [<snapshotName>] \u4f8b\u5982\u4e0a\u9762\u4f8b\u5b50\u7684\u4e2d\u7684 TestSnapshot\uff0c\u6211\u4eec\u8fdb\u884c\u4e86\u4e00\u4e9b\u76ee\u5f55\u548c\u6587\u4ef6\u7684\u64cd\u4f5c\u3002\u7136\u540e\u53ef\u4ee5\u521b\u5efa\u4e00\u4e2a\u5feb\u7167\u5bf9\u6b64\u523b\u7684\u5feb\u7167\u7684\u72b6\u6001\u8fdb\u884c\u4fdd\u5b58\u3002\u5047\u8bbe\u6211\u4eec\u547d\u540d\u5feb\u7167\u540d\u4e3a S1\u3002 #\u4e0b\u9762\u8fd9\u4e9b\u4e3a\u521b\u5efa\u6d4b\u8bd5\u7684\u6587\u4ef6\u548c\u76ee\u5f55 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir2 hdfs dfs -touchz oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1/file1 hdfs dfs -touchz oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir3/file2 #\u901a\u8fc7HDFS\u7684shell\u547d\u4ee4\u884c\u5de5\u5177\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u521b\u5efa\u5feb\u7167\uff0c\u5e76\u5c06\u5feb\u7167\u547d\u540d\u4e3aS1\uff1a hdfs dfs -createSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S1","title":"\u521b\u5efa\u5feb\u7167"},{"location":"oss/dls_snapshot_quickstart/#_3","text":"","title":"\u8bbf\u95ee\u5feb\u7167\u4e2d\u7684\u76ee\u5f55\u548c\u6587\u4ef6"},{"location":"oss/dls_snapshot_quickstart/#_4","text":"\u4e3a\u4e86\u533a\u522b\u4e8e\u6587\u4ef6\u7cfb\u7edf\u73b0\u6709\u7684\u76ee\u5f55\u548c\u6587\u4ef6\uff0c\u8bbf\u95ee\u5feb\u7167\u4e2d\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u9700\u8981\u6dfb\u52a0\u5feb\u7167\u7684\u4fe1\u606f\u6765\u6307\u51fa\u81ea\u5df1\u8981\u8bbf\u95ee\u7684\u5feb\u7167\u540d\u3002\u5bf9\u4e8e\u4e00\u4e2a\u5f00\u542f\u4e86\u5feb\u7167\u7684\u76ee\u5f55\uff0c\u6211\u4eec\u8981\u8bbf\u95ee\u5b83\u4e0b\u9762\u67d0\u4e2a\u5feb\u7167\u7684\u76ee\u5f55\u548c\u6587\u4ef6\uff0c\u6211\u4eec\u53ef\u4ee5\u4e0b\u9762\u7684\u683c\u5f0f\u6765\u8bbf\u95ee\uff1a <snapshotRoot>/.snapshot/<snapshotName>/<actual subPath> \u5176\u4e2d snapshotRoot \u4e3a\u5feb\u7167\u7684\u6839\u76ee\u5f55\uff0c\u4e5f\u5c31\u662f\u5f00\u542f\u5feb\u7167\u547d\u4ee4\u4e2ddlsUri\u53c2\u6570\u5236\u5b9a\u7684\u8def\u5f84\u3002snapshotName\u662f\u4e4b\u524d\u521b\u5efa\u7684\u5feb\u7167\u540d\u3002\u5236\u5b9a\u4e86\u5feb\u7167\u540d\u4e4b\u540e\uff0c\u4e4b\u540e\u7684\u8def\u5f84\u5c31\u662f\u539f\u6765\u5feb\u7167\u6839\u76ee\u5f55\uff08snapshotRoot\uff09\u4e0b\u8981\u8bbf\u95ee\u7684\u8def\u5f84\u3002 \u6bd4\u5982\u4e0a\u9762\u7684\u4f8b\u5b50 TestSnapshot \u76ee\u5f55\uff0c\u6211\u4eec\u5982\u679c\u8981\u5217\u51fa oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u4e0b\u7684\u6587\u4ef6\u3002\u53ef\u4ee5\u7528\u5e38\u89c4\u7684 ls \u547d\u4ee4\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u7531\u4e8e\u6211\u4eec\u5728 TestSnapshot \u4e0b\u5f00\u542f\u7684\u5feb\u7167\uff0c\u5e76\u521b\u5efa\u4e86\u5feb\u7167 S1\u3002\u6211\u4eec\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u8def\u5f84\u6765\u5217\u51fa\u5feb\u7167 S1 \u4e0b\u7684\u76ee\u5f55\u548c\u6587\u4ef6\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/.snapshot/S1/dir1 \u5176\u4e2d .snapshot/S1 \u5c31\u662f\u7528\u6765\u8868\u660e\u76ee\u5f55\u7684\u5feb\u7167\u7684\u4fe1\u606f\u3002","title":"\u5feb\u7167\u76ee\u5f55\u7684\u683c\u5f0f"},{"location":"oss/dls_snapshot_quickstart/#_5","text":"\u5feb\u7167\u7684\u4e00\u4e2a\u5e38\u89c1\u4f5c\u7528\u662f\u8fdb\u884c\u6570\u636e\u5907\u4efd\u548c\u6062\u590d\u3002\u5229\u7528\u5feb\u7167\u53ef\u4ee5\u5bf9\u91cd\u8981\u6570\u636e\u8fdb\u884c\u6062\u590d\uff0c\u9632\u6b62\u7528\u6237\u9519\u8bef\u6027\u7684\u64cd\u4f5c\u3002\u53ea\u8981\u6839\u636e\u4e0a\u9762\u8bb2\u7684 \u5feb\u7167\u76ee\u5f55\u7684\u683c\u5f0f \u5c31\u53ef\u4ee5\u8bbf\u95ee\u5230\u53ea\u8bfb\u526f\u672c\u4e2d\u7684\u6570\u636e\uff0c\u7136\u540e\u53ef\u4ee5\u6839\u636e\u9700\u6c42\u8fdb\u884c\u6062\u590d\u64cd\u4f5c\u3002 \u5728\u4e0a\u9762\u7684\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u5047\u8bbe\u5728\u540e\u9762\u7684\u67d0\u6b21\u4efb\u52a1\u4e2d\u8bef\u5220\u4e86 oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 hdfs dfs -rm -r oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u7531\u4e8e\u6211\u4eec\u5df2\u7ecf\u5bf9 /TestSnapshot \u6dfb\u52a0\u4e86\u5feb\u7167 S1\u3002\u6211\u4eec\u53ef\u4ee5\u7528\u5982\u4e0b\u7684\u547d\u4ee4\u6062\u590d\u6570\u636e\u3002 hdfs dfs -cp oss://oss-dfs-test.<Endpoint>/TestSnapshot/.snapshot/S1/dir1 oss://oss-dfs-test.<Endpoint>/TestSnapshot \u8fd9\u4e2a\u65f6\u5019\u6211\u4eec\u518d\u8bbf\u95ee /TestSnapshot/dir1\uff0c\u5c31\u4f1a\u53d1\u73b0\u539f\u6765\u7684\u8bef\u5220\u7684\u6587\u4ef6\u5939\u548c\u6587\u4ef6\u90fd\u6062\u590d\u4e86\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1","title":"\u5229\u7528\u5feb\u7167\u6062\u590d\u6570\u636e"},{"location":"oss/dls_snapshot_quickstart/#_6","text":"\u5bf9\u4e8e\u4e00\u4e2a\u5df2\u7ecf\u521b\u5efa\u7684\u5feb\u7167\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u91cd\u547d\u540d hdfs dfs -renameSnapshot <path> <oldName> <newName> \u4f8b\u5982\u4e0a\u9762\u4f8b\u5b50\u7684\u4e2d\u7684 TestSnapshot \u7684\u5feb\u7167 S1\uff0c\u6211\u4eec\u53ef\u4ee5\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u91cd\u547d\u540d\u4e3a S100 hdfs dfs -renameSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S1 S100","title":"\u91cd\u547d\u540d\u5feb\u7167"},{"location":"oss/dls_snapshot_quickstart/#_7","text":"\u5f53\u4e0d\u9700\u8981\u5feb\u7167\u7684\u65f6\u5019\uff0c\u53ef\u4ee5\u8fd0\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5220\u9664\u5236\u5b9a\u7684\u5feb\u7167\u3002 hdfs dfs -deleteSnapshot <path> <snapshotName> \u5728\u4e0a\u9762\u7684\u4f8b\u5b50\u4e2d\uff0c\u6211\u4eec\u53ef\u4ee5\u8fd0\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5220\u9664\u4e4b\u524d\u521b\u5efa\u5e76\u91cd\u547d\u540d\u7684\u5feb\u7167 S100 hdfs dfs -deleteSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S100","title":"\u5220\u9664\u5feb\u7167"},{"location":"oss/dls_snapshot_quickstart/#_8","text":"\u5f53\u6211\u4eec\u9700\u8981\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u7684\u65f6\u5019\uff0c\u6211\u4eec\u540c\u6837\u9700\u8981\u4f7f\u7528 JindoSDK \u7684 shell \u547d\u4ee4\u884c\u3002\u5177\u4f53\u7684\u5173\u95ed\u5feb\u7167\u7684\u683c\u5f0f\u4e3a ./jindofs admin -disallowSnapshot -dlsUri <path>","title":"\u5173\u95ed\u5feb\u7167\u529f\u80fd"},{"location":"oss/dls_snapshot_quickstart/#_9","text":"\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u5f53\u547d\u4ee4\u67e5\u770b\u4e24\u4e2a\u5feb\u7167\u4e4b\u95f4\u7684\u5dee\u5f02 ./jindofs admin -snapshotDiff -dlsUri <uri> -fromSnapshot <fromSnapshot> -toSnapshot <toSnapshot>","title":"\u67e5\u770b\u4e24\u4e2a\u5feb\u7167\u4e4b\u95f4\u7684\u5dee\u5f02"},{"location":"oss/dls_snapshot_quickstart/#_10","text":"\u5982\u679c\u9700\u8981\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\uff0c\u9996\u5148\u8981\u786e\u4fdd\u8be5\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u5feb\u7167\u5df2\u7ecf\u88ab\u5220\u9664\u3002\u5220\u9664\u5feb\u7167\u7684\u547d\u4ee4\u53ef\u4ee5\u53c2\u8003\u524d\u9762\u7684\u5220\u9664\u5feb\u7167\u547d\u4ee4\u3002\u5982\u679c\u8be5\u76ee\u5f55\u4e0b\u8fd8\u5b58\u5728\u5feb\u7167\uff0c\u5173\u95ed\u5feb\u7167\u529f\u80fd\u4f1a\u62a5\u9519\u3002 \u5bf9\u4e8e\u6211\u4eec\u524d\u9762\u7684\u4f8b\u5b50 TestSnapshot\uff0c\u5047\u8bbe\u6211\u4eec\u5df2\u7ecf\u5220\u9664\u4e86\u6240\u6709\u5feb\u7167\u3002\u6211\u4eec\u53ef\u4ee5\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u3002 ./jindofs admin -disallowSnapshot -dlsUri oss://oss-dfs-test.<Endpoint>/TestSnapshot","title":"\u6ce8\u610f\u4e8b\u9879"},{"location":"oss/dls_trash_quickstart/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u56de\u6536\u7ad9\u4f7f\u7528\u8bf4\u660e \u00a7 (\u4ece JindoData 4.5.1 \u5f00\u59cb\u652f\u6301) \u4ecb\u7ecd \u00a7 \u5f53\u4ece OSS-HDFS \u670d\u52a1\u5220\u9664\u6587\u4ef6\u65f6\uff0c\u6587\u4ef6\u4e0d\u662f\u7acb\u5373\u88ab\u7acb\u5373\u5220\u9664\u3002\u88ab\u5220\u9664\u7684\u6587\u4ef6\u88ab\u632a\u5230\u4e86 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\u3002\u5f53\u7ecf\u8fc730\u5206\u949f\u540e\uff0cCurrent\u76ee\u5f55\u4f1a\u88ab\u632a\u5230 ```bash /user/ /.Trash/ \u4e5f\u5c31\u662f\u8bf4\u67d0\u4e00\u65f6\u95f4\u6bb5\u5185\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u4f1a\u88ab\u5f52\u7c7b\u5230\u4e00\u4e2a\u5e26\u65f6\u95f4\u6233\u7684\u76ee\u5f55\u4e0b\uff0c\u8868\u793a\u662f\u5728\u8fd9\u4e2a\u65f6\u95f4\u5185\u88ab\u5220\u9664\u7684\uff0c\u76f8\u5f53\u4e8e\u4e00\u6b21checkpoint\u3002\u5f53\u7ecf\u8fc7**7\u5929**\u540e\uff0c\u8fd9\u4e2a<timestamp>\u76ee\u5f55\u5c06\u4f1a\u88ab\u6c38\u8fdc\u5220\u9664\u3002\u56e0\u6b64\u57287\u5929\u5185\uff0c\u60a8\u6709\u673a\u4f1a\u4ece.Trash\u76ee\u5f55\u4e0b\u627e\u5230\u5bf9\u5e94\u65f6\u523b\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u5c06\u5176\u4ece.Trash\u632a\u51fa\uff0c\u4ece\u800c\u6062\u590d\u5b83\u3002 \u6ce8\u610f\uff1a\u8fd9\u9879\u529f\u80fd\uff0c\u662f\u7531\u5ba2\u6237\u7aef\u548c\u670d\u52a1\u7aef\u914d\u5408\uff0c\u65b9\u80fd\u5f62\u6210\u56de\u6536\u7ad9\u7684\u529f\u80fd\u3002\u670d\u52a1\u7aef\u53ea\u8d1f\u8d23\u7ef4\u62a4 ```/user/<username>/.Trash``` \u7684\u5b9a\u65f6\u6e05\u7406\uff0c\u670d\u52a1\u7aef\u7684\u5b9a\u65f6\u6e05\u7406\u662f\u9ed8\u8ba4\u5f00\u542f\u7684\u3002\u5ba2\u6237\u7aef\u8d1f\u8d23\u5c06\u5f85\u5220\u9664\u6587\u4ef6\u632a\u5230.Trash\u76ee\u5f55\u4e0b\u3002 ## Hadoop FileSystem Shell \u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd ```bash hadoop fs -rm oss://bucket/a/b/c \u5ba2\u6237\u7aef\u7684 Hadoop Shell \u547d\u4ee4\u9ed8\u8ba4\u4e0d\u5f00\u542fTrash\u529f\u80fd\uff0c\u56e0\u6b64\u9700\u8981\u5728 core-site.xml \u91cc\u6dfb\u52a0\u4e00\u6761\u914d\u7f6e <property> <name>fs.trash.interval</name> <value>1440</value> </property> (\u8be5\u503c\u53ea\u9700\u5927\u4e8e0\u5373\u53ef) \u6b64\u65f6\u5728\u5ba2\u6237\u7aef\u4f1a\u81ea\u52a8\u5c06rm\u547d\u4ee4\u8f6c\u6362\u4e3a\u4e00\u6761 hadoop fs -mv oss://bucket/a/b/c /user/<username>/.Trash/Current/a/b/c \u547d\u4ee4\uff0c\u56e0\u6b64\uff0c\u60a8\u4e0d\u9700\u8981\u611f\u77e5\u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u670d\u52a1\u7aef\u4f1a\u8d1f\u8d23\u6e05\u7406\u3002 \u5982\u679c\u60a8\u60f3\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\uff0c\u91ca\u653e\u7a7a\u95f4\uff0c\u53ef\u4ee5\u6dfb\u52a0 -skipTrash \u53c2\u6570\uff0c\u6b64\u65f6\u5c06\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\u3002 Hadoop \u751f\u6001\u7ec4\u4ef6\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd \u00a7 Hive/Spark/Flink \u7b49\u7ec4\u4ef6\u5e76\u4e0d\u611f\u77e5 OSS-HDFS \u670d\u52a1\u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u4f7f\u7528 FileSystem(HCFS) \u7684 delete \u63a5\u53e3\u610f\u5473\u7740\u7acb\u5373\u5220\u9664\u3002\u56e0\u6b64\u5982\u679c\u60f3\u8981\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd\uff0c\u9700\u8981\u663e\u5f0f\u5730\u8c03\u7528 FileSystem \u7684 rename \u63a5\u53e3\uff0c\u5c06\u76ee\u6807\u6587\u4ef6\u624b\u52a8\u632a\u5230 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\uff0c\u7531 OSS-HDFS \u670d\u52a1\u7aef\u8d1f\u8d23\u5b9a\u671f\u5220\u9664\u3002\u8fd9\u4e00\u70b9\u4e0a OSS-HDFS \u91c7\u53d6\u4e86\u8ddf\u5f00\u6e90 Hadoop \u76f8\u4f3c\u7684\u7b56\u7565\u3002","title":"OSS-HDFS \u56de\u6536\u7ad9"},{"location":"oss/dls_trash_quickstart/#oss-hdfs-jindofs","text":"(\u4ece JindoData 4.5.1 \u5f00\u59cb\u652f\u6301)","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u56de\u6536\u7ad9\u4f7f\u7528\u8bf4\u660e"},{"location":"oss/dls_trash_quickstart/#_1","text":"\u5f53\u4ece OSS-HDFS \u670d\u52a1\u5220\u9664\u6587\u4ef6\u65f6\uff0c\u6587\u4ef6\u4e0d\u662f\u7acb\u5373\u88ab\u7acb\u5373\u5220\u9664\u3002\u88ab\u5220\u9664\u7684\u6587\u4ef6\u88ab\u632a\u5230\u4e86 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\u3002\u5f53\u7ecf\u8fc730\u5206\u949f\u540e\uff0cCurrent\u76ee\u5f55\u4f1a\u88ab\u632a\u5230 ```bash /user/ /.Trash/ \u4e5f\u5c31\u662f\u8bf4\u67d0\u4e00\u65f6\u95f4\u6bb5\u5185\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u4f1a\u88ab\u5f52\u7c7b\u5230\u4e00\u4e2a\u5e26\u65f6\u95f4\u6233\u7684\u76ee\u5f55\u4e0b\uff0c\u8868\u793a\u662f\u5728\u8fd9\u4e2a\u65f6\u95f4\u5185\u88ab\u5220\u9664\u7684\uff0c\u76f8\u5f53\u4e8e\u4e00\u6b21checkpoint\u3002\u5f53\u7ecf\u8fc7**7\u5929**\u540e\uff0c\u8fd9\u4e2a<timestamp>\u76ee\u5f55\u5c06\u4f1a\u88ab\u6c38\u8fdc\u5220\u9664\u3002\u56e0\u6b64\u57287\u5929\u5185\uff0c\u60a8\u6709\u673a\u4f1a\u4ece.Trash\u76ee\u5f55\u4e0b\u627e\u5230\u5bf9\u5e94\u65f6\u523b\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u5c06\u5176\u4ece.Trash\u632a\u51fa\uff0c\u4ece\u800c\u6062\u590d\u5b83\u3002 \u6ce8\u610f\uff1a\u8fd9\u9879\u529f\u80fd\uff0c\u662f\u7531\u5ba2\u6237\u7aef\u548c\u670d\u52a1\u7aef\u914d\u5408\uff0c\u65b9\u80fd\u5f62\u6210\u56de\u6536\u7ad9\u7684\u529f\u80fd\u3002\u670d\u52a1\u7aef\u53ea\u8d1f\u8d23\u7ef4\u62a4 ```/user/<username>/.Trash``` \u7684\u5b9a\u65f6\u6e05\u7406\uff0c\u670d\u52a1\u7aef\u7684\u5b9a\u65f6\u6e05\u7406\u662f\u9ed8\u8ba4\u5f00\u542f\u7684\u3002\u5ba2\u6237\u7aef\u8d1f\u8d23\u5c06\u5f85\u5220\u9664\u6587\u4ef6\u632a\u5230.Trash\u76ee\u5f55\u4e0b\u3002 ## Hadoop FileSystem Shell \u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd ```bash hadoop fs -rm oss://bucket/a/b/c \u5ba2\u6237\u7aef\u7684 Hadoop Shell \u547d\u4ee4\u9ed8\u8ba4\u4e0d\u5f00\u542fTrash\u529f\u80fd\uff0c\u56e0\u6b64\u9700\u8981\u5728 core-site.xml \u91cc\u6dfb\u52a0\u4e00\u6761\u914d\u7f6e <property> <name>fs.trash.interval</name> <value>1440</value> </property> (\u8be5\u503c\u53ea\u9700\u5927\u4e8e0\u5373\u53ef) \u6b64\u65f6\u5728\u5ba2\u6237\u7aef\u4f1a\u81ea\u52a8\u5c06rm\u547d\u4ee4\u8f6c\u6362\u4e3a\u4e00\u6761 hadoop fs -mv oss://bucket/a/b/c /user/<username>/.Trash/Current/a/b/c \u547d\u4ee4\uff0c\u56e0\u6b64\uff0c\u60a8\u4e0d\u9700\u8981\u611f\u77e5\u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u670d\u52a1\u7aef\u4f1a\u8d1f\u8d23\u6e05\u7406\u3002 \u5982\u679c\u60a8\u60f3\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\uff0c\u91ca\u653e\u7a7a\u95f4\uff0c\u53ef\u4ee5\u6dfb\u52a0 -skipTrash \u53c2\u6570\uff0c\u6b64\u65f6\u5c06\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\u3002","title":"\u4ecb\u7ecd"},{"location":"oss/dls_trash_quickstart/#hadoop","text":"Hive/Spark/Flink \u7b49\u7ec4\u4ef6\u5e76\u4e0d\u611f\u77e5 OSS-HDFS \u670d\u52a1\u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u4f7f\u7528 FileSystem(HCFS) \u7684 delete \u63a5\u53e3\u610f\u5473\u7740\u7acb\u5373\u5220\u9664\u3002\u56e0\u6b64\u5982\u679c\u60f3\u8981\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd\uff0c\u9700\u8981\u663e\u5f0f\u5730\u8c03\u7528 FileSystem \u7684 rename \u63a5\u53e3\uff0c\u5c06\u76ee\u6807\u6587\u4ef6\u624b\u52a8\u632a\u5230 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\uff0c\u7531 OSS-HDFS \u670d\u52a1\u7aef\u8d1f\u8d23\u5b9a\u671f\u5220\u9664\u3002\u8fd9\u4e00\u70b9\u4e0a OSS-HDFS \u91c7\u53d6\u4e86\u8ddf\u5f00\u6e90 Hadoop \u76f8\u4f3c\u7684\u7b56\u7565\u3002","title":"Hadoop \u751f\u6001\u7ec4\u4ef6\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd"},{"location":"oss/oss_introduction/","text":"OSS/OSS-HDFS \u6982\u8ff0 \u00a7 \u6982\u8ff0 \u00a7 \u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8OSS\uff08Object Storage Service\uff09\u662f\u4e00\u6b3e\u6d77\u91cf\u3001\u5b89\u5168\u3001\u4f4e\u6210\u672c\u3001\u9ad8\u53ef\u9760\u7684\u4e91\u5b58\u50a8\u670d\u52a1\uff0c\u53ef\u63d0\u4f9b99.9999999999%\uff0812\u4e2a9\uff09\u7684\u6570\u636e\u6301\u4e45\u6027\uff0c99.995%\u7684\u6570\u636e\u53ef\u7528\u6027\u3002\u591a\u79cd\u5b58\u50a8\u7c7b\u578b\u4f9b\u9009\u62e9\uff0c\u5168\u9762\u4f18\u5316\u5b58\u50a8\u6210\u672c\u3002\u8be6\u60c5\u8bf7\u89c1 \u4ec0\u4e48\u662f\u5bf9\u8c61\u5b58\u50a8OSS \u3002 OSS-HDFS\u670d\u52a1\uff08JindoFS\u670d\u52a1\uff09\u662f\u4e00\u6b3e\u4e91\u539f\u751f\u6570\u636e\u6e56\u5b58\u50a8\u4ea7\u54c1\u3002\u57fa\u4e8e\u7edf\u4e00\u7684\u5143\u6570\u636e\u7ba1\u7406\u80fd\u529b\uff0c\u5728\u5b8c\u5168\u517c\u5bb9HDFS\u6587\u4ef6\u7cfb\u7edf\u63a5\u53e3\u7684\u540c\u65f6\uff0c\u63d0\u4f9b\u5145\u5206\u7684POSIX\u80fd\u529b\u652f\u6301\uff0c\u80fd\u66f4\u597d\u5730\u6ee1\u8db3\u5927\u6570\u636e\u548cAI\u7b49\u9886\u57df\u7684\u6570\u636e\u6e56\u8ba1\u7b97\u573a\u666f\u3002\u8be6\u60c5\u8bf7\u89c1 OSS-HDFS\u670d\u52a1\u6982\u8ff0 \u3002 JindoData \u662f\u963f\u91cc\u4e91\u5f00\u6e90\u5927\u6570\u636e\u56e2\u961f\u81ea\u7814\u7684\u6570\u636e\u6e56\u5b58\u50a8\u52a0\u901f\u5957\u4ef6\uff0c\u9762\u5411\u5927\u6570\u636e\u548c AI \u751f\u6001\uff0c\u4e3a\u963f\u91cc\u4e91\u548c\u4e1a\u754c\u4e3b\u8981\u6570\u636e\u6e56\u5b58\u50a8\u7cfb\u7edf\u63d0\u4f9b\u5168\u65b9\u4f4d\u8bbf\u95ee\u52a0\u901f\u89e3\u51b3\u65b9\u6848\u3002JindoData \u5957\u4ef6\u57fa\u4e8e\u7edf\u4e00\u67b6\u6784\u548c\u5185\u6838\u5b9e\u73b0\uff0c\u4e3b\u8981\u5305\u62ec JindoFS \u5b58\u50a8\u7cfb\u7edf\uff08\u539f JindoFS Block \u6a21\u5f0f\uff09\uff0cJindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff08\u539f JindoFS Cache \u6a21\u5f0f\uff09\uff0cJindoSDK \u5927\u6570\u636e\u4e07\u80fd SDK \u548c\u5168\u9762\u517c\u5bb9\u7684\u751f\u6001\u5de5\u5177\uff08JindoFuse\u3001JindoDistCp\uff09\u3001\u63d2\u4ef6\u652f\u6301\u3002\u8be6\u60c5\u8bf7\u89c1 \u300aJindoData \u6982\u8ff0\u300b \u3002 \u4f7f\u7528 \u00a7 \u5728 EMR \u73af\u5883\u4e2d\uff0c\u5df2\u7ecf\u9ed8\u8ba4\u90e8\u7f72 JindoSDK\uff0c\u60a8\u53ef\u4ee5\u76f4\u63a5\u901a\u8fc7 JindoSDK \u6765\u8bbf\u95ee OSS/OSS-HDFS\u3002 \u5728 \u975e EMR \u73af\u5883\u4e2d\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4e0b\u8f7d\u6700\u65b0\u7248\u672c\u7684 JindoSDK \uff0c\u81ea\u884c\u90e8\u7f72\u4f7f\u7528\uff0cHadoop \u573a\u666f\u90e8\u7f72\u65b9\u5f0f\u53c2\u89c1 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b , AI\u573a\u666f\u90e8\u7f72\u65b9\u5f0f\u53c2\u89c1 \u300a\u5728 AI \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u3002 \u4f18\u70b9 \u00a7 \u4f7f\u7528 OSS/OSS-HDFS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\u6709\u4ee5\u4e0b\u4f18\u52bf\uff1a \u5373\u63d2\u5373\u7528\u3002OSS/OSS-HDFS \u90fd\u662f\u4e91\u539f\u751f\u5b58\u50a8\u670d\u52a1\uff0c\u901a\u8fc7Restful API\u63d0\u4f9b\u670d\u52a1\uff0c\u672c\u8eab\u4e0d\u9700\u90e8\u7f72\u3002\u5728\u963f\u91cc\u4e91 EMR \u96c6\u7fa4\u4e2d\uff0c\u5df2\u7ecf\u9ed8\u8ba4\u5b89\u88c5 JindoSDK\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7 JindoSDK \u76f4\u63a5\u8bbf\u95ee\u3002 \u8282\u7701\u6210\u672c\u3002\u4f7f\u7528 OSS/OSS-HDFS \u5b58\u50a8\u6570\u636e\u53ef\u4ee5\u6709\u6548\u8282\u7701\u6210\u672c\uff0c\u5e76\u7ed3\u5408\u4f4e\u9891/\u5f52\u6863/\u51b7\u5f52\u6863\u7b49\u65b9\u5f0f\uff0c\u53ef\u4ee5\u8fdb\u4e00\u6b65\u4f18\u5316\u51b7\u6570\u636e\u7684\u5b58\u50a8\u6210\u672c\u3002 \u53ef\u6269\u5c55\u6027\u3002OSS/OSS-HDFS \u6709\u7740\u66f4\u597d\u7684\u53ef\u6269\u5c55\u6027\uff0c\u4e0d\u53d7\u786c\u76d8\u5bb9\u91cf\u9650\u5236\uff0c\u65e0\u9700\u4eba\u5de5\u6269\u5bb9\u3002 \u7279\u6027 \u00a7 \u901a\u8fc7 JindoSDK \u4f7f\u7528 OSS/OSS-HDFS \u7684\u7279\u6027\u5bf9\u6bd4\u5982\u4e0b\uff1a \u573a\u666f \u652f\u6301\u7279\u6027 OSS OSS-HDFS \u5927\u6570\u636e\u573a\u666f \uff08Hadoop\uff09 \u652f\u6301\u76ee\u5f55\u3001\u6587\u4ef6\u8bed\u4e49\u548c\u64cd\u4f5c \u652f\u6301 \u652f\u6301 \u6dfb\u52a0\u76ee\u5f55\u3001\u6587\u4ef6\u6743\u9650 \u4e0d\u652f\u6301 \u652f\u6301 \u76ee\u5f55\u539f\u5b50\u6027\u3001rename\u6027\u80fd \u652f\u6301\uff0c\u4f46\u6027\u80fd\u4e0d\u4f73 \u652f\u6301\uff0c\u6beb\u79d2\u7ea7 \u901a\u8fc7 setTimes \u8bbe\u7f6e\u65f6\u95f4 \u4e0d\u652f\u6301 \u652f\u6301 \u6269\u5c55\u5c5e\u6027 XAttrs \u4e0d\u652f\u6301 \u652f\u6301 ACL \u4e0d\u652f\u6301 \u652f\u6301 \u672c\u5730\u8bfb\u7f13\u5b58\u52a0\u901f \u652f\u6301 \u652f\u6301 \u5feb\u7167 Snapshot \u4e0d\u652f\u6301 \u652f\u6301 \u6587\u4ef6 append/flush/sync \u64cd\u4f5c \u4e0d\u652f\u6301 \u652f\u6301 \u6587\u4ef6 truncate \u4e0d\u652f\u6301 \u652f\u6301 \u6821\u9a8c\u548c Checksum \u652f\u6301 \u652f\u6301 HDFS\u56de\u6536\u7ad9\u81ea\u52a8\u6e05\u7406 \u4e0d\u652f\u6301 \u652f\u6301 AI \u573a\u666f\uff08POSIX\uff09 \u5143\u6570\u636e\u4e00\u81f4\u6027 \u5f31 \u5f3a \u6587\u4ef6 append/flush/sync \u64cd\u4f5c \u652f\u6301\uff0c\u4f46\u6709 \u4f7f\u7528\u9650\u5236 \u652f\u6301 \u6587\u4ef6 truncate \u64cd\u4f5c \u4e0d\u652f\u6301 \u652f\u6301 \u968f\u673a\u5199 \u4e0d\u652f\u6301 \u652f\u6301","title":"OSS/OSS-HDFS \u4ecb\u7ecd"},{"location":"oss/oss_introduction/#ossoss-hdfs","text":"","title":"OSS/OSS-HDFS \u6982\u8ff0"},{"location":"oss/oss_introduction/#_1","text":"\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8OSS\uff08Object Storage Service\uff09\u662f\u4e00\u6b3e\u6d77\u91cf\u3001\u5b89\u5168\u3001\u4f4e\u6210\u672c\u3001\u9ad8\u53ef\u9760\u7684\u4e91\u5b58\u50a8\u670d\u52a1\uff0c\u53ef\u63d0\u4f9b99.9999999999%\uff0812\u4e2a9\uff09\u7684\u6570\u636e\u6301\u4e45\u6027\uff0c99.995%\u7684\u6570\u636e\u53ef\u7528\u6027\u3002\u591a\u79cd\u5b58\u50a8\u7c7b\u578b\u4f9b\u9009\u62e9\uff0c\u5168\u9762\u4f18\u5316\u5b58\u50a8\u6210\u672c\u3002\u8be6\u60c5\u8bf7\u89c1 \u4ec0\u4e48\u662f\u5bf9\u8c61\u5b58\u50a8OSS \u3002 OSS-HDFS\u670d\u52a1\uff08JindoFS\u670d\u52a1\uff09\u662f\u4e00\u6b3e\u4e91\u539f\u751f\u6570\u636e\u6e56\u5b58\u50a8\u4ea7\u54c1\u3002\u57fa\u4e8e\u7edf\u4e00\u7684\u5143\u6570\u636e\u7ba1\u7406\u80fd\u529b\uff0c\u5728\u5b8c\u5168\u517c\u5bb9HDFS\u6587\u4ef6\u7cfb\u7edf\u63a5\u53e3\u7684\u540c\u65f6\uff0c\u63d0\u4f9b\u5145\u5206\u7684POSIX\u80fd\u529b\u652f\u6301\uff0c\u80fd\u66f4\u597d\u5730\u6ee1\u8db3\u5927\u6570\u636e\u548cAI\u7b49\u9886\u57df\u7684\u6570\u636e\u6e56\u8ba1\u7b97\u573a\u666f\u3002\u8be6\u60c5\u8bf7\u89c1 OSS-HDFS\u670d\u52a1\u6982\u8ff0 \u3002 JindoData \u662f\u963f\u91cc\u4e91\u5f00\u6e90\u5927\u6570\u636e\u56e2\u961f\u81ea\u7814\u7684\u6570\u636e\u6e56\u5b58\u50a8\u52a0\u901f\u5957\u4ef6\uff0c\u9762\u5411\u5927\u6570\u636e\u548c AI \u751f\u6001\uff0c\u4e3a\u963f\u91cc\u4e91\u548c\u4e1a\u754c\u4e3b\u8981\u6570\u636e\u6e56\u5b58\u50a8\u7cfb\u7edf\u63d0\u4f9b\u5168\u65b9\u4f4d\u8bbf\u95ee\u52a0\u901f\u89e3\u51b3\u65b9\u6848\u3002JindoData \u5957\u4ef6\u57fa\u4e8e\u7edf\u4e00\u67b6\u6784\u548c\u5185\u6838\u5b9e\u73b0\uff0c\u4e3b\u8981\u5305\u62ec JindoFS \u5b58\u50a8\u7cfb\u7edf\uff08\u539f JindoFS Block \u6a21\u5f0f\uff09\uff0cJindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff08\u539f JindoFS Cache \u6a21\u5f0f\uff09\uff0cJindoSDK \u5927\u6570\u636e\u4e07\u80fd SDK \u548c\u5168\u9762\u517c\u5bb9\u7684\u751f\u6001\u5de5\u5177\uff08JindoFuse\u3001JindoDistCp\uff09\u3001\u63d2\u4ef6\u652f\u6301\u3002\u8be6\u60c5\u8bf7\u89c1 \u300aJindoData \u6982\u8ff0\u300b \u3002","title":"\u6982\u8ff0"},{"location":"oss/oss_introduction/#_2","text":"\u5728 EMR \u73af\u5883\u4e2d\uff0c\u5df2\u7ecf\u9ed8\u8ba4\u90e8\u7f72 JindoSDK\uff0c\u60a8\u53ef\u4ee5\u76f4\u63a5\u901a\u8fc7 JindoSDK \u6765\u8bbf\u95ee OSS/OSS-HDFS\u3002 \u5728 \u975e EMR \u73af\u5883\u4e2d\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4e0b\u8f7d\u6700\u65b0\u7248\u672c\u7684 JindoSDK \uff0c\u81ea\u884c\u90e8\u7f72\u4f7f\u7528\uff0cHadoop \u573a\u666f\u90e8\u7f72\u65b9\u5f0f\u53c2\u89c1 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b , AI\u573a\u666f\u90e8\u7f72\u65b9\u5f0f\u53c2\u89c1 \u300a\u5728 AI \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u3002","title":"\u4f7f\u7528"},{"location":"oss/oss_introduction/#_3","text":"\u4f7f\u7528 OSS/OSS-HDFS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\u6709\u4ee5\u4e0b\u4f18\u52bf\uff1a \u5373\u63d2\u5373\u7528\u3002OSS/OSS-HDFS \u90fd\u662f\u4e91\u539f\u751f\u5b58\u50a8\u670d\u52a1\uff0c\u901a\u8fc7Restful API\u63d0\u4f9b\u670d\u52a1\uff0c\u672c\u8eab\u4e0d\u9700\u90e8\u7f72\u3002\u5728\u963f\u91cc\u4e91 EMR \u96c6\u7fa4\u4e2d\uff0c\u5df2\u7ecf\u9ed8\u8ba4\u5b89\u88c5 JindoSDK\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7 JindoSDK \u76f4\u63a5\u8bbf\u95ee\u3002 \u8282\u7701\u6210\u672c\u3002\u4f7f\u7528 OSS/OSS-HDFS \u5b58\u50a8\u6570\u636e\u53ef\u4ee5\u6709\u6548\u8282\u7701\u6210\u672c\uff0c\u5e76\u7ed3\u5408\u4f4e\u9891/\u5f52\u6863/\u51b7\u5f52\u6863\u7b49\u65b9\u5f0f\uff0c\u53ef\u4ee5\u8fdb\u4e00\u6b65\u4f18\u5316\u51b7\u6570\u636e\u7684\u5b58\u50a8\u6210\u672c\u3002 \u53ef\u6269\u5c55\u6027\u3002OSS/OSS-HDFS \u6709\u7740\u66f4\u597d\u7684\u53ef\u6269\u5c55\u6027\uff0c\u4e0d\u53d7\u786c\u76d8\u5bb9\u91cf\u9650\u5236\uff0c\u65e0\u9700\u4eba\u5de5\u6269\u5bb9\u3002","title":"\u4f18\u70b9"},{"location":"oss/oss_introduction/#_4","text":"\u901a\u8fc7 JindoSDK \u4f7f\u7528 OSS/OSS-HDFS \u7684\u7279\u6027\u5bf9\u6bd4\u5982\u4e0b\uff1a \u573a\u666f \u652f\u6301\u7279\u6027 OSS OSS-HDFS \u5927\u6570\u636e\u573a\u666f \uff08Hadoop\uff09 \u652f\u6301\u76ee\u5f55\u3001\u6587\u4ef6\u8bed\u4e49\u548c\u64cd\u4f5c \u652f\u6301 \u652f\u6301 \u6dfb\u52a0\u76ee\u5f55\u3001\u6587\u4ef6\u6743\u9650 \u4e0d\u652f\u6301 \u652f\u6301 \u76ee\u5f55\u539f\u5b50\u6027\u3001rename\u6027\u80fd \u652f\u6301\uff0c\u4f46\u6027\u80fd\u4e0d\u4f73 \u652f\u6301\uff0c\u6beb\u79d2\u7ea7 \u901a\u8fc7 setTimes \u8bbe\u7f6e\u65f6\u95f4 \u4e0d\u652f\u6301 \u652f\u6301 \u6269\u5c55\u5c5e\u6027 XAttrs \u4e0d\u652f\u6301 \u652f\u6301 ACL \u4e0d\u652f\u6301 \u652f\u6301 \u672c\u5730\u8bfb\u7f13\u5b58\u52a0\u901f \u652f\u6301 \u652f\u6301 \u5feb\u7167 Snapshot \u4e0d\u652f\u6301 \u652f\u6301 \u6587\u4ef6 append/flush/sync \u64cd\u4f5c \u4e0d\u652f\u6301 \u652f\u6301 \u6587\u4ef6 truncate \u4e0d\u652f\u6301 \u652f\u6301 \u6821\u9a8c\u548c Checksum \u652f\u6301 \u652f\u6301 HDFS\u56de\u6536\u7ad9\u81ea\u52a8\u6e05\u7406 \u4e0d\u652f\u6301 \u652f\u6301 AI \u573a\u666f\uff08POSIX\uff09 \u5143\u6570\u636e\u4e00\u81f4\u6027 \u5f31 \u5f3a \u6587\u4ef6 append/flush/sync \u64cd\u4f5c \u652f\u6301\uff0c\u4f46\u6709 \u4f7f\u7528\u9650\u5236 \u652f\u6301 \u6587\u4ef6 truncate \u64cd\u4f5c \u4e0d\u652f\u6301 \u652f\u6301 \u968f\u673a\u5199 \u4e0d\u652f\u6301 \u652f\u6301","title":"\u7279\u6027"},{"location":"oss/oss_quickstart/","text":"OSS/OSS-HDFS \u5feb\u901f\u5165\u95e8 \u00a7 \u524d\u7f6e\u6761\u4ef6 \u00a7 \u8bbf\u95ee OSS/OSS-HDFS \u524d\uff0c\u9700\u8981\u521b\u5efa\u5bf9\u5e94\u7684\u5b58\u50a8\u7a7a\u95f4\uff08Bucket\uff09\u3002 \u5f00\u901a OSS \u670d\u52a1 \u3002 \u521b\u5efa OSS Bucket \u3002 \u786e\u8ba4 OSS \u6388\u6743\u3002 \u5728 EMR \u96c6\u7fa4\uff08\u65b0\u7248\u63a7\u5236\u53f0\uff09\u4e2d\u4f7f\u7528\uff0c \u5df2\u9ed8\u8ba4\u5f00\u901a\u3002\u5982\u9047\u95ee\u9898\uff0c\u8be6\u89c1 \u89d2\u8272\u6388\u6743 \u3002 \u5728 EMR \u96c6\u7fa4\uff08\u65e7\u7248\u63a7\u5236\u53f0\uff09\u4e2d\u4f7f\u7528\uff0c \u5df2\u9ed8\u8ba4\u5f00\u901a\u3002\u5982\u9047\u95ee\u9898\uff0c\u8be6\u89c1 \u89d2\u8272\u6388\u6743 \u3002 \u5728\u975e EMR \u73af\u5883\u4e2d\u4f7f\u7528\uff0c\u6388\u6743\u65b9\u5f0f\u8bf7\u53c2\u8003 \u300aOSS/OSS-HDFS \u6388\u6743\u300b \u5f00\u901a\u5e76\u6388\u6743\u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08\u53ef\u9009\u9879\uff0c\u63a8\u8350\u5f00\u901a\uff09\uff1a OSS-HDFS \u670d\u52a1\u4f7f\u7528\u524d\u987b\u77e5 \u3002 \u5f00\u901a\u5e76\u6388\u6743\u8bbf\u95ee OSS-HDFS \u3002 \u786e\u8ba4 JindoSDK \u90e8\u7f72\u7248\u672c\uff1a \u5728 EMR \u96c6\u7fa4\u4e2d\uff0c\u5df2\u9ed8\u8ba4\u90e8\u7f72 JindoSDK\u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u521b\u5efa EMR-3.42.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u6216 EMR-5.8.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u96c6\u7fa4\u3002 \u5728 \u975e EMR \u73af\u5883\u4e2d\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4e0b\u8f7d\u6700\u65b0\u7248\u672c\u7684 JindoSDK \uff0c\u81ea\u884c\u90e8\u7f72\u4f7f\u7528\uff0cHadoop \u573a\u666f\u90e8\u7f72\u65b9\u5f0f\u53c2\u89c1 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b , AI\u573a\u666f\u90e8\u7f72\u65b9\u5f0f\u53c2\u89c1 \u300a\u5728 AI \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u90e8\u7f72 JindoSDK 4.x \u53ca\u4ee5\u4e0a\u7248\u672c\u3002 \u8def\u5f84\u8bf4\u660e \u00a7 \u5b58\u50a8\u7cfb\u7edf \u6839\u8def\u5f84\u793a\u4f8b \u63cf\u8ff0 OSS oss://examplebucket.oss-cn-shanghai-internal.aliyuncs.com/ \u4ee5\u5728\u4e0a\u6d77\u533a\u57df\u5df2\u521b\u5efa\u4e86\u540d\u4e3aexamplebucket\u7684 OSS Bucket\uff0c\u5e76\u4f7f\u7528\u5185\u7f51 endpoint \u8bbf\u95ee\u4e3a\u4f8b\u3002 \u6ce8\uff1a\u5728 EMR \u96c6\u7fa4\u4e2d\u672a\u6302\u8f7d\u516c\u7f51\u7684\u8282\u70b9\uff0c\u9ed8\u8ba4\u4e0d\u652f\u6301\u8bbf\u95ee OSS \u516c\u7f51 endpoint\uff0c\u5373\u9ed8\u8ba4\u4e0d\u652f\u6301\u8de8\u533a\u57df\u8bbf\u95ee\u3002 OSS-HDFS oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u4ee5\u5728\u4e0a\u6d77\u533a\u57df\u5df2\u521b\u5efa\u4e86\u540d\u4e3aexamplebucket\u7684 OSS-HDFS Bucket \u4e3a\u4f8b\u3002 \u6ce8\uff1aOSS-HDFS \u76ee\u524d\u53ea\u652f\u6301\u5185\u7f51\u8bbf\u95ee\uff0c\u5373\u9ed8\u8ba4\u4e0d\u652f\u6301\u8de8\u533a\u57df\u8bbf\u95ee\u3002 \u8bbf\u95ee OSS \u548c OSS-HDFS \u7684\u65b9\u5f0f\uff0c\u9664\u8def\u5f84\u4e2d\u7684 endpoint \u5916\uff0c\u4f7f\u7528\u65b9\u5f0f\u5747\u76f8\u540c\u3002 \u8bbf\u95ee\u65b9\u5f0f \u00a7 \u8bbf\u95ee\u65b9\u5f0f \u793a\u4f8b \u63cf\u8ff0 Hadoop Shell \u547d\u4ee4 hadoop fs -ls oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ JindoSDK \u4e2d\u7684JindoOssFileSystem \u662f Hadoop FileSystem \u7684\u4e00\u79cd\u5b9e\u73b0\u3002 \u5728\u6267\u884c Hadoop Shell \u53ef\u4ee5\u901a\u8fc7\u8bc6\u522b\u8def\u5f84\u4e2d\u7684 endpoint \u6765\u8bbf\u95ee OSS/OSS-HDFS\u3002 \u66f4\u591a\u4f7f\u7528\u65b9\u5f0f\uff0c\u8be6\u89c1 \u300a\u901a\u8fc7 Hadoop Shell \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS\u300b Jindo CLI \u547d\u4ee4 jindo fs -ls oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ Jindo CLI \u5b9e\u73b0\u4e86\u7c7b\u4f3c Hadoop Shell \u7684\u65b9\u5f0f\u6765\u8bbf\u95ee OSS/OSS-HDFS\u3002 \u6b64\u5916\uff0c\u8fd8\u652f\u6301\u4e86\u66f4\u591a\u7684\u529f\u80fd\uff0c\u6bd4\u5982\u5f52\u6863/\u7f13\u5b58/\u9519\u8bef\u5206\u6790\u7b49\u3002\u8be6\u89c1 \u300a\u901a\u8fc7 Jindo CLI \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS\u300b POSIX \u547d\u4ee4 mkdir -p /mnt/oss jindo-fuse /mnt/oss -ouri=oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ ls /mnt/oss \u901a\u8fc7\u5b9e\u73b0 FUSE API\uff0cjindo-fuse \u53ef\u4ee5\u901a\u8fc7\u6302\u8f7d OSS/OSS-HDFS \u8def\u5f84\u5230\u672c\u5730\u8def\u5f84\u7684\u65b9\u5f0f\uff0c\u4f7f\u7528\u6237\u53ef\u4ee5\u50cf\u8bbf\u95ee\u672c\u5730\u6587\u4ef6\u4e00\u6837\u8bbf\u95ee OSS/OSS-HDFS\u3002 \u8be6\u89c1 \u300a\u901a\u8fc7 POSIX \u8bbf\u95ee OSS/OSS-HDFS\u300b OSS \u63a7\u5236\u53f0 \u53ef\u4ee5\u901a\u8fc7\u6253\u5f00 OSS \u63a7\u5236\u53f0 -> \u6587\u4ef6\u5217\u8868 -> OSS\u6587\u4ef6/HDFS \u6765\u8bbf\u95ee OSS/OSS-HDFS","title":"OSS/OSS-HDFS \u5feb\u901f\u5165\u95e8"},{"location":"oss/oss_quickstart/#ossoss-hdfs","text":"","title":"OSS/OSS-HDFS \u5feb\u901f\u5165\u95e8"},{"location":"oss/oss_quickstart/#_1","text":"\u8bbf\u95ee OSS/OSS-HDFS \u524d\uff0c\u9700\u8981\u521b\u5efa\u5bf9\u5e94\u7684\u5b58\u50a8\u7a7a\u95f4\uff08Bucket\uff09\u3002 \u5f00\u901a OSS \u670d\u52a1 \u3002 \u521b\u5efa OSS Bucket \u3002 \u786e\u8ba4 OSS \u6388\u6743\u3002 \u5728 EMR \u96c6\u7fa4\uff08\u65b0\u7248\u63a7\u5236\u53f0\uff09\u4e2d\u4f7f\u7528\uff0c \u5df2\u9ed8\u8ba4\u5f00\u901a\u3002\u5982\u9047\u95ee\u9898\uff0c\u8be6\u89c1 \u89d2\u8272\u6388\u6743 \u3002 \u5728 EMR \u96c6\u7fa4\uff08\u65e7\u7248\u63a7\u5236\u53f0\uff09\u4e2d\u4f7f\u7528\uff0c \u5df2\u9ed8\u8ba4\u5f00\u901a\u3002\u5982\u9047\u95ee\u9898\uff0c\u8be6\u89c1 \u89d2\u8272\u6388\u6743 \u3002 \u5728\u975e EMR \u73af\u5883\u4e2d\u4f7f\u7528\uff0c\u6388\u6743\u65b9\u5f0f\u8bf7\u53c2\u8003 \u300aOSS/OSS-HDFS \u6388\u6743\u300b \u5f00\u901a\u5e76\u6388\u6743\u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08\u53ef\u9009\u9879\uff0c\u63a8\u8350\u5f00\u901a\uff09\uff1a OSS-HDFS \u670d\u52a1\u4f7f\u7528\u524d\u987b\u77e5 \u3002 \u5f00\u901a\u5e76\u6388\u6743\u8bbf\u95ee OSS-HDFS \u3002 \u786e\u8ba4 JindoSDK \u90e8\u7f72\u7248\u672c\uff1a \u5728 EMR \u96c6\u7fa4\u4e2d\uff0c\u5df2\u9ed8\u8ba4\u90e8\u7f72 JindoSDK\u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u521b\u5efa EMR-3.42.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u6216 EMR-5.8.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u96c6\u7fa4\u3002 \u5728 \u975e EMR \u73af\u5883\u4e2d\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4e0b\u8f7d\u6700\u65b0\u7248\u672c\u7684 JindoSDK \uff0c\u81ea\u884c\u90e8\u7f72\u4f7f\u7528\uff0cHadoop \u573a\u666f\u90e8\u7f72\u65b9\u5f0f\u53c2\u89c1 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b , AI\u573a\u666f\u90e8\u7f72\u65b9\u5f0f\u53c2\u89c1 \u300a\u5728 AI \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u90e8\u7f72 JindoSDK 4.x \u53ca\u4ee5\u4e0a\u7248\u672c\u3002","title":"\u524d\u7f6e\u6761\u4ef6"},{"location":"oss/oss_quickstart/#_2","text":"\u5b58\u50a8\u7cfb\u7edf \u6839\u8def\u5f84\u793a\u4f8b \u63cf\u8ff0 OSS oss://examplebucket.oss-cn-shanghai-internal.aliyuncs.com/ \u4ee5\u5728\u4e0a\u6d77\u533a\u57df\u5df2\u521b\u5efa\u4e86\u540d\u4e3aexamplebucket\u7684 OSS Bucket\uff0c\u5e76\u4f7f\u7528\u5185\u7f51 endpoint \u8bbf\u95ee\u4e3a\u4f8b\u3002 \u6ce8\uff1a\u5728 EMR \u96c6\u7fa4\u4e2d\u672a\u6302\u8f7d\u516c\u7f51\u7684\u8282\u70b9\uff0c\u9ed8\u8ba4\u4e0d\u652f\u6301\u8bbf\u95ee OSS \u516c\u7f51 endpoint\uff0c\u5373\u9ed8\u8ba4\u4e0d\u652f\u6301\u8de8\u533a\u57df\u8bbf\u95ee\u3002 OSS-HDFS oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u4ee5\u5728\u4e0a\u6d77\u533a\u57df\u5df2\u521b\u5efa\u4e86\u540d\u4e3aexamplebucket\u7684 OSS-HDFS Bucket \u4e3a\u4f8b\u3002 \u6ce8\uff1aOSS-HDFS \u76ee\u524d\u53ea\u652f\u6301\u5185\u7f51\u8bbf\u95ee\uff0c\u5373\u9ed8\u8ba4\u4e0d\u652f\u6301\u8de8\u533a\u57df\u8bbf\u95ee\u3002 \u8bbf\u95ee OSS \u548c OSS-HDFS \u7684\u65b9\u5f0f\uff0c\u9664\u8def\u5f84\u4e2d\u7684 endpoint \u5916\uff0c\u4f7f\u7528\u65b9\u5f0f\u5747\u76f8\u540c\u3002","title":"\u8def\u5f84\u8bf4\u660e"},{"location":"oss/oss_quickstart/#_3","text":"\u8bbf\u95ee\u65b9\u5f0f \u793a\u4f8b \u63cf\u8ff0 Hadoop Shell \u547d\u4ee4 hadoop fs -ls oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ JindoSDK \u4e2d\u7684JindoOssFileSystem \u662f Hadoop FileSystem \u7684\u4e00\u79cd\u5b9e\u73b0\u3002 \u5728\u6267\u884c Hadoop Shell \u53ef\u4ee5\u901a\u8fc7\u8bc6\u522b\u8def\u5f84\u4e2d\u7684 endpoint \u6765\u8bbf\u95ee OSS/OSS-HDFS\u3002 \u66f4\u591a\u4f7f\u7528\u65b9\u5f0f\uff0c\u8be6\u89c1 \u300a\u901a\u8fc7 Hadoop Shell \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS\u300b Jindo CLI \u547d\u4ee4 jindo fs -ls oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ Jindo CLI \u5b9e\u73b0\u4e86\u7c7b\u4f3c Hadoop Shell \u7684\u65b9\u5f0f\u6765\u8bbf\u95ee OSS/OSS-HDFS\u3002 \u6b64\u5916\uff0c\u8fd8\u652f\u6301\u4e86\u66f4\u591a\u7684\u529f\u80fd\uff0c\u6bd4\u5982\u5f52\u6863/\u7f13\u5b58/\u9519\u8bef\u5206\u6790\u7b49\u3002\u8be6\u89c1 \u300a\u901a\u8fc7 Jindo CLI \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS\u300b POSIX \u547d\u4ee4 mkdir -p /mnt/oss jindo-fuse /mnt/oss -ouri=oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ ls /mnt/oss \u901a\u8fc7\u5b9e\u73b0 FUSE API\uff0cjindo-fuse \u53ef\u4ee5\u901a\u8fc7\u6302\u8f7d OSS/OSS-HDFS \u8def\u5f84\u5230\u672c\u5730\u8def\u5f84\u7684\u65b9\u5f0f\uff0c\u4f7f\u7528\u6237\u53ef\u4ee5\u50cf\u8bbf\u95ee\u672c\u5730\u6587\u4ef6\u4e00\u6837\u8bbf\u95ee OSS/OSS-HDFS\u3002 \u8be6\u89c1 \u300a\u901a\u8fc7 POSIX \u8bbf\u95ee OSS/OSS-HDFS\u300b OSS \u63a7\u5236\u53f0 \u53ef\u4ee5\u901a\u8fc7\u6253\u5f00 OSS \u63a7\u5236\u53f0 -> \u6587\u4ef6\u5217\u8868 -> OSS\u6587\u4ef6/HDFS \u6765\u8bbf\u95ee OSS/OSS-HDFS","title":"\u8bbf\u95ee\u65b9\u5f0f"},{"location":"oss/oss_ram_policy/","text":"OSS/OSS-HDFS \u6388\u6743 \u00a7 \u6388\u6743\u8bbf\u95ee OSS \u00a7 \u6388\u6743\u64cd\u4f5c\u8d26\u53f7\u5bf9OSS\u7684\u64cd\u4f5c\u6743\u9650\u65f6\uff0c\u652f\u6301\u901a\u8fc7\u81ea\u5b9a\u4e49\u7b56\u7565\u7075\u6d3b\u5b9a\u4e49RAM\u7528\u6237\u5bf9OSS\u6570\u636e\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u8be6\u7ec6\u6b65\u9aa4\u5982\u4e0b\u3002 \u767b\u5f55 RAM\u63a7\u5236\u53f0 \u3002 \u5728\u5de6\u4fa7\u5bfc\u822a\u680f\uff0c\u9009\u62e9 \u6743\u9650\u7ba1\u7406 > \u6743\u9650\u7b56\u7565 \u3002 \u5728 \u6743\u9650\u7b56\u7565 \u9875\u9762\uff0c\u5355\u51fb \u521b\u5efa\u6743\u9650\u7b56\u7565 \u3002 \u5728 \u521b\u5efa\u6743\u9650\u7b56\u7565 \u9875\u9762\uff0c\u5355\u51fb \u811a\u672c\u7f16\u8f91 \u9875\u7b7e\u3002 \u8f93\u5165\u6743\u9650\u7b56\u7565\u5185\u5bb9\uff0c\u7136\u540e\u5355\u51fb \u4e0b\u4e00\u6b65\uff1a\u7f16\u8f91\u57fa\u672c\u4fe1\u606f \u3002 OSS \u63d0\u4f9b\u4e86\u5b8c\u6574\u7684\u6570\u636e\u6743\u9650\u7ba1\u63a7\u4f53\u7cfb\uff0c\u5b8c\u6574\u7684OSS\u6388\u6743\u7b56\u7565\u8bf7\u53c2\u89c1 RAM Policy\u6982\u8ff0 \u3002 \u6ce8\u610f \u8bf7\u6839\u636eRAM\u7528\u6237\u9700\u8981\u4f7f\u7528\u7684\u6743\u9650\uff0c\u8c28\u614e\u5b9a\u4e49\u6743\u9650\u7b56\u7565\u3002\u5bf9\u4e8e\u4f7f\u7528hive\u3001spark\u7684\u7528\u6237\uff0c\u65e0\u8bba\u662f\u5426\u5f00\u542f\u591a\u7248\u672c\u90fd\u9700\u8981\u914d\u7f6e oss:ListObjectVersions \u3001 oss:DeleteObjectVersion \uff0c\u6211\u4eec\u5bf9\u4e8e\u4e00\u4e9b\u5e38\u89c1\u7684\u4e34\u65f6\u76ee\u5f55\uff0c\u6bd4\u5982\"_temporary\"\u3001\".staging\"\u3001\".hive-staging\"\u3001\"__magic\" \u4f1a\u89e6\u53d1\u591a\u7248\u672c\u64cd\u4f5c\u3002 { \"Version\": \"1\", \"Statement\": [ { \"Action\": [ \"oss:ListObjects\", \"oss:GetBucketInfo\", \"oss:PutObject\", \"oss:GetObject\", \"oss:DeleteObject\", \"oss:AbortMultipartUpload\", \"oss:ListParts\", \"oss:RestoreObject\", \"oss:ListObjectVersions\", \"oss:DeleteObjectVersion\" ], \"Resource\": [ \"acs:oss:*:*:<example-bucket>\", \"acs:oss:*:*:<example-bucket>/*\" ], \"Effect\": \"Allow\" }, { \"Action\": [ \"oss:ListBuckets\" ], \"Resource\": \"*\", \"Effect\": \"Allow\" } ] } \u4e0a\u8ff0\u81ea\u5b9a\u4e49\u7b56\u7565\u4e2d\u7684 <example-bucket> \u9700\u8981\u66ff\u6362\u4e3a\u88ab\u6388\u6743\u7684Bucket\u540d\u79f0\u3002 \u8f93\u5165\u6743\u9650\u7b56\u7565 \u540d\u79f0 \u548c \u5907\u6ce8 \uff0c\u5e76\u5355\u51fb \u786e\u5b9a \u3002 \u5176\u4ed6\u81ea\u5b9a\u4e49\u573a\u666f\u793a\u4f8b \u00a7 \u53ea\u8bfb\u8bbf\u95ee OSS\uff1a \u6ce8\u610f \u8bf7\u6839\u636eRAM\u7528\u6237\u9700\u8981\u4f7f\u7528\u7684\u6743\u9650\uff0c\u8c28\u614e\u5b9a\u4e49\u6743\u9650\u7b56\u7565\u3002\u5bf9\u4e8e\u4f7f\u7528hive\u3001spark\u7684\u7528\u6237\uff0c\u65e0\u8bba\u662f\u5426\u5f00\u542f\u591a\u7248\u672c\u90fd\u9700\u8981\u914d\u7f6e oss:ListObjectVersions , \u6211\u4eec\u5bf9\u4e8e\u4e00\u4e9b\u5e38\u89c1\u7684\u4e34\u65f6\u76ee\u5f55\uff0c\u6bd4\u5982\"_temporary\"\u3001\".staging\"\u3001\".hive-staging\"\u3001\"__magic\" \u4f1a\u89e6\u53d1\u591a\u7248\u672c\u64cd\u4f5c\u3002 { \"Version\": \"1\", \"Statement\": [ { \"Action\": [ \"oss:ListObjects\", \"oss:GetBucketInfo\", \"oss:GetObject\", \"oss:RestoreObject\", \"oss:ListObjectVersions\" ], \"Resource\": [ \"acs:oss:*:*:<yourBucketName>\", \"acs:oss:*:*:<yourBucketName>/*\" ], \"Effect\": \"Allow\" }, { \"Action\": [ \"oss:ListBuckets\" ], \"Resource\": \"*\", \"Effect\": \"Allow\" } ] } \u8bbf\u95ee\u5f00\u542f\u4e86\u591a\u7248\u672c\u529f\u80fd\u7684 OSS\uff08\u4e0d\u5efa\u8bae\u5f00\u542f\u591a\u7248\u672c\uff0c\u5bb9\u6613\u4ea7\u751f\u6027\u80fd\u95ee\u9898\uff09\uff1a { \"Version\": \"1\", \"Statement\": [ { \"Action\": [ \"oss:ListObjects\", \"oss:GetBucketInfo\", \"oss:PutObject\", \"oss:GetObject\", \"oss:DeleteObject\", \"oss:AbortMultipartUpload\", \"oss:ListParts\", \"oss:RestoreObject\", \"oss:ListObjectVersions\", \"oss:GetObjectVersion\", \"oss:DeleteObjectVersion\", \"oss:RestoreObjectVersion\" ], \"Resource\": [ \"acs:oss:*:*:<yourBucketName>\", \"acs:oss:*:*:<yourBucketName>/*\" ], \"Effect\": \"Allow\" }, { \"Action\": [ \"oss:ListBuckets\" ], \"Resource\": \"*\", \"Effect\": \"Allow\" } ] } Bucket Policy \u6388\u6743 \u00a7 \u5efa\u8bae\u4f7f\u7528\u201c\u5b8c\u5168\u63a7\u5236\u201d\uff0c\u907f\u514d\u51fa\u73b0\u4ee5\u4e0b\u95ee\u9898\uff1a \u300a\u4f7f\u7528\u5b50\u8d26\u53f7\u8bbf\u95ee OSS \u51fa\u73b0 AccessDenied \u95ee\u9898\u6392\u67e5\u300b \u6388\u6743\u8bbf\u95ee OSS-HDFS \u00a7 { \"Statement\": [ { \"Effect\": \"Allow\", \"Action\": \"oss:ListObjects\", \"Resource\": [ \"acs:oss:*:*:*\" ] }, { \"Effect\": \"Allow\", \"Action\": [ \"oss:GetBucketInfo\", \"oss:PostDataLakeStorageFileOperation\", \"oss:PostDataLakeStorageAdminOperation\" ], \"Resource\": \"*\" }, { \"Effect\": \"Allow\", \"Action\": \"oss:*\", \"Resource\": [ \"acs:oss:*:*:*/.dlsdata\", \"acs:oss:*:*:*/.dlsdata*\" ] } ], \"Version\": \"1\" }","title":"OSS/OSS-HDFS RAM \u914d\u7f6e"},{"location":"oss/oss_ram_policy/#ossoss-hdfs","text":"","title":"OSS/OSS-HDFS \u6388\u6743"},{"location":"oss/oss_ram_policy/#oss","text":"\u6388\u6743\u64cd\u4f5c\u8d26\u53f7\u5bf9OSS\u7684\u64cd\u4f5c\u6743\u9650\u65f6\uff0c\u652f\u6301\u901a\u8fc7\u81ea\u5b9a\u4e49\u7b56\u7565\u7075\u6d3b\u5b9a\u4e49RAM\u7528\u6237\u5bf9OSS\u6570\u636e\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u8be6\u7ec6\u6b65\u9aa4\u5982\u4e0b\u3002 \u767b\u5f55 RAM\u63a7\u5236\u53f0 \u3002 \u5728\u5de6\u4fa7\u5bfc\u822a\u680f\uff0c\u9009\u62e9 \u6743\u9650\u7ba1\u7406 > \u6743\u9650\u7b56\u7565 \u3002 \u5728 \u6743\u9650\u7b56\u7565 \u9875\u9762\uff0c\u5355\u51fb \u521b\u5efa\u6743\u9650\u7b56\u7565 \u3002 \u5728 \u521b\u5efa\u6743\u9650\u7b56\u7565 \u9875\u9762\uff0c\u5355\u51fb \u811a\u672c\u7f16\u8f91 \u9875\u7b7e\u3002 \u8f93\u5165\u6743\u9650\u7b56\u7565\u5185\u5bb9\uff0c\u7136\u540e\u5355\u51fb \u4e0b\u4e00\u6b65\uff1a\u7f16\u8f91\u57fa\u672c\u4fe1\u606f \u3002 OSS \u63d0\u4f9b\u4e86\u5b8c\u6574\u7684\u6570\u636e\u6743\u9650\u7ba1\u63a7\u4f53\u7cfb\uff0c\u5b8c\u6574\u7684OSS\u6388\u6743\u7b56\u7565\u8bf7\u53c2\u89c1 RAM Policy\u6982\u8ff0 \u3002 \u6ce8\u610f \u8bf7\u6839\u636eRAM\u7528\u6237\u9700\u8981\u4f7f\u7528\u7684\u6743\u9650\uff0c\u8c28\u614e\u5b9a\u4e49\u6743\u9650\u7b56\u7565\u3002\u5bf9\u4e8e\u4f7f\u7528hive\u3001spark\u7684\u7528\u6237\uff0c\u65e0\u8bba\u662f\u5426\u5f00\u542f\u591a\u7248\u672c\u90fd\u9700\u8981\u914d\u7f6e oss:ListObjectVersions \u3001 oss:DeleteObjectVersion \uff0c\u6211\u4eec\u5bf9\u4e8e\u4e00\u4e9b\u5e38\u89c1\u7684\u4e34\u65f6\u76ee\u5f55\uff0c\u6bd4\u5982\"_temporary\"\u3001\".staging\"\u3001\".hive-staging\"\u3001\"__magic\" \u4f1a\u89e6\u53d1\u591a\u7248\u672c\u64cd\u4f5c\u3002 { \"Version\": \"1\", \"Statement\": [ { \"Action\": [ \"oss:ListObjects\", \"oss:GetBucketInfo\", \"oss:PutObject\", \"oss:GetObject\", \"oss:DeleteObject\", \"oss:AbortMultipartUpload\", \"oss:ListParts\", \"oss:RestoreObject\", \"oss:ListObjectVersions\", \"oss:DeleteObjectVersion\" ], \"Resource\": [ \"acs:oss:*:*:<example-bucket>\", \"acs:oss:*:*:<example-bucket>/*\" ], \"Effect\": \"Allow\" }, { \"Action\": [ \"oss:ListBuckets\" ], \"Resource\": \"*\", \"Effect\": \"Allow\" } ] } \u4e0a\u8ff0\u81ea\u5b9a\u4e49\u7b56\u7565\u4e2d\u7684 <example-bucket> \u9700\u8981\u66ff\u6362\u4e3a\u88ab\u6388\u6743\u7684Bucket\u540d\u79f0\u3002 \u8f93\u5165\u6743\u9650\u7b56\u7565 \u540d\u79f0 \u548c \u5907\u6ce8 \uff0c\u5e76\u5355\u51fb \u786e\u5b9a \u3002","title":"\u6388\u6743\u8bbf\u95ee OSS"},{"location":"oss/oss_ram_policy/#_1","text":"\u53ea\u8bfb\u8bbf\u95ee OSS\uff1a \u6ce8\u610f \u8bf7\u6839\u636eRAM\u7528\u6237\u9700\u8981\u4f7f\u7528\u7684\u6743\u9650\uff0c\u8c28\u614e\u5b9a\u4e49\u6743\u9650\u7b56\u7565\u3002\u5bf9\u4e8e\u4f7f\u7528hive\u3001spark\u7684\u7528\u6237\uff0c\u65e0\u8bba\u662f\u5426\u5f00\u542f\u591a\u7248\u672c\u90fd\u9700\u8981\u914d\u7f6e oss:ListObjectVersions , \u6211\u4eec\u5bf9\u4e8e\u4e00\u4e9b\u5e38\u89c1\u7684\u4e34\u65f6\u76ee\u5f55\uff0c\u6bd4\u5982\"_temporary\"\u3001\".staging\"\u3001\".hive-staging\"\u3001\"__magic\" \u4f1a\u89e6\u53d1\u591a\u7248\u672c\u64cd\u4f5c\u3002 { \"Version\": \"1\", \"Statement\": [ { \"Action\": [ \"oss:ListObjects\", \"oss:GetBucketInfo\", \"oss:GetObject\", \"oss:RestoreObject\", \"oss:ListObjectVersions\" ], \"Resource\": [ \"acs:oss:*:*:<yourBucketName>\", \"acs:oss:*:*:<yourBucketName>/*\" ], \"Effect\": \"Allow\" }, { \"Action\": [ \"oss:ListBuckets\" ], \"Resource\": \"*\", \"Effect\": \"Allow\" } ] } \u8bbf\u95ee\u5f00\u542f\u4e86\u591a\u7248\u672c\u529f\u80fd\u7684 OSS\uff08\u4e0d\u5efa\u8bae\u5f00\u542f\u591a\u7248\u672c\uff0c\u5bb9\u6613\u4ea7\u751f\u6027\u80fd\u95ee\u9898\uff09\uff1a { \"Version\": \"1\", \"Statement\": [ { \"Action\": [ \"oss:ListObjects\", \"oss:GetBucketInfo\", \"oss:PutObject\", \"oss:GetObject\", \"oss:DeleteObject\", \"oss:AbortMultipartUpload\", \"oss:ListParts\", \"oss:RestoreObject\", \"oss:ListObjectVersions\", \"oss:GetObjectVersion\", \"oss:DeleteObjectVersion\", \"oss:RestoreObjectVersion\" ], \"Resource\": [ \"acs:oss:*:*:<yourBucketName>\", \"acs:oss:*:*:<yourBucketName>/*\" ], \"Effect\": \"Allow\" }, { \"Action\": [ \"oss:ListBuckets\" ], \"Resource\": \"*\", \"Effect\": \"Allow\" } ] }","title":"\u5176\u4ed6\u81ea\u5b9a\u4e49\u573a\u666f\u793a\u4f8b"},{"location":"oss/oss_ram_policy/#bucket-policy","text":"\u5efa\u8bae\u4f7f\u7528\u201c\u5b8c\u5168\u63a7\u5236\u201d\uff0c\u907f\u514d\u51fa\u73b0\u4ee5\u4e0b\u95ee\u9898\uff1a \u300a\u4f7f\u7528\u5b50\u8d26\u53f7\u8bbf\u95ee OSS \u51fa\u73b0 AccessDenied \u95ee\u9898\u6392\u67e5\u300b","title":"Bucket Policy \u6388\u6743"},{"location":"oss/oss_ram_policy/#oss-hdfs","text":"{ \"Statement\": [ { \"Effect\": \"Allow\", \"Action\": \"oss:ListObjects\", \"Resource\": [ \"acs:oss:*:*:*\" ] }, { \"Effect\": \"Allow\", \"Action\": [ \"oss:GetBucketInfo\", \"oss:PostDataLakeStorageFileOperation\", \"oss:PostDataLakeStorageAdminOperation\" ], \"Resource\": \"*\" }, { \"Effect\": \"Allow\", \"Action\": \"oss:*\", \"Resource\": [ \"acs:oss:*:*:*/.dlsdata\", \"acs:oss:*:*:*/.dlsdata*\" ] } ], \"Version\": \"1\" }","title":"\u6388\u6743\u8bbf\u95ee OSS-HDFS"},{"location":"oss/issues/oss_access_denied/","text":"","title":"Oss access denied"},{"location":"oss/practices/oss_performance/","text":"OSS/OSS-HDFS \u7684\u6027\u80fd\u4f18\u5316\u6700\u4f73\u5b9e\u8df5 \u00a7 \u8d8a\u6765\u8d8a\u591a\u7684\u7528\u6237\u6784\u5efa\u6570\u636e\u6e56\uff0c\u5e76\u501f\u52a9 OSS \u6765\u5b58\u50a8\u6570\u636e\u3002OSS \u53ef\u63d0\u4f9b99.9999999999%\uff0812\u4e2a9\uff09\u7684\u6570\u636e\u6301\u4e45\u6027\uff0c99.995%\u7684\u6570\u636e\u53ef\u7528\u6027\u3002\u591a\u79cd\u5b58\u50a8\u7c7b\u578b\u4f9b\u9009\u62e9\uff0c\u5168\u9762\u4f18\u5316\u5b58\u50a8\u6210\u672c\u3002 \u4ece OSS \u4e0a\u4f20\u548c\u4e0b\u8f7d\u6570\u636e\u65f6\uff0c\u60a8\u7684\u5e94\u7528\u7a0b\u5e8f\u53ef\u4ee5\u8f7b\u677e\u5b9e\u73b0\u6bcf\u79d2\u6570\u5343\u4e2a\u4e8b\u52a1\u7684\u8bf7\u6c42\u6027\u80fd\u3002\u60a8\u7684\u5e94\u7528\u7a0b\u5e8f\u53ef\u4ee5\u5b9e\u73b0\u5b58\u50a8\u7a7a\u95f4\uff08Bucket\uff09\u4e2d\u6bcf\u4e2a\u5206\u533a\uff08OSS \u6309\u5bf9\u8c61\u540d\u7684\u5b57\u6bcd\u5e8f\u6392\u5217\u5bf9\u8c61\uff0c\u5e76\u6309\u7167\u6570\u636e\u91cf\u548c\u8bf7\u6c42 QPS \u6765\u62c6\u5206\u5206\u533a\uff09\u6bcf\u79d2\u81f3\u5c11 3,500 \u4e2a PUT/COPY/POST/DELETE \u8bf7\u6c42\u548c 5,500 \u4e2a GET/HEAD \u8bf7\u6c42\uff0c\u800c\u5b58\u50a8\u7a7a\u95f4\u4e2d\u7684\u524d\u7f00\u6570\u91cf\u662f\u6ca1\u6709\u9650\u5236\u7684\uff0c\u56e0\u6b64\u60a8\u53ef\u4ee5\u901a\u8fc7\u5e76\u884c\u8bfb\u53d6\u6765\u63d0\u9ad8\u8bfb\u53d6\u6216\u5199\u5165\u6027\u80fd\u3002\u4e5f\u5c31\u662f\u8bf4\uff0c\u5982\u679c\u60a8\u7684\u8bf7\u6c42\u7684\u6587\u4ef6\u524d\u7f00\u8bbe\u8ba1\u5408\u7406\uff0c\u5728\u67d0\u4e2a OSS Bucket \u4e2d\u521b\u5efa\u4e8610\u4e2a\u524d\u7f00\u6765\u5e76\u884c\u4e0a\u4f20\uff0c\u5219\u53ef\u4ee5\u5c06\u6570\u636e\u5199\u5165\u6027\u80fd\u6269\u5c55\u5230\u6bcf\u79d2 35,000 \u4e2a PUT \u8bf7\u6c42\u3002 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1) \u662f JindoFS \u5b58\u50a8\u7cfb\u7edf\u5728\u963f\u91cc\u4e91\u4e0a\u7684\u670d\u52a1\u5316\u90e8\u7f72\u5f62\u6001\uff0c\u548c\u963f\u91cc\u4e91 OSS \u6df1\u5ea6\u878d\u5408\uff0c\u5f00\u7bb1\u5373\u7528\uff0c\u65e0\u987b\u5728 EMR \u96c6\u7fa4\u4e2d\u90e8\u7f72\u7ef4\u62a4 JindoFS\uff0c\u514d\u8fd0\u7ef4\u3002OSS-HDFS \u670d\u52a1\u5177\u4f53\u4ecb\u7ecd\u8bf7\u53c2\u8003 OSS-HDFS\u670d\u52a1\u6982\u8ff0 \u3002 \u7ba1\u7406 HTTP \u901f\u5ea6\u7f13\u6162\u54cd\u5e94\u7684\u6700\u4f73\u5b9e\u8df5 \u00a7 \u8bbf\u95ee OSS/OSS-HDFS \u6570\u636e\u65f6\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528\u4ee5\u4e0b\u65b9\u6cd5\u63d0\u9ad8 HTTP \u8bf7\u6c42\u7684\u54cd\u5e94\u901f\u5ea6\uff1a \u5c06 OSS/OSS-HDFS \u548c ECS \u5b9e\u4f8b\u914d\u7f6e\u5728\u540c\u4e00\u5730\u57df \u901a\u8fc7\u6c34\u5e73\u6269\u5c55\u548c\u5e76\u884c\u8bf7\u6c42\u5b9e\u73b0\u9ad8\u541e\u5410\u91cf \u5bf9\u65f6\u5ef6\u654f\u611f\u7684\u5e94\u7528\u7a0b\u5e8f\u8fdb\u884c\u8d85\u65f6\u91cd\u8bd5 \u5bf9\u9891\u7e41\u8bbf\u95ee\u7684\u5185\u5bb9\u4f7f\u7528\u7f13\u5b58 \u4f7f\u7528\u6700\u65b0\u7248\u672c JindoSDK \u6211\u4eec\u5efa\u8bae\u60a8\u6839\u636e\u81ea\u5df1\u7684\u4f7f\u7528\u573a\u666f\u9009\u62e9\u5e76\u5e94\u7528\u6700\u9002\u5408\u7684\u9009\u9879\uff0c\u4ee5\u4f18\u5316 OSS/OSS-HDFS \u4e0a\u7684\u6570\u636e\u5904\u7406\u3002\u5728\u4ee5\u4e0b\u5404\u8282\u4e2d\uff0c\u6211\u4eec\u5c06\u4ecb\u7ecd\u6bcf\u79cd\u65b9\u6cd5\u7684\u6700\u4f73\u5b9e\u8df5\u3002 \u5c06 OSS/OSS-HDFS \u548c ECS \u5b9e\u4f8b\u914d\u7f6e\u5728\u540c\u4e00\u5730\u57df \u00a7 OSS\u5b58\u50a8\u7a7a\u95f4\u540d\u79f0\u662f\u5168\u5c40\u552f\u4e00\u7684\uff0c\u60a8\u5728\u521b\u5efa\u5b58\u50a8\u7a7a\u95f4\u65f6\u5fc5\u987b\u6307\u5b9a\u5730\u57df\uff0c\u4e14\u521b\u5efa\u4ee5\u540e\u4e0d\u80fd\u66f4\u6539\u540d\u79f0\u548c\u5730\u57df\u3002\u4e3a\u4e86\u4f18\u5316\u6027\u80fd\uff0c\u6211\u4eec\u5efa\u8bae\u60a8\u5c3d\u53ef\u80fd\u4ece\u540c\u4e00\u963f\u91cc\u4e91\u5730\u57df\u4e2d\u7684 ECS \u5b9e\u4f8b\u8bbf\u95ee OSS \u5b58\u50a8\u7a7a\u95f4\uff0c\u4ece\u800c\u964d\u4f4e\u7f51\u7edc\u5ef6\u8fdf\u548c\u6570\u636e\u4f20\u8f93\u6210\u672c\u3002\u8be6\u60c5\u8bf7\u53c2\u89c1 ECS\u5b9e\u4f8b\u901a\u8fc7OSS\u5185\u7f51\u5730\u5740\u8bbf\u95eeOSS\u8d44\u6e90 \u3002 \u901a\u8fc7\u6c34\u5e73\u6269\u5c55\u548c\u5e76\u53d1/\u5e76\u884c\u8bf7\u6c42\u5b9e\u73b0\u9ad8\u541e\u5410\u91cf \u00a7 OSS \u662f\u4e00\u4e2a\u8d85\u5927\u7684\u5206\u5e03\u5f0f\u7cfb\u7edf\u3002\u4e3a\u4e86\u5e2e\u52a9\u60a8\u5229\u7528\u5176\u89c4\u6a21\uff0c\u5efa\u8bae\u60a8\u5c06\u5e76\u884c\u8bf7\u6c42\u6c34\u5e73\u6269\u5c55\u5230 OSS \u670d\u52a1\u7ec8\u7aef\u8282\u70b9\uff0c\u8fd9\u79cd\u6269\u5c55\u65b9\u5f0f\u6709\u52a9\u4e8e\u901a\u8fc7\u7f51\u7edc\u5c06\u8d1f\u8f7d\u5206\u5e03\u5728\u591a\u4e2a\u8def\u5f84\u4e0a\u3002 OSS-HDFS \u4f7f\u7528\u4e86\u4e0a\u8ff0\u6700\u4f73\u5b9e\u8df5\uff0c\u901a\u8fc7\u5143\u6570\u636e\u670d\u52a1\u6765\u652f\u6301\u5c06\u901a\u8fc7\u6587\u4ef6\u5757\u7684\u65b9\u5f0f\uff0c\u5c06\u6587\u4ef6\u6c34\u5e73\u6269\u5c55\u5230 \u591a\u4e2a OSS \u670d\u52a1\u7ec8\u7aef\u4e0a\uff0c\u4ee5\u83b7\u53d6\u6700\u4f73\u7684\u8bfb\u5199\u6027\u80fd\u3002 \u5bf9\u4e8e\u9ad8\u541e\u5410\u91cf\u4f20\u8f93\uff0c\u60a8\u53ef\u4ee5\u6839\u636e\u60a8\u7684 ECS \u89c4\u683c\u548c\u5e94\u7528\u7a0b\u5e8f\uff0c\u8c03\u6574\u7279\u5b9a\u53c2\u6570\uff0c\u5728\u591a\u4e2a\u7ebf\u7a0b\u6216\u591a\u4e2a\u5b9e\u4f8b\u4e2d\u63a7\u5236\u4e0a\u4f20\u548c\u4e0b\u8f7d\u6570\u636e\u7684\u5e76\u53d1/\u5e76\u884c\u5ea6\u6765\u4f18\u5316\u541e\u5410\u91cf\u3002 \u6d4b\u91cf\u6027\u80fd\u6307\u6807 \u00a7 \u6839\u636eCPU\u3001\u7f51\u7edc\u541e\u5410\u91cf\u7684\u8981\u6c42\u6765\u8bc4\u4f30\u4e0d\u540c\u7684 ECS \u5b9e\u4f8b\u89c4\u683c\u3002\u6709\u5173\u5b9e\u4f8b\u89c4\u683c\u7684\u66f4\u591a\u4fe1\u606f\uff0c\u8bf7\u53c2\u89c1 ECS\u5b9e\u4f8b\u89c4\u683c \u3002\u6b64\u5916\uff0c\u5efa\u8bae\u60a8\u5728\u6d4b\u91cf\u6027\u80fd\u65f6\u4f7f\u7528HTTP\u5206\u6790\u5de5\u5177\u67e5\u770bDNS\u67e5\u8be2\u65f6\u95f4\u3001\u65f6\u5ef6\u548c\u6570\u636e\u4f20\u8f93\u901f\u5ea6 \u5f53\u60a8\u8981\u8c03\u6574\u5e76\u53d1\u7684\u8bf7\u6c42\u6570\u65f6\uff0c\u6027\u80fd\u6d4b\u91cf\u975e\u5e38\u91cd\u8981\u3002\u5efa\u8bae\u4ece\u5355\u4e2a\u8bf7\u6c42\u5f00\u59cb\uff0c\u6d4b\u91cf\u5f53\u524d\u7684\u7f51\u7edc\u5e26\u5bbd\u4ee5\u53ca\u5176\u4ed6\u8d44\u6e90\u7684\u4f7f\u7528\u60c5\u51b5\uff0c\u4ece\u800c\u8bc6\u522b\u74f6\u9888\u8d44\u6e90\uff08\u5373\u4f7f\u7528\u7387\u6700\u9ad8\u7684\u8d44\u6e90\uff09\uff0c\u4ee5\u53ca\u53ef\u80fd\u7684\u5e76\u53d1\u8bf7\u6c42\u6570\u3002\u4f8b\u5982\uff0c\u5982\u679c\u4e00\u6b21\u5904\u7406\u4e00\u4e2a\u8bf7\u6c42\u5bfc\u81f4CPU\u4f7f\u7528\u7387\u4e3a10%\uff0c\u5219\u8868\u660e\u6700\u591a\u53ef\u4ee5\u652f\u630110\u4e2a\u5e76\u53d1\u8bf7\u6c42\u3002 \u8c03\u6574\u5e76\u53d1\u4e0e\u5e76\u884c\u7684\u53c2\u6570 \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7\u4fee\u6539 fs.oss.download.thread.concurrency \u548c fs.oss.upload.thread.concurrency \u8c03\u6574\u5355\u4e2a\u8fdb\u7a0b\u5185\u4e0a\u4f20\u548c\u4e0b\u8f7d\u7684\u5e76\u53d1\u6570\u3002 \u5982\u679c\u60a8\u5728\u8fd0\u884c MapReduce \u4efb\u52a1\u65f6\u6216\u8005 Spark \u4efb\u52a1\uff0c\u60a8\u8fd8\u6709\u4ee5\u4e0b\u9009\u62e9\uff1a \u5bf9\u4e8e\u63d0\u4ea4 MapReduce \u4efb\u52a1\uff0c\u53ef\u4ee5\u901a\u8fc7 hadoop \u53c2\u6570 mapreduce.job.maps \u548c \u53c2\u6570mapreduce.job.reduces \u63a7\u5236\u5e76\u884c\u7684\u6267\u884c\u7a0b\u5e8f\u6570\u91cf\u3002 \u5bf9\u4e8e\u63d0\u4ea4 Spark \u6267\u884c\u7a0b\u5e8f\uff0c\u53ef\u4ee5\u901a\u8fc7\u9009\u9879 --num-executors \u6216\u8005 spark \u53c2\u6570 spark.executor.instance \u63a7\u5236\u5e76\u884c\u7684\u6267\u884c\u7a0b\u5e8f\u6570\u91cf\u3002 \u5bf9\u65f6\u5ef6\u654f\u611f\u7684\u5e94\u7528\u7a0b\u5e8f\u8fdb\u884c\u8d85\u65f6\u91cd\u8bd5 \u00a7 OSS \u9488\u5bf9\u7ba1\u63a7\u7c7bAPI\uff0c\u5982GetService\uff08ListBuckets\uff09\u3001PutBucket\u3001GetBucketLifecycle\u7b49\u8fdb\u884cQPS\u9650\u5236\u3002\u5982\u679c\u5e94\u7528\u7a0b\u5e8f\u4ea7\u751f\u9ad8\u8bf7\u6c42\u901f\u7387\uff0c\u53ef\u80fd\u4f1a\u6536\u5230 HTTP 503 \u51cf\u901f\u54cd\u5e94\u3002\u5982\u679c\u53d1\u751f\u8fd9\u7c7b\u9519\u8bef\uff0c\u5efa\u8bae\u60a8\u5ef6\u8fdf\u51e0\u79d2\u540e\u8fdb\u884c\u91cd\u8bd5\u3002 OSS\u4f1a\u5bf9\u8d26\u53f7\u7ea7\u7684\u603b\u8bbf\u95eeQPS\u505a\u9ed8\u8ba4\u9650\u5236\uff08\u8d26\u53f7\u7ea710,000/s\uff09\uff0c\u5982\u679c\u60a8\u9700\u8981\u66f4\u9ad8\u7684QPS\uff0c\u8bf7\u8054\u7cfb \u6280\u672f\u652f\u6301 \u3002\u6ce8\u610f\uff0c\u5728\u6574\u4f53\u8bbf\u95eeQPS\u672a\u8d85\u8fc7\u4e0a\u8ff0\u9608\u503c\u7684\u60c5\u51b5\u4e0b\uff0c\u5982\u679c\u8bf7\u6c42\u96c6\u4e2d\u5728\u7279\u5b9a\u5206\u533a\uff0c\u670d\u52a1\u7aef\u4e5f\u53ef\u80fd\u4f1a\u56e0\u4e3a\u8d85\u8fc7\u5355\u5206\u533a\u7684\u670d\u52a1\u80fd\u529b\u800c\u9650\u6d41\u5e76\u8fd4\u56de503\uff1b\u5982\u679c\u8bf7\u6c42\u524d\u7f00\u5408\u7406\u6253\u6563\uff08\u53c2\u89c1 OSS\u6027\u80fd\u4e0e\u6269\u5c55\u6027\u6700\u4f73\u5b9e\u8df5 \uff09\uff0cOSS\u4f1a\u81ea\u52a8\u6269\u5c55\u5206\u533a\u6570\u91cf\u6765\u652f\u6301\u66f4\u9ad8QPS\u7684\u8bbf\u95ee\uff0c\u60a8\u53ea\u9700\u8981\u7b49\u5f85\u5e76\u91cd\u8bd5\u5373\u53ef\u3002 \u60a8\u53ef\u4ee5\u901a\u8fc7\u8c03\u6574 fs.oss.retry.count \u63a7\u5236\u8bbf\u95ee OSS/OSS-HDFS \u7684\u91cd\u8bd5\u6b21\u6570\uff0c fs.oss.retry.interval.millisecond \u63a7\u5236\u8bbf\u95ee OSS/OSS-HDFS \u7684\u91cd\u8bd5\u95f4\u9694\uff08\u95f4\u9694\u4f1a\u968f\u91cd\u8bd5\u6b21\u6570\u7684\u589e\u52a0\u800c\u500d\u589e\uff09\u3002\u4e5f\u53ef\u4ee5\u6839\u636e\u7f51\u7edc\u60c5\u51b5\u6765\u8c03\u6574 fs.oss.timeout.millisecond \u8d85\u65f6\u65f6\u95f4\uff0c\u8fdb\u884c\u5408\u7406\u7684\u8d85\u65f6\u91cd\u8bd5\u3002 \u5bf9\u9891\u7e41\u8bbf\u95ee\u7684\u5185\u5bb9\u4f7f\u7528\u7f13\u5b58\u52a0\u901f \u00a7 \u5982\u679c\u60a8\u7684\u5e94\u7528\u7a0b\u5e8f\u9700\u8981\u652f\u6301\u540c\u4e00\u5730\u533a\u5927\u91cf\u8bbf\u95ee\u540c\u4e00\u9759\u6001\u6587\u4ef6\u7684\u573a\u666f\uff0c\u53ef\u4ee5\u4f7f\u7528 JindoCache \u7f13\u5b58\u670d\u52a1\u8fdb\u884c\u52a0\u901f\u3002JindoCache \u7f13\u5b58\u670d\u52a1\u5c06\u6587\u4ef6\u4ee5\u5757\u7684\u5f62\u5f0f\u5206\u6563\u5b58\u50a8\u5728\u5206\u5e03\u5f0f\u7f13\u5b58\u670d\u52a1\u4e2d\uff0c\u4ece\u800c\u907f\u514d\u91cd\u590d\u4ece OSS/OSS-HDFS \u53cd\u590d\u62c9\u53d6\u6570\u636e\uff0c\u6709\u6548\u51cf\u5c11\u8bbf\u95ee\u65f6\u5ef6\uff0c\u589e\u52a0\u8ba1\u7b97\u8d44\u6e90\u7684\u5229\u7528\u7387\u3002 \u4f7f\u7528\u6700\u65b0\u7248\u672c JindoSDK \u00a7 \u6700\u65b0\u7248\u672c\u7684 JindoSDK \u63d0\u4f9b\u4e86\u4f18\u5316\u7684\u81ea\u9002\u5e94\u914d\u7f6e\u548c\u9884\u8bfb\u7b97\u6cd5\uff0c\u5e76\u5b9a\u671f\u66f4\u65b0\u4ee5\u9075\u5faa\u65b0\u7684\u6700\u4f73\u5b9e\u8df5\u3002\u4f8b\u5982\uff0c\u5904\u7406\u4e0d\u540c\u7f51\u7edc\u9519\u8bef\u7684\u91cd\u8bd5\uff0c\u81ea\u9002\u5e94\u7684\u5e76\u53d1\u63a7\u5236\u7b49\u3002 \u4e0b\u8f7d\u94fe\u63a5","title":"OSS/OSS-HDFS \u7684\u6027\u80fd\u4f18\u5316\u6700\u4f73\u5b9e\u8df5"},{"location":"oss/practices/oss_performance/#ossoss-hdfs","text":"\u8d8a\u6765\u8d8a\u591a\u7684\u7528\u6237\u6784\u5efa\u6570\u636e\u6e56\uff0c\u5e76\u501f\u52a9 OSS \u6765\u5b58\u50a8\u6570\u636e\u3002OSS \u53ef\u63d0\u4f9b99.9999999999%\uff0812\u4e2a9\uff09\u7684\u6570\u636e\u6301\u4e45\u6027\uff0c99.995%\u7684\u6570\u636e\u53ef\u7528\u6027\u3002\u591a\u79cd\u5b58\u50a8\u7c7b\u578b\u4f9b\u9009\u62e9\uff0c\u5168\u9762\u4f18\u5316\u5b58\u50a8\u6210\u672c\u3002 \u4ece OSS \u4e0a\u4f20\u548c\u4e0b\u8f7d\u6570\u636e\u65f6\uff0c\u60a8\u7684\u5e94\u7528\u7a0b\u5e8f\u53ef\u4ee5\u8f7b\u677e\u5b9e\u73b0\u6bcf\u79d2\u6570\u5343\u4e2a\u4e8b\u52a1\u7684\u8bf7\u6c42\u6027\u80fd\u3002\u60a8\u7684\u5e94\u7528\u7a0b\u5e8f\u53ef\u4ee5\u5b9e\u73b0\u5b58\u50a8\u7a7a\u95f4\uff08Bucket\uff09\u4e2d\u6bcf\u4e2a\u5206\u533a\uff08OSS \u6309\u5bf9\u8c61\u540d\u7684\u5b57\u6bcd\u5e8f\u6392\u5217\u5bf9\u8c61\uff0c\u5e76\u6309\u7167\u6570\u636e\u91cf\u548c\u8bf7\u6c42 QPS \u6765\u62c6\u5206\u5206\u533a\uff09\u6bcf\u79d2\u81f3\u5c11 3,500 \u4e2a PUT/COPY/POST/DELETE \u8bf7\u6c42\u548c 5,500 \u4e2a GET/HEAD \u8bf7\u6c42\uff0c\u800c\u5b58\u50a8\u7a7a\u95f4\u4e2d\u7684\u524d\u7f00\u6570\u91cf\u662f\u6ca1\u6709\u9650\u5236\u7684\uff0c\u56e0\u6b64\u60a8\u53ef\u4ee5\u901a\u8fc7\u5e76\u884c\u8bfb\u53d6\u6765\u63d0\u9ad8\u8bfb\u53d6\u6216\u5199\u5165\u6027\u80fd\u3002\u4e5f\u5c31\u662f\u8bf4\uff0c\u5982\u679c\u60a8\u7684\u8bf7\u6c42\u7684\u6587\u4ef6\u524d\u7f00\u8bbe\u8ba1\u5408\u7406\uff0c\u5728\u67d0\u4e2a OSS Bucket \u4e2d\u521b\u5efa\u4e8610\u4e2a\u524d\u7f00\u6765\u5e76\u884c\u4e0a\u4f20\uff0c\u5219\u53ef\u4ee5\u5c06\u6570\u636e\u5199\u5165\u6027\u80fd\u6269\u5c55\u5230\u6bcf\u79d2 35,000 \u4e2a PUT \u8bf7\u6c42\u3002 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1) \u662f JindoFS \u5b58\u50a8\u7cfb\u7edf\u5728\u963f\u91cc\u4e91\u4e0a\u7684\u670d\u52a1\u5316\u90e8\u7f72\u5f62\u6001\uff0c\u548c\u963f\u91cc\u4e91 OSS \u6df1\u5ea6\u878d\u5408\uff0c\u5f00\u7bb1\u5373\u7528\uff0c\u65e0\u987b\u5728 EMR \u96c6\u7fa4\u4e2d\u90e8\u7f72\u7ef4\u62a4 JindoFS\uff0c\u514d\u8fd0\u7ef4\u3002OSS-HDFS \u670d\u52a1\u5177\u4f53\u4ecb\u7ecd\u8bf7\u53c2\u8003 OSS-HDFS\u670d\u52a1\u6982\u8ff0 \u3002","title":"OSS/OSS-HDFS \u7684\u6027\u80fd\u4f18\u5316\u6700\u4f73\u5b9e\u8df5"},{"location":"oss/practices/oss_performance/#http","text":"\u8bbf\u95ee OSS/OSS-HDFS \u6570\u636e\u65f6\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528\u4ee5\u4e0b\u65b9\u6cd5\u63d0\u9ad8 HTTP \u8bf7\u6c42\u7684\u54cd\u5e94\u901f\u5ea6\uff1a \u5c06 OSS/OSS-HDFS \u548c ECS \u5b9e\u4f8b\u914d\u7f6e\u5728\u540c\u4e00\u5730\u57df \u901a\u8fc7\u6c34\u5e73\u6269\u5c55\u548c\u5e76\u884c\u8bf7\u6c42\u5b9e\u73b0\u9ad8\u541e\u5410\u91cf \u5bf9\u65f6\u5ef6\u654f\u611f\u7684\u5e94\u7528\u7a0b\u5e8f\u8fdb\u884c\u8d85\u65f6\u91cd\u8bd5 \u5bf9\u9891\u7e41\u8bbf\u95ee\u7684\u5185\u5bb9\u4f7f\u7528\u7f13\u5b58 \u4f7f\u7528\u6700\u65b0\u7248\u672c JindoSDK \u6211\u4eec\u5efa\u8bae\u60a8\u6839\u636e\u81ea\u5df1\u7684\u4f7f\u7528\u573a\u666f\u9009\u62e9\u5e76\u5e94\u7528\u6700\u9002\u5408\u7684\u9009\u9879\uff0c\u4ee5\u4f18\u5316 OSS/OSS-HDFS \u4e0a\u7684\u6570\u636e\u5904\u7406\u3002\u5728\u4ee5\u4e0b\u5404\u8282\u4e2d\uff0c\u6211\u4eec\u5c06\u4ecb\u7ecd\u6bcf\u79cd\u65b9\u6cd5\u7684\u6700\u4f73\u5b9e\u8df5\u3002","title":"\u7ba1\u7406 HTTP \u901f\u5ea6\u7f13\u6162\u54cd\u5e94\u7684\u6700\u4f73\u5b9e\u8df5"},{"location":"oss/practices/oss_performance/#ossoss-hdfs-ecs","text":"OSS\u5b58\u50a8\u7a7a\u95f4\u540d\u79f0\u662f\u5168\u5c40\u552f\u4e00\u7684\uff0c\u60a8\u5728\u521b\u5efa\u5b58\u50a8\u7a7a\u95f4\u65f6\u5fc5\u987b\u6307\u5b9a\u5730\u57df\uff0c\u4e14\u521b\u5efa\u4ee5\u540e\u4e0d\u80fd\u66f4\u6539\u540d\u79f0\u548c\u5730\u57df\u3002\u4e3a\u4e86\u4f18\u5316\u6027\u80fd\uff0c\u6211\u4eec\u5efa\u8bae\u60a8\u5c3d\u53ef\u80fd\u4ece\u540c\u4e00\u963f\u91cc\u4e91\u5730\u57df\u4e2d\u7684 ECS \u5b9e\u4f8b\u8bbf\u95ee OSS \u5b58\u50a8\u7a7a\u95f4\uff0c\u4ece\u800c\u964d\u4f4e\u7f51\u7edc\u5ef6\u8fdf\u548c\u6570\u636e\u4f20\u8f93\u6210\u672c\u3002\u8be6\u60c5\u8bf7\u53c2\u89c1 ECS\u5b9e\u4f8b\u901a\u8fc7OSS\u5185\u7f51\u5730\u5740\u8bbf\u95eeOSS\u8d44\u6e90 \u3002","title":"\u5c06 OSS/OSS-HDFS \u548c ECS \u5b9e\u4f8b\u914d\u7f6e\u5728\u540c\u4e00\u5730\u57df"},{"location":"oss/practices/oss_performance/#_1","text":"OSS \u662f\u4e00\u4e2a\u8d85\u5927\u7684\u5206\u5e03\u5f0f\u7cfb\u7edf\u3002\u4e3a\u4e86\u5e2e\u52a9\u60a8\u5229\u7528\u5176\u89c4\u6a21\uff0c\u5efa\u8bae\u60a8\u5c06\u5e76\u884c\u8bf7\u6c42\u6c34\u5e73\u6269\u5c55\u5230 OSS \u670d\u52a1\u7ec8\u7aef\u8282\u70b9\uff0c\u8fd9\u79cd\u6269\u5c55\u65b9\u5f0f\u6709\u52a9\u4e8e\u901a\u8fc7\u7f51\u7edc\u5c06\u8d1f\u8f7d\u5206\u5e03\u5728\u591a\u4e2a\u8def\u5f84\u4e0a\u3002 OSS-HDFS \u4f7f\u7528\u4e86\u4e0a\u8ff0\u6700\u4f73\u5b9e\u8df5\uff0c\u901a\u8fc7\u5143\u6570\u636e\u670d\u52a1\u6765\u652f\u6301\u5c06\u901a\u8fc7\u6587\u4ef6\u5757\u7684\u65b9\u5f0f\uff0c\u5c06\u6587\u4ef6\u6c34\u5e73\u6269\u5c55\u5230 \u591a\u4e2a OSS \u670d\u52a1\u7ec8\u7aef\u4e0a\uff0c\u4ee5\u83b7\u53d6\u6700\u4f73\u7684\u8bfb\u5199\u6027\u80fd\u3002 \u5bf9\u4e8e\u9ad8\u541e\u5410\u91cf\u4f20\u8f93\uff0c\u60a8\u53ef\u4ee5\u6839\u636e\u60a8\u7684 ECS \u89c4\u683c\u548c\u5e94\u7528\u7a0b\u5e8f\uff0c\u8c03\u6574\u7279\u5b9a\u53c2\u6570\uff0c\u5728\u591a\u4e2a\u7ebf\u7a0b\u6216\u591a\u4e2a\u5b9e\u4f8b\u4e2d\u63a7\u5236\u4e0a\u4f20\u548c\u4e0b\u8f7d\u6570\u636e\u7684\u5e76\u53d1/\u5e76\u884c\u5ea6\u6765\u4f18\u5316\u541e\u5410\u91cf\u3002","title":"\u901a\u8fc7\u6c34\u5e73\u6269\u5c55\u548c\u5e76\u53d1/\u5e76\u884c\u8bf7\u6c42\u5b9e\u73b0\u9ad8\u541e\u5410\u91cf"},{"location":"oss/practices/oss_performance/#_2","text":"\u6839\u636eCPU\u3001\u7f51\u7edc\u541e\u5410\u91cf\u7684\u8981\u6c42\u6765\u8bc4\u4f30\u4e0d\u540c\u7684 ECS \u5b9e\u4f8b\u89c4\u683c\u3002\u6709\u5173\u5b9e\u4f8b\u89c4\u683c\u7684\u66f4\u591a\u4fe1\u606f\uff0c\u8bf7\u53c2\u89c1 ECS\u5b9e\u4f8b\u89c4\u683c \u3002\u6b64\u5916\uff0c\u5efa\u8bae\u60a8\u5728\u6d4b\u91cf\u6027\u80fd\u65f6\u4f7f\u7528HTTP\u5206\u6790\u5de5\u5177\u67e5\u770bDNS\u67e5\u8be2\u65f6\u95f4\u3001\u65f6\u5ef6\u548c\u6570\u636e\u4f20\u8f93\u901f\u5ea6 \u5f53\u60a8\u8981\u8c03\u6574\u5e76\u53d1\u7684\u8bf7\u6c42\u6570\u65f6\uff0c\u6027\u80fd\u6d4b\u91cf\u975e\u5e38\u91cd\u8981\u3002\u5efa\u8bae\u4ece\u5355\u4e2a\u8bf7\u6c42\u5f00\u59cb\uff0c\u6d4b\u91cf\u5f53\u524d\u7684\u7f51\u7edc\u5e26\u5bbd\u4ee5\u53ca\u5176\u4ed6\u8d44\u6e90\u7684\u4f7f\u7528\u60c5\u51b5\uff0c\u4ece\u800c\u8bc6\u522b\u74f6\u9888\u8d44\u6e90\uff08\u5373\u4f7f\u7528\u7387\u6700\u9ad8\u7684\u8d44\u6e90\uff09\uff0c\u4ee5\u53ca\u53ef\u80fd\u7684\u5e76\u53d1\u8bf7\u6c42\u6570\u3002\u4f8b\u5982\uff0c\u5982\u679c\u4e00\u6b21\u5904\u7406\u4e00\u4e2a\u8bf7\u6c42\u5bfc\u81f4CPU\u4f7f\u7528\u7387\u4e3a10%\uff0c\u5219\u8868\u660e\u6700\u591a\u53ef\u4ee5\u652f\u630110\u4e2a\u5e76\u53d1\u8bf7\u6c42\u3002","title":"\u6d4b\u91cf\u6027\u80fd\u6307\u6807"},{"location":"oss/practices/oss_performance/#_3","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7\u4fee\u6539 fs.oss.download.thread.concurrency \u548c fs.oss.upload.thread.concurrency \u8c03\u6574\u5355\u4e2a\u8fdb\u7a0b\u5185\u4e0a\u4f20\u548c\u4e0b\u8f7d\u7684\u5e76\u53d1\u6570\u3002 \u5982\u679c\u60a8\u5728\u8fd0\u884c MapReduce \u4efb\u52a1\u65f6\u6216\u8005 Spark \u4efb\u52a1\uff0c\u60a8\u8fd8\u6709\u4ee5\u4e0b\u9009\u62e9\uff1a \u5bf9\u4e8e\u63d0\u4ea4 MapReduce \u4efb\u52a1\uff0c\u53ef\u4ee5\u901a\u8fc7 hadoop \u53c2\u6570 mapreduce.job.maps \u548c \u53c2\u6570mapreduce.job.reduces \u63a7\u5236\u5e76\u884c\u7684\u6267\u884c\u7a0b\u5e8f\u6570\u91cf\u3002 \u5bf9\u4e8e\u63d0\u4ea4 Spark \u6267\u884c\u7a0b\u5e8f\uff0c\u53ef\u4ee5\u901a\u8fc7\u9009\u9879 --num-executors \u6216\u8005 spark \u53c2\u6570 spark.executor.instance \u63a7\u5236\u5e76\u884c\u7684\u6267\u884c\u7a0b\u5e8f\u6570\u91cf\u3002","title":"\u8c03\u6574\u5e76\u53d1\u4e0e\u5e76\u884c\u7684\u53c2\u6570"},{"location":"oss/practices/oss_performance/#_4","text":"OSS \u9488\u5bf9\u7ba1\u63a7\u7c7bAPI\uff0c\u5982GetService\uff08ListBuckets\uff09\u3001PutBucket\u3001GetBucketLifecycle\u7b49\u8fdb\u884cQPS\u9650\u5236\u3002\u5982\u679c\u5e94\u7528\u7a0b\u5e8f\u4ea7\u751f\u9ad8\u8bf7\u6c42\u901f\u7387\uff0c\u53ef\u80fd\u4f1a\u6536\u5230 HTTP 503 \u51cf\u901f\u54cd\u5e94\u3002\u5982\u679c\u53d1\u751f\u8fd9\u7c7b\u9519\u8bef\uff0c\u5efa\u8bae\u60a8\u5ef6\u8fdf\u51e0\u79d2\u540e\u8fdb\u884c\u91cd\u8bd5\u3002 OSS\u4f1a\u5bf9\u8d26\u53f7\u7ea7\u7684\u603b\u8bbf\u95eeQPS\u505a\u9ed8\u8ba4\u9650\u5236\uff08\u8d26\u53f7\u7ea710,000/s\uff09\uff0c\u5982\u679c\u60a8\u9700\u8981\u66f4\u9ad8\u7684QPS\uff0c\u8bf7\u8054\u7cfb \u6280\u672f\u652f\u6301 \u3002\u6ce8\u610f\uff0c\u5728\u6574\u4f53\u8bbf\u95eeQPS\u672a\u8d85\u8fc7\u4e0a\u8ff0\u9608\u503c\u7684\u60c5\u51b5\u4e0b\uff0c\u5982\u679c\u8bf7\u6c42\u96c6\u4e2d\u5728\u7279\u5b9a\u5206\u533a\uff0c\u670d\u52a1\u7aef\u4e5f\u53ef\u80fd\u4f1a\u56e0\u4e3a\u8d85\u8fc7\u5355\u5206\u533a\u7684\u670d\u52a1\u80fd\u529b\u800c\u9650\u6d41\u5e76\u8fd4\u56de503\uff1b\u5982\u679c\u8bf7\u6c42\u524d\u7f00\u5408\u7406\u6253\u6563\uff08\u53c2\u89c1 OSS\u6027\u80fd\u4e0e\u6269\u5c55\u6027\u6700\u4f73\u5b9e\u8df5 \uff09\uff0cOSS\u4f1a\u81ea\u52a8\u6269\u5c55\u5206\u533a\u6570\u91cf\u6765\u652f\u6301\u66f4\u9ad8QPS\u7684\u8bbf\u95ee\uff0c\u60a8\u53ea\u9700\u8981\u7b49\u5f85\u5e76\u91cd\u8bd5\u5373\u53ef\u3002 \u60a8\u53ef\u4ee5\u901a\u8fc7\u8c03\u6574 fs.oss.retry.count \u63a7\u5236\u8bbf\u95ee OSS/OSS-HDFS \u7684\u91cd\u8bd5\u6b21\u6570\uff0c fs.oss.retry.interval.millisecond \u63a7\u5236\u8bbf\u95ee OSS/OSS-HDFS \u7684\u91cd\u8bd5\u95f4\u9694\uff08\u95f4\u9694\u4f1a\u968f\u91cd\u8bd5\u6b21\u6570\u7684\u589e\u52a0\u800c\u500d\u589e\uff09\u3002\u4e5f\u53ef\u4ee5\u6839\u636e\u7f51\u7edc\u60c5\u51b5\u6765\u8c03\u6574 fs.oss.timeout.millisecond \u8d85\u65f6\u65f6\u95f4\uff0c\u8fdb\u884c\u5408\u7406\u7684\u8d85\u65f6\u91cd\u8bd5\u3002","title":"\u5bf9\u65f6\u5ef6\u654f\u611f\u7684\u5e94\u7528\u7a0b\u5e8f\u8fdb\u884c\u8d85\u65f6\u91cd\u8bd5"},{"location":"oss/practices/oss_performance/#_5","text":"\u5982\u679c\u60a8\u7684\u5e94\u7528\u7a0b\u5e8f\u9700\u8981\u652f\u6301\u540c\u4e00\u5730\u533a\u5927\u91cf\u8bbf\u95ee\u540c\u4e00\u9759\u6001\u6587\u4ef6\u7684\u573a\u666f\uff0c\u53ef\u4ee5\u4f7f\u7528 JindoCache \u7f13\u5b58\u670d\u52a1\u8fdb\u884c\u52a0\u901f\u3002JindoCache \u7f13\u5b58\u670d\u52a1\u5c06\u6587\u4ef6\u4ee5\u5757\u7684\u5f62\u5f0f\u5206\u6563\u5b58\u50a8\u5728\u5206\u5e03\u5f0f\u7f13\u5b58\u670d\u52a1\u4e2d\uff0c\u4ece\u800c\u907f\u514d\u91cd\u590d\u4ece OSS/OSS-HDFS \u53cd\u590d\u62c9\u53d6\u6570\u636e\uff0c\u6709\u6548\u51cf\u5c11\u8bbf\u95ee\u65f6\u5ef6\uff0c\u589e\u52a0\u8ba1\u7b97\u8d44\u6e90\u7684\u5229\u7528\u7387\u3002","title":"\u5bf9\u9891\u7e41\u8bbf\u95ee\u7684\u5185\u5bb9\u4f7f\u7528\u7f13\u5b58\u52a0\u901f"},{"location":"oss/practices/oss_performance/#jindosdk","text":"\u6700\u65b0\u7248\u672c\u7684 JindoSDK \u63d0\u4f9b\u4e86\u4f18\u5316\u7684\u81ea\u9002\u5e94\u914d\u7f6e\u548c\u9884\u8bfb\u7b97\u6cd5\uff0c\u5e76\u5b9a\u671f\u66f4\u65b0\u4ee5\u9075\u5faa\u65b0\u7684\u6700\u4f73\u5b9e\u8df5\u3002\u4f8b\u5982\uff0c\u5904\u7406\u4e0d\u540c\u7f51\u7edc\u9519\u8bef\u7684\u91cd\u8bd5\uff0c\u81ea\u9002\u5e94\u7684\u5e76\u53d1\u63a7\u5236\u7b49\u3002 \u4e0b\u8f7d\u94fe\u63a5","title":"\u4f7f\u7528\u6700\u65b0\u7248\u672c JindoSDK"},{"location":"oss/usages/oss_hadoop_shell/","text":"\u901a\u8fc7 Hadoop Shell \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS \u00a7 \u73af\u5883\u51c6\u5907 \u00a7 EMR\u73af\u5883\u4e2d\uff0c\u9ed8\u8ba4\u5df2\u5b89\u88c5 JindoSDK\uff0c\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528\u3002\u6ce8\u610f\uff1a \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u521b\u5efaEMR-3.42.0\u53ca\u4ee5\u4e0a\u7248\u672c\u6216EMR-5.8.0\u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u96c6\u7fa4\u3002 \u975e EMR \u73af\u5883\uff0c\u8bf7\u5148\u5b89\u88c5\u90e8\u7f72 JindoSDK\u3002\u90e8\u7f72\u65b9\u5f0f\u8bf7\u53c2\u8003 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u90e8\u7f72 JindoSDK 4.x \u53ca\u4ee5\u4e0a\u7248\u672c\u3002 \u901a\u8fc7 Hadoop Shell \u547d\u4ee4\u8bbf\u95ee \u00a7 \u901a\u8fc7 hadoop \u547d\u4ee4\u8bbf\u95ee OSS \u548c OSS-HDFS \u7684\u65b9\u5f0f\u57fa\u672c\u76f8\u540c\uff0c\u4ec5\u8def\u5f84\u4e2d\u7684 endpoint \u7565\u6709\u4e0d\u540c\u3002\u6839\u8def\u5f84\u793a\u4f8b\u53ef\u53c2\u89c1\uff1a \u300aOSS/OSS-HDFS \u5feb\u901f\u5165\u95e8\u300b \u4e0a\u4f20\u6587\u4ef6 \u5c06\u672c\u5730\u6839\u76ee\u5f55\u4e0b\u7684examplefile.txt\u6587\u4ef6\u4e0a\u4f20\u81f3examplebucket\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -put examplefile.txt oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u65b0\u5efa\u76ee\u5f55 \u5728examplebucket\u4e0b\u521b\u5efa\u540d\u4e3adir/\u7684\u76ee\u5f55\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -mkdir oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/dir/ \u67e5\u770b\u6587\u4ef6\u6216\u76ee\u5f55\u4fe1\u606f \u67e5\u770bexamplebucket\u4e0b\u7684\u6587\u4ef6\u6216\u76ee\u5f55\u4fe1\u606f\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -ls oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u83b7\u53d6\u6587\u4ef6\u6216\u76ee\u5f55\u5927\u5c0f \u83b7\u53d6examplebucket\u4e0b\u6240\u6709\u6587\u4ef6\u6216\u76ee\u5f55\u7684\u5927\u5c0f\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -du oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u67e5\u770b\u6587\u4ef6\u5185\u5bb9 \u67e5\u770bexamplebucket\u4e0b\u540d\u4e3alocalfile.txt\u7684\u6587\u4ef6\u5185\u5bb9\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -cat oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/localfile.txt \u91cd\u8981 \u67e5\u770b\u6587\u4ef6\u5185\u5bb9\u65f6\uff0c\u6587\u4ef6\u5185\u5bb9\u5c06\u4ee5\u7eaf\u6587\u672c\u5f62\u5f0f\u6253\u5370\u5230\u5c4f\u5e55\u4e0a\u3002\u5982\u679c\u6587\u4ef6\u5185\u5bb9\u8fdb\u884c\u4e86\u7279\u5b9a\u683c\u5f0f\u7684\u7f16\u7801\uff0c\u8bf7\u4f7f\u7528HDFS\u7684Java API\u8bfb\u53d6\u5e76\u89e3\u7801\u6587\u4ef6\u5185\u5bb9\u3002 \u62f7\u8d1d\u76ee\u5f55\u6216\u6587\u4ef6 \u4f8b\u5982\uff0c\u5c06examplebucket\u4e0b\u6839\u76ee\u5f55subdir1\u62f7\u8d1d\u5230\u76ee\u5f55subdir2\u4e0b\uff0c\u4e14\u6839\u76ee\u5f55subdir1\u6240\u5728\u7684\u4f4d\u7f6e\u3001\u6839\u76ee\u5f55\u4e0b\u7684\u6587\u4ef6\u548c\u5b50\u76ee\u5f55\u7ed3\u6784\u548c\u5185\u5bb9\u4fdd\u6301\u4e0d\u53d8\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -cp oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/subdir1 oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/subdir2/subdir1 \u79fb\u52a8\u76ee\u5f55\u6216\u6587\u4ef6 \u4f8b\u5982\uff0c\u5c06examplebucket\u6839\u76ee\u5f55srcdir\u53ca\u5176\u5305\u542b\u7684\u6587\u4ef6\u6216\u8005\u5b50\u76ee\u5f55\u79fb\u52a8\u81f3\u53e6\u4e00\u4e2a\u6839\u76ee\u5f55destdir\u4e0b\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -mv oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/srcdir oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/destdir \u4e0b\u8f7d\u6587\u4ef6 \u5c06examplebucket\u4e0b\u7684exampleobject.txt\u4e0b\u8f7d\u5230\u672c\u5730\u6839\u76ee\u5f55\u6587\u4ef6\u5939/tmp\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -get oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/exampleobject.txt /tmp/ \u5220\u9664\u76ee\u5f55\u6216\u6587\u4ef6 \u5220\u9664examplebucket\u4e0bdestfolder/\u76ee\u5f55\u53ca\u5176\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6587\u4ef6\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -rm oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/destfolder/","title":"HadoopShell"},{"location":"oss/usages/oss_hadoop_shell/#hadoop-shell-ossoss-hdfs","text":"","title":"\u901a\u8fc7 Hadoop Shell \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS"},{"location":"oss/usages/oss_hadoop_shell/#_1","text":"EMR\u73af\u5883\u4e2d\uff0c\u9ed8\u8ba4\u5df2\u5b89\u88c5 JindoSDK\uff0c\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528\u3002\u6ce8\u610f\uff1a \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u521b\u5efaEMR-3.42.0\u53ca\u4ee5\u4e0a\u7248\u672c\u6216EMR-5.8.0\u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u96c6\u7fa4\u3002 \u975e EMR \u73af\u5883\uff0c\u8bf7\u5148\u5b89\u88c5\u90e8\u7f72 JindoSDK\u3002\u90e8\u7f72\u65b9\u5f0f\u8bf7\u53c2\u8003 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u90e8\u7f72 JindoSDK 4.x \u53ca\u4ee5\u4e0a\u7248\u672c\u3002","title":"\u73af\u5883\u51c6\u5907"},{"location":"oss/usages/oss_hadoop_shell/#hadoop-shell","text":"\u901a\u8fc7 hadoop \u547d\u4ee4\u8bbf\u95ee OSS \u548c OSS-HDFS \u7684\u65b9\u5f0f\u57fa\u672c\u76f8\u540c\uff0c\u4ec5\u8def\u5f84\u4e2d\u7684 endpoint \u7565\u6709\u4e0d\u540c\u3002\u6839\u8def\u5f84\u793a\u4f8b\u53ef\u53c2\u89c1\uff1a \u300aOSS/OSS-HDFS \u5feb\u901f\u5165\u95e8\u300b \u4e0a\u4f20\u6587\u4ef6 \u5c06\u672c\u5730\u6839\u76ee\u5f55\u4e0b\u7684examplefile.txt\u6587\u4ef6\u4e0a\u4f20\u81f3examplebucket\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -put examplefile.txt oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u65b0\u5efa\u76ee\u5f55 \u5728examplebucket\u4e0b\u521b\u5efa\u540d\u4e3adir/\u7684\u76ee\u5f55\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -mkdir oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/dir/ \u67e5\u770b\u6587\u4ef6\u6216\u76ee\u5f55\u4fe1\u606f \u67e5\u770bexamplebucket\u4e0b\u7684\u6587\u4ef6\u6216\u76ee\u5f55\u4fe1\u606f\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -ls oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u83b7\u53d6\u6587\u4ef6\u6216\u76ee\u5f55\u5927\u5c0f \u83b7\u53d6examplebucket\u4e0b\u6240\u6709\u6587\u4ef6\u6216\u76ee\u5f55\u7684\u5927\u5c0f\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -du oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u67e5\u770b\u6587\u4ef6\u5185\u5bb9 \u67e5\u770bexamplebucket\u4e0b\u540d\u4e3alocalfile.txt\u7684\u6587\u4ef6\u5185\u5bb9\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -cat oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/localfile.txt \u91cd\u8981 \u67e5\u770b\u6587\u4ef6\u5185\u5bb9\u65f6\uff0c\u6587\u4ef6\u5185\u5bb9\u5c06\u4ee5\u7eaf\u6587\u672c\u5f62\u5f0f\u6253\u5370\u5230\u5c4f\u5e55\u4e0a\u3002\u5982\u679c\u6587\u4ef6\u5185\u5bb9\u8fdb\u884c\u4e86\u7279\u5b9a\u683c\u5f0f\u7684\u7f16\u7801\uff0c\u8bf7\u4f7f\u7528HDFS\u7684Java API\u8bfb\u53d6\u5e76\u89e3\u7801\u6587\u4ef6\u5185\u5bb9\u3002 \u62f7\u8d1d\u76ee\u5f55\u6216\u6587\u4ef6 \u4f8b\u5982\uff0c\u5c06examplebucket\u4e0b\u6839\u76ee\u5f55subdir1\u62f7\u8d1d\u5230\u76ee\u5f55subdir2\u4e0b\uff0c\u4e14\u6839\u76ee\u5f55subdir1\u6240\u5728\u7684\u4f4d\u7f6e\u3001\u6839\u76ee\u5f55\u4e0b\u7684\u6587\u4ef6\u548c\u5b50\u76ee\u5f55\u7ed3\u6784\u548c\u5185\u5bb9\u4fdd\u6301\u4e0d\u53d8\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -cp oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/subdir1 oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/subdir2/subdir1 \u79fb\u52a8\u76ee\u5f55\u6216\u6587\u4ef6 \u4f8b\u5982\uff0c\u5c06examplebucket\u6839\u76ee\u5f55srcdir\u53ca\u5176\u5305\u542b\u7684\u6587\u4ef6\u6216\u8005\u5b50\u76ee\u5f55\u79fb\u52a8\u81f3\u53e6\u4e00\u4e2a\u6839\u76ee\u5f55destdir\u4e0b\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -mv oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/srcdir oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/destdir \u4e0b\u8f7d\u6587\u4ef6 \u5c06examplebucket\u4e0b\u7684exampleobject.txt\u4e0b\u8f7d\u5230\u672c\u5730\u6839\u76ee\u5f55\u6587\u4ef6\u5939/tmp\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -get oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/exampleobject.txt /tmp/ \u5220\u9664\u76ee\u5f55\u6216\u6587\u4ef6 \u5220\u9664examplebucket\u4e0bdestfolder/\u76ee\u5f55\u53ca\u5176\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6587\u4ef6\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -rm oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/destfolder/","title":"\u901a\u8fc7 Hadoop Shell \u547d\u4ee4\u8bbf\u95ee"},{"location":"oss/usages/oss_jindo_cli/","text":"\u901a\u8fc7 Jindo CLI \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS \u00a7 \u73af\u5883\u51c6\u5907 \u00a7 EMR\u73af\u5883\u4e2d\uff0c\u9ed8\u8ba4\u5df2\u5b89\u88c5 JindoSDK\uff0c\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528\u3002\u6ce8\u610f\uff1a \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u521b\u5efaEMR-3.44.0\u53ca\u4ee5\u4e0a\u7248\u672c\u6216EMR-5.10.0\u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u96c6\u7fa4\u3002 \u975e EMR \u73af\u5883\uff0c\u8bf7\u5148\u5b89\u88c5\u90e8\u7f72 JindoSDK\u3002\u90e8\u7f72\u65b9\u5f0f\u8bf7\u53c2\u8003 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u90e8\u7f72 JindoSDK 4.6.x \u53ca\u4ee5\u4e0a\u7248\u672c\u3002 \u901a\u8fc7 Jindo CLI \u547d\u4ee4\u8bbf\u95ee \u00a7 \u901a\u8fc7 Jindo CLI \u547d\u4ee4\u8bbf\u95ee OSS \u548c OSS-HDFS \u7684\u65b9\u5f0f\u57fa\u672c\u76f8\u540c\uff0c\u4ec5\u8def\u5f84\u4e2d\u7684 endpoint \u7565\u6709\u4e0d\u540c\u3002\u6839\u8def\u5f84\u793a\u4f8b\u53ef\u53c2\u89c1\uff1a \u300aOSS/OSS-HDFS \u5feb\u901f\u5165\u95e8\u300b \u4e0a\u4f20\u6587\u4ef6 \u5c06\u672c\u5730\u6839\u76ee\u5f55\u4e0b\u7684examplefile.txt\u6587\u4ef6\u4e0a\u4f20\u81f3examplebucket\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -put examplefile.txt oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u65b0\u5efa\u76ee\u5f55 \u5728examplebucket\u4e0b\u521b\u5efa\u540d\u4e3adir/\u7684\u76ee\u5f55\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -mkdir oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/dir/ \u67e5\u770b\u6587\u4ef6\u6216\u76ee\u5f55\u4fe1\u606f \u67e5\u770bexamplebucket\u4e0b\u7684\u6587\u4ef6\u6216\u76ee\u5f55\u4fe1\u606f\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -ls oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u83b7\u53d6\u6587\u4ef6\u6216\u76ee\u5f55\u5927\u5c0f \u83b7\u53d6examplebucket\u4e0b\u6240\u6709\u6587\u4ef6\u6216\u76ee\u5f55\u7684\u5927\u5c0f\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -du oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u67e5\u770b\u6587\u4ef6\u5185\u5bb9 \u67e5\u770bexamplebucket\u4e0b\u540d\u4e3alocalfile.txt\u7684\u6587\u4ef6\u5185\u5bb9\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -cat oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/localfile.txt \u91cd\u8981 \u67e5\u770b\u6587\u4ef6\u5185\u5bb9\u65f6\uff0c\u6587\u4ef6\u5185\u5bb9\u5c06\u4ee5\u7eaf\u6587\u672c\u5f62\u5f0f\u6253\u5370\u5230\u5c4f\u5e55\u4e0a\u3002\u5982\u679c\u6587\u4ef6\u5185\u5bb9\u8fdb\u884c\u4e86\u7279\u5b9a\u683c\u5f0f\u7684\u7f16\u7801\uff0c\u8bf7\u4f7f\u7528HDFS\u7684Java API\u8bfb\u53d6\u5e76\u89e3\u7801\u6587\u4ef6\u5185\u5bb9\u3002 \u4e0b\u8f7d\u6587\u4ef6 \u5c06examplebucket\u4e0b\u7684exampleobject.txt\u4e0b\u8f7d\u5230\u672c\u5730\u6839\u76ee\u5f55\u6587\u4ef6\u5939/tmp\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -get oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/exampleobject.txt /tmp/ \u5220\u9664\u76ee\u5f55\u6216\u6587\u4ef6 \u5220\u9664examplebucket\u4e0bdestfolder/\u76ee\u5f55\u53ca\u5176\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6587\u4ef6\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -rm oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/destfolder/ \u9644\u5f55 \u00a7 \u66f4\u591a\u547d\u4ee4\u53ca\u63cf\u8ff0\uff0c\u8be6\u89c1 \u300aJindo CLI \u53c2\u6570\u8bf4\u660e\u300b","title":"JindoCli \u5feb\u901f\u5165\u95e8"},{"location":"oss/usages/oss_jindo_cli/#jindo-cli-ossoss-hdfs","text":"","title":"\u901a\u8fc7 Jindo CLI \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS"},{"location":"oss/usages/oss_jindo_cli/#_1","text":"EMR\u73af\u5883\u4e2d\uff0c\u9ed8\u8ba4\u5df2\u5b89\u88c5 JindoSDK\uff0c\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528\u3002\u6ce8\u610f\uff1a \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u521b\u5efaEMR-3.44.0\u53ca\u4ee5\u4e0a\u7248\u672c\u6216EMR-5.10.0\u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u96c6\u7fa4\u3002 \u975e EMR \u73af\u5883\uff0c\u8bf7\u5148\u5b89\u88c5\u90e8\u7f72 JindoSDK\u3002\u90e8\u7f72\u65b9\u5f0f\u8bf7\u53c2\u8003 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u90e8\u7f72 JindoSDK 4.6.x \u53ca\u4ee5\u4e0a\u7248\u672c\u3002","title":"\u73af\u5883\u51c6\u5907"},{"location":"oss/usages/oss_jindo_cli/#jindo-cli","text":"\u901a\u8fc7 Jindo CLI \u547d\u4ee4\u8bbf\u95ee OSS \u548c OSS-HDFS \u7684\u65b9\u5f0f\u57fa\u672c\u76f8\u540c\uff0c\u4ec5\u8def\u5f84\u4e2d\u7684 endpoint \u7565\u6709\u4e0d\u540c\u3002\u6839\u8def\u5f84\u793a\u4f8b\u53ef\u53c2\u89c1\uff1a \u300aOSS/OSS-HDFS \u5feb\u901f\u5165\u95e8\u300b \u4e0a\u4f20\u6587\u4ef6 \u5c06\u672c\u5730\u6839\u76ee\u5f55\u4e0b\u7684examplefile.txt\u6587\u4ef6\u4e0a\u4f20\u81f3examplebucket\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -put examplefile.txt oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u65b0\u5efa\u76ee\u5f55 \u5728examplebucket\u4e0b\u521b\u5efa\u540d\u4e3adir/\u7684\u76ee\u5f55\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -mkdir oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/dir/ \u67e5\u770b\u6587\u4ef6\u6216\u76ee\u5f55\u4fe1\u606f \u67e5\u770bexamplebucket\u4e0b\u7684\u6587\u4ef6\u6216\u76ee\u5f55\u4fe1\u606f\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -ls oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u83b7\u53d6\u6587\u4ef6\u6216\u76ee\u5f55\u5927\u5c0f \u83b7\u53d6examplebucket\u4e0b\u6240\u6709\u6587\u4ef6\u6216\u76ee\u5f55\u7684\u5927\u5c0f\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -du oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u67e5\u770b\u6587\u4ef6\u5185\u5bb9 \u67e5\u770bexamplebucket\u4e0b\u540d\u4e3alocalfile.txt\u7684\u6587\u4ef6\u5185\u5bb9\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -cat oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/localfile.txt \u91cd\u8981 \u67e5\u770b\u6587\u4ef6\u5185\u5bb9\u65f6\uff0c\u6587\u4ef6\u5185\u5bb9\u5c06\u4ee5\u7eaf\u6587\u672c\u5f62\u5f0f\u6253\u5370\u5230\u5c4f\u5e55\u4e0a\u3002\u5982\u679c\u6587\u4ef6\u5185\u5bb9\u8fdb\u884c\u4e86\u7279\u5b9a\u683c\u5f0f\u7684\u7f16\u7801\uff0c\u8bf7\u4f7f\u7528HDFS\u7684Java API\u8bfb\u53d6\u5e76\u89e3\u7801\u6587\u4ef6\u5185\u5bb9\u3002 \u4e0b\u8f7d\u6587\u4ef6 \u5c06examplebucket\u4e0b\u7684exampleobject.txt\u4e0b\u8f7d\u5230\u672c\u5730\u6839\u76ee\u5f55\u6587\u4ef6\u5939/tmp\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -get oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/exampleobject.txt /tmp/ \u5220\u9664\u76ee\u5f55\u6216\u6587\u4ef6 \u5220\u9664examplebucket\u4e0bdestfolder/\u76ee\u5f55\u53ca\u5176\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6587\u4ef6\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -rm oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/destfolder/","title":"\u901a\u8fc7 Jindo CLI \u547d\u4ee4\u8bbf\u95ee"},{"location":"oss/usages/oss_jindo_cli/#_2","text":"\u66f4\u591a\u547d\u4ee4\u53ca\u63cf\u8ff0\uff0c\u8be6\u89c1 \u300aJindo CLI \u53c2\u6570\u8bf4\u660e\u300b","title":"\u9644\u5f55"},{"location":"oss/usages/oss_posix/","text":"\u901a\u8fc7 POSIX \u8bbf\u95ee OSS/OSS-HDFS \u00a7 \u73af\u5883\u51c6\u5907 \u00a7 EMR\u73af\u5883\u4e2d\uff0c\u9ed8\u8ba4\u5df2\u5b89\u88c5 JindoSDK\uff0c\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528\u3002\u6ce8\u610f\uff1a \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u521b\u5efaEMR-3.42.0\u53ca\u4ee5\u4e0a\u7248\u672c\u6216EMR-5.8.0\u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u96c6\u7fa4\u3002 \u975e EMR \u73af\u5883\uff0c\u8bf7\u5148\u5b89\u88c5\u90e8\u7f72 JindoSDK\u3002\u90e8\u7f72\u65b9\u5f0f\u8bf7\u53c2\u8003 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u90e8\u7f72 JindoSDK 4.x \u53ca\u4ee5\u4e0a\u7248\u672c\u3002 \u4f9d\u8d56\u51c6\u5907 \u00a7 EMR-3.44.0\u53ca\u4ee5\u4e0a\u7248\u672c\u6216EMR-5.10.0\u53ca\u4ee5\u4e0a\u7248\u672c\uff0c\u5df2\u9ed8\u8ba4\u5b89\u88c5\u6240\u9700\u4f9d\u8d56\uff0c\u53ef\u8df3\u8fc7\u672c\u7ae0\u8282\u3002 JindoSDK 4.5.0 \u53ca\u4e4b\u524d\u7248\u672c # CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev JindoSDK 4.5.1 \u53ca\u4ee5\u4e0a\u7248\u672c\uff0c\u9700\u4f9d\u8d56libfuse 3.7+\uff0c\u4ee5\u5b89\u88c5fuse-3.11\u4e3a\u4f8b\uff1a # build fuse required meson & ninja, for debian: apt install -y pkg-config meson ninja-build sudo yum install -y meson ninja-build # compile fuse required newer g++ (only CentOS) sudo yum install -y scl-utils sudo yum install -y alinux-release-experimentals sudo yum install -y devtoolset-8-gcc devtoolset-8-gdb devtoolset-8-binutils devtoolset-8-make devtoolset-8-gcc-c++ sudo su -c \"echo 'source /opt/rh/devtoolset-8/enable' > /etc/profile.d/g++.sh\" source /opt/rh/devtoolset-8/enable sudo ln -s /opt/rh/devtoolset-8/root/bin/gcc /usr/local/bin/gcc sudo ln -s /opt/rh/devtoolset-8/root/bin/g++ /usr/local/bin/g++ # compile & install libfuse wget https://github.com/libfuse/libfuse/releases/download/fuse-3.11.0/fuse-3.11.0.tar.xz xz -d fuse-3.11.0.tar.xz tar xf fuse-3.11.0.tar cd fuse-3.11.0/ mkdir build; cd build meson .. sudo ninja install \u6302\u8f7d JindoFuse \u00a7 \u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002 \u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a mkdir -p <mountpoint> \u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a jindo-fuse <mount_point> -ouri=[<osspath>] \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 <oss_path> \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 <mount_point> \u3002 <mount_point> \u9700\u66ff\u6362\u4e3a\u4e00\u4e2a\u672c\u5730\u8def\u5f84\u3002 <oss_path> \u9700\u66ff\u6362\u4e3a\u5f85\u6620\u5c04\u7684 OSS/OSS-HDFS \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a Bucket \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002\u5982 oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/subdir/ \u6302\u8f7d OSS \u548c OSS-HDFS \u8def\u5f84\u7684\u65b9\u5f0f\u57fa\u672c\u76f8\u540c\uff0c\u4ec5\u8def\u5f84\u4e2d\u7684 endpoint \u7565\u6709\u4e0d\u540c\u3002\u6839\u8def\u5f84\u793a\u4f8b\u53ef\u53c2\u89c1\uff1a \u300aOSS/OSS-HDFS \u5feb\u901f\u5165\u95e8\u300b \u786e\u8ba4\u6302\u8f7d\u6210\u529f\uff0cjindo-fuse \u8fdb\u7a0b\u5b58\u5728\uff0c\u4e14\u542f\u52a8\u53c2\u6570\u4e0e\u9884\u671f\u4e00\u81f4 ps -ef | grep jindo-fuse \u8bbf\u95ee JindoFuse \u00a7 \u5982\u679c\u5c06 JindoFS \u670d\u52a1\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/ \u5378\u8f7d JindoFuse \u00a7 \u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point> \u81ea\u52a8\u5378\u8f7d JindoFuse \u00a7 \u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u9644\u5f55 \u00a7 \u66f4\u591a\u547d\u4ee4\u53ca\u63cf\u8ff0\uff0c\u8be6\u89c1 \u300aJindoFUSE \u4f7f\u7528\u6307\u5357\u300b","title":"\u7528 POSIX \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS"},{"location":"oss/usages/oss_posix/#posix-ossoss-hdfs","text":"","title":"\u901a\u8fc7 POSIX \u8bbf\u95ee OSS/OSS-HDFS"},{"location":"oss/usages/oss_posix/#_1","text":"EMR\u73af\u5883\u4e2d\uff0c\u9ed8\u8ba4\u5df2\u5b89\u88c5 JindoSDK\uff0c\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528\u3002\u6ce8\u610f\uff1a \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u521b\u5efaEMR-3.42.0\u53ca\u4ee5\u4e0a\u7248\u672c\u6216EMR-5.8.0\u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u96c6\u7fa4\u3002 \u975e EMR \u73af\u5883\uff0c\u8bf7\u5148\u5b89\u88c5\u90e8\u7f72 JindoSDK\u3002\u90e8\u7f72\u65b9\u5f0f\u8bf7\u53c2\u8003 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u90e8\u7f72 JindoSDK 4.x \u53ca\u4ee5\u4e0a\u7248\u672c\u3002","title":"\u73af\u5883\u51c6\u5907"},{"location":"oss/usages/oss_posix/#_2","text":"EMR-3.44.0\u53ca\u4ee5\u4e0a\u7248\u672c\u6216EMR-5.10.0\u53ca\u4ee5\u4e0a\u7248\u672c\uff0c\u5df2\u9ed8\u8ba4\u5b89\u88c5\u6240\u9700\u4f9d\u8d56\uff0c\u53ef\u8df3\u8fc7\u672c\u7ae0\u8282\u3002 JindoSDK 4.5.0 \u53ca\u4e4b\u524d\u7248\u672c # CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev JindoSDK 4.5.1 \u53ca\u4ee5\u4e0a\u7248\u672c\uff0c\u9700\u4f9d\u8d56libfuse 3.7+\uff0c\u4ee5\u5b89\u88c5fuse-3.11\u4e3a\u4f8b\uff1a # build fuse required meson & ninja, for debian: apt install -y pkg-config meson ninja-build sudo yum install -y meson ninja-build # compile fuse required newer g++ (only CentOS) sudo yum install -y scl-utils sudo yum install -y alinux-release-experimentals sudo yum install -y devtoolset-8-gcc devtoolset-8-gdb devtoolset-8-binutils devtoolset-8-make devtoolset-8-gcc-c++ sudo su -c \"echo 'source /opt/rh/devtoolset-8/enable' > /etc/profile.d/g++.sh\" source /opt/rh/devtoolset-8/enable sudo ln -s /opt/rh/devtoolset-8/root/bin/gcc /usr/local/bin/gcc sudo ln -s /opt/rh/devtoolset-8/root/bin/g++ /usr/local/bin/g++ # compile & install libfuse wget https://github.com/libfuse/libfuse/releases/download/fuse-3.11.0/fuse-3.11.0.tar.xz xz -d fuse-3.11.0.tar.xz tar xf fuse-3.11.0.tar cd fuse-3.11.0/ mkdir build; cd build meson .. sudo ninja install","title":"\u4f9d\u8d56\u51c6\u5907"},{"location":"oss/usages/oss_posix/#jindofuse","text":"\u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002 \u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a mkdir -p <mountpoint> \u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a jindo-fuse <mount_point> -ouri=[<osspath>] \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 <oss_path> \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 <mount_point> \u3002 <mount_point> \u9700\u66ff\u6362\u4e3a\u4e00\u4e2a\u672c\u5730\u8def\u5f84\u3002 <oss_path> \u9700\u66ff\u6362\u4e3a\u5f85\u6620\u5c04\u7684 OSS/OSS-HDFS \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a Bucket \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002\u5982 oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/subdir/ \u6302\u8f7d OSS \u548c OSS-HDFS \u8def\u5f84\u7684\u65b9\u5f0f\u57fa\u672c\u76f8\u540c\uff0c\u4ec5\u8def\u5f84\u4e2d\u7684 endpoint \u7565\u6709\u4e0d\u540c\u3002\u6839\u8def\u5f84\u793a\u4f8b\u53ef\u53c2\u89c1\uff1a \u300aOSS/OSS-HDFS \u5feb\u901f\u5165\u95e8\u300b \u786e\u8ba4\u6302\u8f7d\u6210\u529f\uff0cjindo-fuse \u8fdb\u7a0b\u5b58\u5728\uff0c\u4e14\u542f\u52a8\u53c2\u6570\u4e0e\u9884\u671f\u4e00\u81f4 ps -ef | grep jindo-fuse","title":"\u6302\u8f7d JindoFuse"},{"location":"oss/usages/oss_posix/#jindofuse_1","text":"\u5982\u679c\u5c06 JindoFS \u670d\u52a1\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/","title":"\u8bbf\u95ee JindoFuse"},{"location":"oss/usages/oss_posix/#jindofuse_2","text":"\u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point>","title":"\u5378\u8f7d JindoFuse"},{"location":"oss/usages/oss_posix/#jindofuse_3","text":"\u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002","title":"\u81ea\u52a8\u5378\u8f7d JindoFuse"},{"location":"oss/usages/oss_posix/#_3","text":"\u66f4\u591a\u547d\u4ee4\u53ca\u63cf\u8ff0\uff0c\u8be6\u89c1 \u300aJindoFUSE \u4f7f\u7528\u6307\u5357\u300b","title":"\u9644\u5f55"},{"location":"upgrade/emr2_upgrade_jindocache/","text":"EMR \u65b0\u7248\u96c6\u7fa4 JindoCache \u5347\u7ea7\u6587\u6863 \u00a7 \u573a\u666f\u4e00\uff1a\u5347\u7ea7\u5df2\u6709\u65b0\u7248\u96c6\u7fa4 \u00a7 \u82e5\u5df2\u6709\u65b0\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u3002\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u6216\u8005\u9700\u8981\u4f7f\u7528 JindoCache \u7684\u65b0\u529f\u80fd, \u53ef\u4ee5\u6839\u636e\u4e0b\u9762\u7684\u6b65\u9aa4\u5b8c\u6210 JindoCache \u5347\u7ea7\u3002 1\u3001\u505c\u6b62\u5df2\u6709\u7684 JindoCache \u96c6\u7fa4 \u00a7 \u5728EMR\u96c6\u7fa4\u7684 JindoCache \u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u505c\u6b62 \u3002 2\u3001\u51c6\u5907\u8f6f\u4ef6\u5305\u548c\u5347\u7ea7\u811a\u672c\u3001\u914d\u7f6e \u00a7 \u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728emr-user\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528emr-user\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - emr-user cd /home/emr-user/ wget http://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindocache-patches.tar.gz tar zxf jindocache-patches.tar.gz \u4e0b\u8f7d jindocache \u8f6f\u4ef6\u5305 jindocache-{VERSION}-{PLATFORM}.tar.gz\uff0c\u89e3\u538b\u540e\u5230 jindocache-patches\u76ee\u5f55\u4e0b\u3002 \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 jindocache \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86 \u5e73\u53f0\u4e3a\u4f8b: cd jindocache-patches wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindocache-6.7.3-linux.tar.gz tar -zxvf jindocache-6.7.3-linux.tar.gz ls -l jindocache-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-rw-r-- 1 emr-user emr-user 29 6\u6708 6 11:21 hosts drwxr-xr-x 7 emr-user emr-user 4096 8\u6708 8 16:19 jindocache-6.7.3-linux -rw-r--r-- 1 emr-user emr-user 276406433 8\u6708 12 14:32 jindocache-6.7.3-linux.tar.gz -rwxr-xr-x 1 emr-user emr-user 1241 6\u6708 6 11:18 upgrade.sh 3\u3001\u914d\u7f6e\u5347\u7ea7\u8282\u70b9\u4fe1\u606f \u00a7 \u7f16\u8f91 patch \u5305\u4e0b\u7684 hosts \u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684 hostname\uff0c\u5982 master-1-1 \u6216 core-1-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindocache-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a master-1-1 core-1-1 core-1-2 \u53ef\u5c1d\u8bd5\u4f7f\u7528\u811a\u672c\u83b7\u53d6\u5168\u90e8\u8282\u70b9\u4fe1\u606f\uff0c\u5982\u679c hosts \u83b7\u53d6\u5931\u8d25\uff0c\u9700\u8981\u624b\u52a8\u8865\u5168 cat /usr/local/taihao-executor-all/data/cache/.cluster_context | jq --raw-output '.nodes[].hostname.alias[]' > hosts 4\u3001\u6267\u884c\u5347\u7ea7 \u00a7 \u901a\u8fc7 upgrade.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 bash upgrade.sh $NEW_jindocache_VERSION $PLATFORM \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 jindocache \u5347\u7ea7\u5230 linux \u5e73\u53f0 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash upgrade.sh 6.7.3 linux \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 Start upgrade JindoCache Service to 6.7.3 with platform linux [1] 11:14:25 [SUCCESS] core-1-2 [2] 11:14:26 [SUCCESS] core-1-1 [3] 11:14:27 [SUCCESS] master-1-1 [1] 11:14:52 [SUCCESS] core-1-2 [2] 11:14:52 [SUCCESS] master-1-1 [3] 11:14:52 [SUCCESS] core-1-1 [1] 11:14:55 [SUCCESS] core-1-2 [2] 11:14:55 [SUCCESS] master-1-1 [3] 11:14:55 [SUCCESS] core-1-1 [1] 11:14:56 [SUCCESS] core-1-1 [2] 11:14:56 [SUCCESS] core-1-2 [3] 11:14:58 [SUCCESS] master-1-1 [1] 11:14:58 [SUCCESS] core-1-2 [2] 11:14:58 [SUCCESS] core-1-1 [3] 11:15:00 [SUCCESS] master-1-1 [1] 11:15:01 [SUCCESS] master-1-1 [2] 11:15:01 [SUCCESS] core-1-2 [3] 11:15:02 [SUCCESS] core-1-1 [1] 11:15:02 [SUCCESS] core-1-2 [2] 11:15:02 [SUCCESS] core-1-1 [3] 11:15:02 [SUCCESS] master-1-1 Successfully upgrade JindoCache Service to 6.7.3 with platform linux, Please restart JindoCache Service 5\u3001\u786e\u8ba4\u5347\u7ea7\u6210\u529f \u00a7 [root@master-1-1]# jindocache -version Version: 6.7.3 CommitId: ###### 6\u3001\u5347\u7ea7\u540e\u542f\u52a8 JindoCache \u670d\u52a1 \u00a7 \u5728EMR\u96c6\u7fa4\u7684 JindoCache \u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u542f\u52a8 \u3002 \u542f\u52a8\u5b8c\u6210\u540e\uff0c\u786e\u8ba4\u96c6\u7fa4\u670d\u52a1\u6b63\u5e38 [root@master-1-1]# jindocache -report Namespace Address: master-1-1:8101 Rpc Port: 8101 Started: Thu Jun 6 11:06:05 2024 Version: 6.7.3 Live Nodes: 2 Decommission Nodes: 0 Total Disk Capacity: 625.519GB Used Disk Capacity: 0B Total MEM Capacity: 0B Used MEM Capacity: 0B \u573a\u666f\u4e8c\uff1a\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4 \u00a7 \u82e5\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 JindoCache, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 JindoCache \u5347\u7ea7\u3002 1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u4e0b\u8f7d\u7684 jindocache-patches.tar.gz, jindocache-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindocache.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 jindocache \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindocache-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindocache-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindocache.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindocache.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindocache-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindocache-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindocache.sh -gen $NEW_JINDOCACHE_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 jindocache \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindocache.sh -gen 6.7.3 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindocache-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindocache-bootstrap-patches.tar.gz \u3002 2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ hadoop dfs -put jindocache-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindocache.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindocache.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindocache-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindocache-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindocache.sh \u3002 3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u00a7 \u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOCACHE\u3002 update_jindocache \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindocache.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindocache-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002 \u573a\u666f\u4e09\uff1a\u65b0\u5efa\u96c6\u7fa4 \u00a7 \u82e5\u65b0\u5efa EMR \u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 jindocache, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 jindocache \u5347\u7ea7\u3002 1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u4e0b\u8f7d\u7684 jindocache-patches.tar.gz \uff0cjindocache-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindocache.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 jindocache \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindocache-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindocache-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindocache.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindocache.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindocache-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindocache-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindocache.sh -gen $NEW_JINDOCACHE_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 Jindocache \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindocache.sh -gen 6.7.3 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindocache-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindocache-bootstrap-patches.tar.gz \u3002 2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ hadoop dfs -put jindocache-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindocache.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindocache.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindocache-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindocache-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindocache.sh \u3002 3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u00a7 \u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOCACHE\u3002 update_jindocache \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindocache.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindocache-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0ax86\u7248\u672c\uff08\u65b0\u7248\u63a7\u5236\u53f0\uff09"},{"location":"upgrade/emr2_upgrade_jindocache/#emr-jindocache","text":"","title":"EMR \u65b0\u7248\u96c6\u7fa4 JindoCache \u5347\u7ea7\u6587\u6863"},{"location":"upgrade/emr2_upgrade_jindocache/#_1","text":"\u82e5\u5df2\u6709\u65b0\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u3002\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u6216\u8005\u9700\u8981\u4f7f\u7528 JindoCache \u7684\u65b0\u529f\u80fd, \u53ef\u4ee5\u6839\u636e\u4e0b\u9762\u7684\u6b65\u9aa4\u5b8c\u6210 JindoCache \u5347\u7ea7\u3002","title":"\u573a\u666f\u4e00\uff1a\u5347\u7ea7\u5df2\u6709\u65b0\u7248\u96c6\u7fa4"},{"location":"upgrade/emr2_upgrade_jindocache/#1-jindocache","text":"\u5728EMR\u96c6\u7fa4\u7684 JindoCache \u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u505c\u6b62 \u3002","title":"1\u3001\u505c\u6b62\u5df2\u6709\u7684 JindoCache \u96c6\u7fa4"},{"location":"upgrade/emr2_upgrade_jindocache/#2","text":"\u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728emr-user\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528emr-user\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - emr-user cd /home/emr-user/ wget http://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindocache-patches.tar.gz tar zxf jindocache-patches.tar.gz \u4e0b\u8f7d jindocache \u8f6f\u4ef6\u5305 jindocache-{VERSION}-{PLATFORM}.tar.gz\uff0c\u89e3\u538b\u540e\u5230 jindocache-patches\u76ee\u5f55\u4e0b\u3002 \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 jindocache \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86 \u5e73\u53f0\u4e3a\u4f8b: cd jindocache-patches wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindocache-6.7.3-linux.tar.gz tar -zxvf jindocache-6.7.3-linux.tar.gz ls -l jindocache-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-rw-r-- 1 emr-user emr-user 29 6\u6708 6 11:21 hosts drwxr-xr-x 7 emr-user emr-user 4096 8\u6708 8 16:19 jindocache-6.7.3-linux -rw-r--r-- 1 emr-user emr-user 276406433 8\u6708 12 14:32 jindocache-6.7.3-linux.tar.gz -rwxr-xr-x 1 emr-user emr-user 1241 6\u6708 6 11:18 upgrade.sh","title":"2\u3001\u51c6\u5907\u8f6f\u4ef6\u5305\u548c\u5347\u7ea7\u811a\u672c\u3001\u914d\u7f6e"},{"location":"upgrade/emr2_upgrade_jindocache/#3","text":"\u7f16\u8f91 patch \u5305\u4e0b\u7684 hosts \u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684 hostname\uff0c\u5982 master-1-1 \u6216 core-1-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindocache-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a master-1-1 core-1-1 core-1-2 \u53ef\u5c1d\u8bd5\u4f7f\u7528\u811a\u672c\u83b7\u53d6\u5168\u90e8\u8282\u70b9\u4fe1\u606f\uff0c\u5982\u679c hosts \u83b7\u53d6\u5931\u8d25\uff0c\u9700\u8981\u624b\u52a8\u8865\u5168 cat /usr/local/taihao-executor-all/data/cache/.cluster_context | jq --raw-output '.nodes[].hostname.alias[]' > hosts","title":"3\u3001\u914d\u7f6e\u5347\u7ea7\u8282\u70b9\u4fe1\u606f"},{"location":"upgrade/emr2_upgrade_jindocache/#4","text":"\u901a\u8fc7 upgrade.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 bash upgrade.sh $NEW_jindocache_VERSION $PLATFORM \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 jindocache \u5347\u7ea7\u5230 linux \u5e73\u53f0 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash upgrade.sh 6.7.3 linux \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 Start upgrade JindoCache Service to 6.7.3 with platform linux [1] 11:14:25 [SUCCESS] core-1-2 [2] 11:14:26 [SUCCESS] core-1-1 [3] 11:14:27 [SUCCESS] master-1-1 [1] 11:14:52 [SUCCESS] core-1-2 [2] 11:14:52 [SUCCESS] master-1-1 [3] 11:14:52 [SUCCESS] core-1-1 [1] 11:14:55 [SUCCESS] core-1-2 [2] 11:14:55 [SUCCESS] master-1-1 [3] 11:14:55 [SUCCESS] core-1-1 [1] 11:14:56 [SUCCESS] core-1-1 [2] 11:14:56 [SUCCESS] core-1-2 [3] 11:14:58 [SUCCESS] master-1-1 [1] 11:14:58 [SUCCESS] core-1-2 [2] 11:14:58 [SUCCESS] core-1-1 [3] 11:15:00 [SUCCESS] master-1-1 [1] 11:15:01 [SUCCESS] master-1-1 [2] 11:15:01 [SUCCESS] core-1-2 [3] 11:15:02 [SUCCESS] core-1-1 [1] 11:15:02 [SUCCESS] core-1-2 [2] 11:15:02 [SUCCESS] core-1-1 [3] 11:15:02 [SUCCESS] master-1-1 Successfully upgrade JindoCache Service to 6.7.3 with platform linux, Please restart JindoCache Service","title":"4\u3001\u6267\u884c\u5347\u7ea7"},{"location":"upgrade/emr2_upgrade_jindocache/#5","text":"[root@master-1-1]# jindocache -version Version: 6.7.3 CommitId: ######","title":"5\u3001\u786e\u8ba4\u5347\u7ea7\u6210\u529f"},{"location":"upgrade/emr2_upgrade_jindocache/#6-jindocache","text":"\u5728EMR\u96c6\u7fa4\u7684 JindoCache \u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u542f\u52a8 \u3002 \u542f\u52a8\u5b8c\u6210\u540e\uff0c\u786e\u8ba4\u96c6\u7fa4\u670d\u52a1\u6b63\u5e38 [root@master-1-1]# jindocache -report Namespace Address: master-1-1:8101 Rpc Port: 8101 Started: Thu Jun 6 11:06:05 2024 Version: 6.7.3 Live Nodes: 2 Decommission Nodes: 0 Total Disk Capacity: 625.519GB Used Disk Capacity: 0B Total MEM Capacity: 0B Used MEM Capacity: 0B","title":"6\u3001\u5347\u7ea7\u540e\u542f\u52a8 JindoCache \u670d\u52a1"},{"location":"upgrade/emr2_upgrade_jindocache/#_2","text":"\u82e5\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 JindoCache, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 JindoCache \u5347\u7ea7\u3002","title":"\u573a\u666f\u4e8c\uff1a\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4"},{"location":"upgrade/emr2_upgrade_jindocache/#1","text":"\u4e0b\u8f7d\u7684 jindocache-patches.tar.gz, jindocache-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindocache.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 jindocache \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindocache-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindocache-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindocache.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindocache.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindocache-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindocache-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindocache.sh -gen $NEW_JINDOCACHE_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 jindocache \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindocache.sh -gen 6.7.3 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindocache-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindocache-bootstrap-patches.tar.gz \u3002","title":"1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindocache/#2_1","text":"\u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ hadoop dfs -put jindocache-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindocache.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindocache.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindocache-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindocache-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindocache.sh \u3002","title":"2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindocache/#3-emr","text":"\u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOCACHE\u3002 update_jindocache \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindocache.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindocache-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c"},{"location":"upgrade/emr2_upgrade_jindocache/#_3","text":"\u82e5\u65b0\u5efa EMR \u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 jindocache, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 jindocache \u5347\u7ea7\u3002","title":"\u573a\u666f\u4e09\uff1a\u65b0\u5efa\u96c6\u7fa4"},{"location":"upgrade/emr2_upgrade_jindocache/#1_1","text":"\u4e0b\u8f7d\u7684 jindocache-patches.tar.gz \uff0cjindocache-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindocache.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 jindocache \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindocache-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindocache-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindocache.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindocache.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindocache-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindocache-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindocache.sh -gen $NEW_JINDOCACHE_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 Jindocache \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindocache.sh -gen 6.7.3 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindocache-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindocache-bootstrap-patches.tar.gz \u3002","title":"1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindocache/#2_2","text":"\u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ hadoop dfs -put jindocache-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindocache.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindocache.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindocache-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindocache-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindocache.sh \u3002","title":"2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindocache/#3-emr_1","text":"\u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOCACHE\u3002 update_jindocache \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindocache.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindocache-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c"},{"location":"upgrade/emr2_upgrade_jindosdk/","text":"EMR \u65b0\u7248\u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6587\u6863 \u00a7 \u573a\u666f\u4e00\uff1a\u5347\u7ea7\u5df2\u6709\u65b0\u7248\u96c6\u7fa4 \u00a7 \u82e5\u5df2\u6709\u65b0\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u3002\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u6216\u8005\u9700\u8981\u4f7f\u7528 JindoSDK \u7684\u65b0\u529f\u80fd\uff0c\u5177\u4f53\u67e5\u770b \u7248\u672c\u8bf4\u660e , \u53ef\u4ee5\u6839\u636e\u4e0b\u9762\u7684\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002 1. \u51c6\u5907\u8f6f\u4ef6\u5305\u548c\u5347\u7ea7\u811a\u672c\u3001\u914d\u7f6e \u00a7 \u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728emr-user\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528emr-user\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - emr-user cd /home/emr-user/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz \u4e0b\u8f7d JindoSDK \u8f6f\u4ef6\u5305 jindosdk-{VERSION}-{PLATFORM}.tar.gz\uff0c\u653e\u5728\u89e3\u538b\u540e\u7684\u76ee\u5f55\u3002 \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86 \u5e73\u53f0\u4e3a\u4f8b: cd jindosdk-patches wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 emr-user emr-user 2439 May 01 00:00 apply_all.sh -rwxrwxr-x 1 emr-user emr-user 7315 May 01 00:00 apply.sh -rw-rw-r-- 1 emr-user emr-user 40 May 01 00:00 hosts -rw-r----- 1 emr-user emr-user xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rwxrwxr-x 1 emr-user emr-user 1112 May 01 00:00 revert_all.sh -rwxrwxr-x 1 emr-user emr-user 2042 May 01 00:00 revert.sh \u6ce8\u610f \uff1a\u5982\u679c\u4ece4.6.8\u4ee5\u4e0b\u7248\u672c\u5347\u7ea7\u52304.6.9\u4ee5\u4e0a\u62166.x\u7248\u672c\u65f6\uff0c\u7531\u4e8e JindoCommitter \u9ed8\u8ba4\u4f7f\u7528\u7684\u4f5c\u4e1a\u4e34\u65f6\u8def\u5f84\u53d1\u751f\u53d8\u5316\uff0c\u9700\u8981\u5728\u5347\u7ea7\u524d \u5148\u8bbe\u7f6e fs.jdo.committer.allow.concurrent=false \uff08core-site.xml\uff09 \u6216\u5728 Spark \u914d\u7f6e\u4e2d\u8bbe\u7f6e spark.hadoop.fs.jdo.committer.allow.concurrent=false \uff0c\u786e\u4fdd\u5347\u7ea7\u671f\u95f4\u4e0d\u4f1a\u51fa\u73b0\u4e22\u6570\u636e\u7684\u60c5\u51b5\u3002 \u540e\u7eed\u5728\u5305\u542bGATEWAY\u8282\u70b9\u7684\u6240\u6709JindoSDK\u5168\u90e8\u5347\u7ea7\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u62e9\u673a\u53bb\u6389\u8be5\u914d\u7f6e\u3002 2. \u914d\u7f6e\u5347\u7ea7\u8282\u70b9\u4fe1\u606f \u00a7 \u7f16\u8f91 patch \u5305\u4e0b\u7684 hosts \u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684 hostname\uff0c\u5982 master-1-1 \u6216 core-1-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a master-1-1 core-1-1 core-1-2 \u53ef\u5c1d\u8bd5\u4f7f\u7528\u811a\u672c\u83b7\u53d6\u5168\u90e8\u8282\u70b9\u4fe1\u606f\uff0c\u5982\u679c hosts \u83b7\u53d6\u5931\u8d25\uff0c\u9700\u8981\u624b\u52a8\u8865\u5168 cat /usr/local/taihao-executor-all/data/cache/.cluster_context | jq --raw-output '.nodes[].hostname.alias[]' > hosts 3. \u6267\u884c\u5347\u7ea7 \u00a7 \u901a\u8fc7 apply_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./apply_all.sh $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: ./apply_all.sh 6.7.3 \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... master-1-1 >>> updating ... core-1-1 >>> updating ... core-1-2 ### DONE 4. \u4fee\u6539\u96c6\u7fa4\u914d\u7f6e \u00a7 4.1 \u517c\u5bb9\u8001\u7248\u672c EMR Ranger \u7ec4\u4ef6 \u00a7 \u6ce8\u610f \uff1a\u5982\u679c\u5f00\u542f\u4e86 EMR Ranger\uff0c\u4e14\u4ece EMR-3.51.1/EMR-5.17.1 \u53ca\u4e4b\u524d\u7248\u672c\u5347\u7ea7 JindoSDK \u5230 6.5.0 \uff5e 6.7.2 \u65f6\u6709\u517c\u5bb9\u95ee\u9898\uff0c\u5efa\u8bae\u5347\u7ea7\u5230 6.7.3 \u53ca\u4ee5\u4e0a\u7248\u672c, \u5e76\u6309\u4ee5\u4e0b\u65b9\u5f0f\u4fee\u6539\u96c6\u7fa4\u914d\u7f6e\u3002 a. \u5728 HADOOP-COMMON \u670d\u52a1\u7684 \u914d\u7f6e \u9875\u9762\uff0c\u5355\u51fb core-sites.xml \u9875\u7b7e\u3002 b. \u5728 core-sites.xml \u9875\u9762\uff0c\u641c\u7d22 \u914d\u7f6e\u9879\u540d\u79f0 \u3002 c. \u4fee\u6539\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 fs.jdo.plugin.dir \u5c06\u52a0\u8f7d plugin \u76ee\u5f55\uff0c\u5c06 /opt/apps/RANGER/jindoauth-current/plugins \u4fee\u6539\u4e3a /opt/apps/JINDOSDK/jindosdk-current/plugins \u3002 5. \u786e\u8ba4\u5347\u7ea7\u6210\u529f \u00a7 ls -l /opt/apps/JINDOSDK/jindosdk-current/lib \u4ee5\u4ece\u96c6\u7fa4\u9ed8\u8ba4\u7248\u672c 6.2.0 \u5347\u7ea7\u4e3a 6.7.3 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a lrwxrwxrwx 1 root root 64 Apr 12 11:08 jindo-core-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-core-6.7.3.jar lrwxrwxrwx 1 root root 82 Apr 12 11:08 jindo-core-linux-el7-aarch64-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-core-linux-el7-aarch64-6.7.3.jar lrwxrwxrwx 1 root root 63 Apr 12 11:08 jindo-sdk-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-sdk-6.7.3.jar lrwxrwxrwx 1 root root 50 Apr 12 11:08 native -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/native lrwxrwxrwx 1 root root 57 Apr 12 11:08 site-packages -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/site-packages 6. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1 \u00a7 \u8bf4\u660e: \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u91cd\u542f \u3002 \u573a\u666f\u4e8c\uff1a\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4 \u00a7 \u82e5\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 JindoSDK, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002 1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindosdk.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindosdk.sh -gen 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002 2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002 3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u00a7 \u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002 4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d \u00a7 \u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542f Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u573a\u666f\u4e09\uff1a\u65b0\u5efa\u96c6\u7fa4 \u00a7 \u82e5\u65b0\u5efa EMR \u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 JindoSDK, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002 1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindosdk.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen-full $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindosdk.sh -gen-full 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002 2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002 3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u00a7 \u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002 4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d \u00a7 \u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542f Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u573a\u666f\u56db\uff1a\u56de\u6eda JindoSDK \u5230\u96c6\u7fa4\u9ed8\u8ba4\u7248\u672c \u00a7 \u82e5\u5df2\u6709\u65b0\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u3002\u5728\u5347\u7ea7\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u9700\u8981\u56de\u6eda\u5230\u96c6\u7fa4\u9ed8\u8ba4\u7684JindoSDK\u7248\u672c 1. \u51c6\u5907\u56de\u6eda\u811a\u672c \u00a7 \u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728emr-user\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528emr-user\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - emr-user cd /home/emr-user/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz cd jindosdk-patches ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 emr-user emr-user 2439 May 01 00:00 apply_all.sh -rwxrwxr-x 1 emr-user emr-user 7315 May 01 00:00 apply.sh -rw-rw-r-- 1 emr-user emr-user 40 May 01 00:00 hosts -rwxrwxr-x 1 emr-user emr-user 1112 May 01 00:00 revert_all.sh -rwxrwxr-x 1 emr-user emr-user 2042 May 01 00:00 revert.sh 2. \u914d\u7f6e\u56de\u6eda\u8282\u70b9\u4fe1\u606f \u00a7 \u7f16\u8f91 patch \u5305\u4e0b\u7684 hosts \u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684 hostname\uff0c\u5982 master-1-1 \u6216 core-1-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a master-1-1 core-1-1 core-1-2 \u53ef\u5c1d\u8bd5\u4f7f\u7528\u811a\u672c\u83b7\u53d6\u5168\u90e8\u8282\u70b9\u4fe1\u606f\uff0c\u5982\u679c hosts \u83b7\u53d6\u5931\u8d25\uff0c\u9700\u8981\u624b\u52a8\u8865\u5168 cat /usr/local/taihao-executor-all/data/cache/.cluster_context | jq --raw-output '.nodes[].hostname.alias[]' > hosts 3. \u6267\u884c\u56de\u6eda \u00a7 \u901a\u8fc7 revert_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./revert_all.sh \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... master-1-1 >>> updating ... core-1-1 >>> updating ... core-1-2 ### DONE 4. \u786e\u8ba4\u56de\u6eda\u6210\u529f \u00a7 ls -l /opt/apps/JINDOSDK/jindosdk-current/lib \u4ee5\u56de\u6eda\u4e3a 6.2.0 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r--r-- 1 root root 1253740 Apr 24 17:40 jindo-core-6.2.0.jar -rw-r--r-- 1 root root 13110547 Apr 24 17:40 jindo-core-linux-el7-aarch64-6.2.0.jar -rw-r--r-- 1 root root 4432227 Apr 24 17:40 jindo-sdk-6.2.0.jar drwxr-xr-x 2 root root 4096 Apr 24 17:40 native 5. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1 \u00a7 \u8bf4\u660e: \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u91cd\u542f \u3002","title":"EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0ax86\u7248\u672c\uff08\u65b0\u7248\u63a7\u5236\u53f0\uff09"},{"location":"upgrade/emr2_upgrade_jindosdk/#emr-jindosdk","text":"","title":"EMR \u65b0\u7248\u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6587\u6863"},{"location":"upgrade/emr2_upgrade_jindosdk/#_1","text":"\u82e5\u5df2\u6709\u65b0\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u3002\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u6216\u8005\u9700\u8981\u4f7f\u7528 JindoSDK \u7684\u65b0\u529f\u80fd\uff0c\u5177\u4f53\u67e5\u770b \u7248\u672c\u8bf4\u660e , \u53ef\u4ee5\u6839\u636e\u4e0b\u9762\u7684\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002","title":"\u573a\u666f\u4e00\uff1a\u5347\u7ea7\u5df2\u6709\u65b0\u7248\u96c6\u7fa4"},{"location":"upgrade/emr2_upgrade_jindosdk/#1","text":"\u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728emr-user\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528emr-user\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - emr-user cd /home/emr-user/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz \u4e0b\u8f7d JindoSDK \u8f6f\u4ef6\u5305 jindosdk-{VERSION}-{PLATFORM}.tar.gz\uff0c\u653e\u5728\u89e3\u538b\u540e\u7684\u76ee\u5f55\u3002 \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86 \u5e73\u53f0\u4e3a\u4f8b: cd jindosdk-patches wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 emr-user emr-user 2439 May 01 00:00 apply_all.sh -rwxrwxr-x 1 emr-user emr-user 7315 May 01 00:00 apply.sh -rw-rw-r-- 1 emr-user emr-user 40 May 01 00:00 hosts -rw-r----- 1 emr-user emr-user xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rwxrwxr-x 1 emr-user emr-user 1112 May 01 00:00 revert_all.sh -rwxrwxr-x 1 emr-user emr-user 2042 May 01 00:00 revert.sh \u6ce8\u610f \uff1a\u5982\u679c\u4ece4.6.8\u4ee5\u4e0b\u7248\u672c\u5347\u7ea7\u52304.6.9\u4ee5\u4e0a\u62166.x\u7248\u672c\u65f6\uff0c\u7531\u4e8e JindoCommitter \u9ed8\u8ba4\u4f7f\u7528\u7684\u4f5c\u4e1a\u4e34\u65f6\u8def\u5f84\u53d1\u751f\u53d8\u5316\uff0c\u9700\u8981\u5728\u5347\u7ea7\u524d \u5148\u8bbe\u7f6e fs.jdo.committer.allow.concurrent=false \uff08core-site.xml\uff09 \u6216\u5728 Spark \u914d\u7f6e\u4e2d\u8bbe\u7f6e spark.hadoop.fs.jdo.committer.allow.concurrent=false \uff0c\u786e\u4fdd\u5347\u7ea7\u671f\u95f4\u4e0d\u4f1a\u51fa\u73b0\u4e22\u6570\u636e\u7684\u60c5\u51b5\u3002 \u540e\u7eed\u5728\u5305\u542bGATEWAY\u8282\u70b9\u7684\u6240\u6709JindoSDK\u5168\u90e8\u5347\u7ea7\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u62e9\u673a\u53bb\u6389\u8be5\u914d\u7f6e\u3002","title":"1. \u51c6\u5907\u8f6f\u4ef6\u5305\u548c\u5347\u7ea7\u811a\u672c\u3001\u914d\u7f6e"},{"location":"upgrade/emr2_upgrade_jindosdk/#2","text":"\u7f16\u8f91 patch \u5305\u4e0b\u7684 hosts \u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684 hostname\uff0c\u5982 master-1-1 \u6216 core-1-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a master-1-1 core-1-1 core-1-2 \u53ef\u5c1d\u8bd5\u4f7f\u7528\u811a\u672c\u83b7\u53d6\u5168\u90e8\u8282\u70b9\u4fe1\u606f\uff0c\u5982\u679c hosts \u83b7\u53d6\u5931\u8d25\uff0c\u9700\u8981\u624b\u52a8\u8865\u5168 cat /usr/local/taihao-executor-all/data/cache/.cluster_context | jq --raw-output '.nodes[].hostname.alias[]' > hosts","title":"2. \u914d\u7f6e\u5347\u7ea7\u8282\u70b9\u4fe1\u606f"},{"location":"upgrade/emr2_upgrade_jindosdk/#3","text":"\u901a\u8fc7 apply_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./apply_all.sh $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: ./apply_all.sh 6.7.3 \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... master-1-1 >>> updating ... core-1-1 >>> updating ... core-1-2 ### DONE","title":"3. \u6267\u884c\u5347\u7ea7"},{"location":"upgrade/emr2_upgrade_jindosdk/#4","text":"","title":"4. \u4fee\u6539\u96c6\u7fa4\u914d\u7f6e"},{"location":"upgrade/emr2_upgrade_jindosdk/#41-emr-ranger","text":"\u6ce8\u610f \uff1a\u5982\u679c\u5f00\u542f\u4e86 EMR Ranger\uff0c\u4e14\u4ece EMR-3.51.1/EMR-5.17.1 \u53ca\u4e4b\u524d\u7248\u672c\u5347\u7ea7 JindoSDK \u5230 6.5.0 \uff5e 6.7.2 \u65f6\u6709\u517c\u5bb9\u95ee\u9898\uff0c\u5efa\u8bae\u5347\u7ea7\u5230 6.7.3 \u53ca\u4ee5\u4e0a\u7248\u672c, \u5e76\u6309\u4ee5\u4e0b\u65b9\u5f0f\u4fee\u6539\u96c6\u7fa4\u914d\u7f6e\u3002 a. \u5728 HADOOP-COMMON \u670d\u52a1\u7684 \u914d\u7f6e \u9875\u9762\uff0c\u5355\u51fb core-sites.xml \u9875\u7b7e\u3002 b. \u5728 core-sites.xml \u9875\u9762\uff0c\u641c\u7d22 \u914d\u7f6e\u9879\u540d\u79f0 \u3002 c. \u4fee\u6539\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 fs.jdo.plugin.dir \u5c06\u52a0\u8f7d plugin \u76ee\u5f55\uff0c\u5c06 /opt/apps/RANGER/jindoauth-current/plugins \u4fee\u6539\u4e3a /opt/apps/JINDOSDK/jindosdk-current/plugins \u3002","title":"4.1 \u517c\u5bb9\u8001\u7248\u672c EMR Ranger \u7ec4\u4ef6"},{"location":"upgrade/emr2_upgrade_jindosdk/#5","text":"ls -l /opt/apps/JINDOSDK/jindosdk-current/lib \u4ee5\u4ece\u96c6\u7fa4\u9ed8\u8ba4\u7248\u672c 6.2.0 \u5347\u7ea7\u4e3a 6.7.3 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a lrwxrwxrwx 1 root root 64 Apr 12 11:08 jindo-core-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-core-6.7.3.jar lrwxrwxrwx 1 root root 82 Apr 12 11:08 jindo-core-linux-el7-aarch64-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-core-linux-el7-aarch64-6.7.3.jar lrwxrwxrwx 1 root root 63 Apr 12 11:08 jindo-sdk-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-sdk-6.7.3.jar lrwxrwxrwx 1 root root 50 Apr 12 11:08 native -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/native lrwxrwxrwx 1 root root 57 Apr 12 11:08 site-packages -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/site-packages","title":"5. \u786e\u8ba4\u5347\u7ea7\u6210\u529f"},{"location":"upgrade/emr2_upgrade_jindosdk/#6","text":"\u8bf4\u660e: \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u91cd\u542f \u3002","title":"6. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1"},{"location":"upgrade/emr2_upgrade_jindosdk/#_2","text":"\u82e5\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 JindoSDK, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002","title":"\u573a\u666f\u4e8c\uff1a\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4"},{"location":"upgrade/emr2_upgrade_jindosdk/#1_1","text":"\u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindosdk.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindosdk.sh -gen 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002","title":"1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindosdk/#2_1","text":"\u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002","title":"2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindosdk/#3-emr","text":"\u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c"},{"location":"upgrade/emr2_upgrade_jindosdk/#4_1","text":"\u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542f Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002","title":"4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d"},{"location":"upgrade/emr2_upgrade_jindosdk/#_3","text":"\u82e5\u65b0\u5efa EMR \u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 JindoSDK, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002","title":"\u573a\u666f\u4e09\uff1a\u65b0\u5efa\u96c6\u7fa4"},{"location":"upgrade/emr2_upgrade_jindosdk/#1_2","text":"\u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindosdk.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen-full $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindosdk.sh -gen-full 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002","title":"1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindosdk/#2_2","text":"\u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002","title":"2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindosdk/#3-emr_1","text":"\u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c"},{"location":"upgrade/emr2_upgrade_jindosdk/#4_2","text":"\u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542f Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002","title":"4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d"},{"location":"upgrade/emr2_upgrade_jindosdk/#jindosdk","text":"\u82e5\u5df2\u6709\u65b0\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u3002\u5728\u5347\u7ea7\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u9700\u8981\u56de\u6eda\u5230\u96c6\u7fa4\u9ed8\u8ba4\u7684JindoSDK\u7248\u672c","title":"\u573a\u666f\u56db\uff1a\u56de\u6eda JindoSDK \u5230\u96c6\u7fa4\u9ed8\u8ba4\u7248\u672c"},{"location":"upgrade/emr2_upgrade_jindosdk/#1_3","text":"\u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728emr-user\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528emr-user\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - emr-user cd /home/emr-user/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz cd jindosdk-patches ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 emr-user emr-user 2439 May 01 00:00 apply_all.sh -rwxrwxr-x 1 emr-user emr-user 7315 May 01 00:00 apply.sh -rw-rw-r-- 1 emr-user emr-user 40 May 01 00:00 hosts -rwxrwxr-x 1 emr-user emr-user 1112 May 01 00:00 revert_all.sh -rwxrwxr-x 1 emr-user emr-user 2042 May 01 00:00 revert.sh","title":"1. \u51c6\u5907\u56de\u6eda\u811a\u672c"},{"location":"upgrade/emr2_upgrade_jindosdk/#2_3","text":"\u7f16\u8f91 patch \u5305\u4e0b\u7684 hosts \u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684 hostname\uff0c\u5982 master-1-1 \u6216 core-1-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a master-1-1 core-1-1 core-1-2 \u53ef\u5c1d\u8bd5\u4f7f\u7528\u811a\u672c\u83b7\u53d6\u5168\u90e8\u8282\u70b9\u4fe1\u606f\uff0c\u5982\u679c hosts \u83b7\u53d6\u5931\u8d25\uff0c\u9700\u8981\u624b\u52a8\u8865\u5168 cat /usr/local/taihao-executor-all/data/cache/.cluster_context | jq --raw-output '.nodes[].hostname.alias[]' > hosts","title":"2. \u914d\u7f6e\u56de\u6eda\u8282\u70b9\u4fe1\u606f"},{"location":"upgrade/emr2_upgrade_jindosdk/#3_1","text":"\u901a\u8fc7 revert_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./revert_all.sh \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... master-1-1 >>> updating ... core-1-1 >>> updating ... core-1-2 ### DONE","title":"3. \u6267\u884c\u56de\u6eda"},{"location":"upgrade/emr2_upgrade_jindosdk/#4_3","text":"ls -l /opt/apps/JINDOSDK/jindosdk-current/lib \u4ee5\u56de\u6eda\u4e3a 6.2.0 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r--r-- 1 root root 1253740 Apr 24 17:40 jindo-core-6.2.0.jar -rw-r--r-- 1 root root 13110547 Apr 24 17:40 jindo-core-linux-el7-aarch64-6.2.0.jar -rw-r--r-- 1 root root 4432227 Apr 24 17:40 jindo-sdk-6.2.0.jar drwxr-xr-x 2 root root 4096 Apr 24 17:40 native","title":"4. \u786e\u8ba4\u56de\u6eda\u6210\u529f"},{"location":"upgrade/emr2_upgrade_jindosdk/#5_1","text":"\u8bf4\u660e: \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u91cd\u542f \u3002","title":"5. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/","text":"EMR \u65b0\u7248\u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6587\u6863\uff08arm/\u501a\u5929\u73af\u5883\uff09 \u00a7 \u573a\u666f\u4e00\uff1a\u5347\u7ea7\u5df2\u6709\u65b0\u7248\u96c6\u7fa4 \u00a7 \u82e5\u5df2\u6709\u65b0\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u3002\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u6216\u8005\u9700\u8981\u4f7f\u7528 JindoSDK \u7684\u65b0\u529f\u80fd\uff0c\u5177\u4f53\u67e5\u770b \u7248\u672c\u8bf4\u660e , \u53ef\u4ee5\u6839\u636e\u4e0b\u9762\u7684\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002 1. \u51c6\u5907\u8f6f\u4ef6\u5305\u548c\u5347\u7ea7\u811a\u672c\u3001\u914d\u7f6e \u00a7 \u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728emr-user\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528emr-user\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - emr-user cd /home/emr-user/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz \u4e0b\u8f7d JindoSDK \u8f6f\u4ef6\u5305 jindosdk-{VERSION}-{PLATFORM}.tar.gz\uff0c\u653e\u5728\u89e3\u538b\u540e\u7684\u76ee\u5f55\u3002 \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux arm \u5e73\u53f0\u4e3a\u4f8b: cd jindosdk-patches wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux-el7-aarch64.tar.gz ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 emr-user emr-user 575 May 01 00:00 apply_all.sh -rwxrwxr-x 1 emr-user emr-user 4047 May 01 00:00 apply.sh -rw-rw-r-- 1 emr-user emr-user 40 May 01 00:00 hosts -rw-r----- 1 emr-user emr-user xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux-el7-aarch64.tar.gz \u6ce8\u610f \uff1a\u5982\u679c\u4ece4.6.8\u4ee5\u4e0b\u7248\u672c\u5347\u7ea7\u52304.6.9\u4ee5\u4e0a\u62166.x\u7248\u672c\u65f6\uff0c\u7531\u4e8e JindoCommitter \u9ed8\u8ba4\u4f7f\u7528\u7684\u4f5c\u4e1a\u4e34\u65f6\u8def\u5f84\u53d1\u751f\u53d8\u5316\uff0c\u9700\u8981\u5728\u5347\u7ea7\u524d \u5148\u8bbe\u7f6e fs.jdo.committer.allow.concurrent=false \uff08core-site.xml\uff09 \u6216\u5728 Spark \u914d\u7f6e\u4e2d\u8bbe\u7f6e spark.hadoop.fs.jdo.committer.allow.concurrent=false \uff0c\u786e\u4fdd\u5347\u7ea7\u671f\u95f4\u4e0d\u4f1a\u51fa\u73b0\u4e22\u6570\u636e\u7684\u60c5\u51b5\u3002 \u540e\u7eed\u5728\u5305\u542bGATEWAY\u8282\u70b9\u7684\u6240\u6709JindoSDK\u5168\u90e8\u5347\u7ea7\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u62e9\u673a\u53bb\u6389\u8be5\u914d\u7f6e\u3002 2. \u914d\u7f6e\u5347\u7ea7\u8282\u70b9\u4fe1\u606f \u00a7 \u7f16\u8f91 patch \u5305\u4e0b\u7684 hosts \u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684 hostname\uff0c\u5982 master-1-1 \u6216 core-1-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a master-1-1 core-1-1 core-1-2 \u53ef\u5c1d\u8bd5\u4f7f\u7528\u811a\u672c\u83b7\u53d6\u5168\u90e8\u8282\u70b9\u4fe1\u606f\uff0c\u5982\u679c hosts \u83b7\u53d6\u5931\u8d25\uff0c\u9700\u8981\u624b\u52a8\u8865\u5168 cat /usr/local/taihao-executor-all/data/cache/.cluster_context | jq --raw-output '.nodes[].hostname.alias[]' > hosts 3. \u6267\u884c\u5347\u7ea7 \u00a7 \u901a\u8fc7 apply_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./apply_all.sh $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: ./apply_all.sh 6.7.3 \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... master-1-1 >>> updating ... core-1-1 >>> updating ... core-1-2 ### DONE 4. \u786e\u8ba4\u5347\u7ea7\u6210\u529f \u00a7 ls -l /opt/apps/JINDOSDK/jindosdk-current/lib \u4ee5\u4ece\u96c6\u7fa4\u9ed8\u8ba4\u7248\u672c 6.2.0 \u5347\u7ea7\u4e3a 6.7.3 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a lrwxrwxrwx 1 root root 64 Apr 12 11:08 jindo-core-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-core-6.7.3.jar lrwxrwxrwx 1 root root 82 Apr 12 11:08 jindo-core-linux-el7-aarch64-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-core-linux-el7-aarch64-6.7.3.jar lrwxrwxrwx 1 root root 63 Apr 12 11:08 jindo-sdk-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-sdk-6.7.3.jar lrwxrwxrwx 1 root root 50 Apr 12 11:08 native -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/native lrwxrwxrwx 1 root root 57 Apr 12 11:08 site-packages -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/site-packages 5. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1 \u00a7 \u8bf4\u660e: \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u91cd\u542f \u3002 \u573a\u666f\u4e8c\uff1a\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4 \u00a7 \u82e5\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 JindoSDK, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002 1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindosdk.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux aarch64\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux-el7-aarch64.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux-el7-aarch64.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindosdk.sh -gen 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002 2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002 3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u00a7 \u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002 4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d \u00a7 \u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542f Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u573a\u666f\u4e09\uff1a\u65b0\u5efa\u96c6\u7fa4 \u00a7 \u82e5\u65b0\u5efa EMR \u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 JindoSDK, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002 1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindosdk.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux aarch64\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux-el7-aarch64.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux-el7-aarch64.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen-full $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindosdk.sh -gen-full 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002 2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002 3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u00a7 \u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002 4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d \u00a7 \u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542f Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u573a\u666f\u56db\uff1a\u56de\u6eda JindoSDK \u5230\u96c6\u7fa4\u521d\u59cb\u7248\u672c \u00a7 \u82e5\u5df2\u6709\u65b0\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u3002\u5728\u5347\u7ea7\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u9700\u8981\u56de\u6eda\u5230\u96c6\u7fa4\u521d\u59cbJindoSDk\u7248\u672c 1. \u51c6\u5907\u56de\u6eda\u811a\u672c \u00a7 \u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728emr-user\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528emr-user\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - emr-user cd /home/emr-user/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz cd jindosdk-patches ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 emr-user emr-user 2439 May 01 00:00 apply_all.sh -rwxrwxr-x 1 emr-user emr-user 7315 May 01 00:00 apply.sh -rw-rw-r-- 1 emr-user emr-user 40 May 01 00:00 hosts -rwxrwxr-x 1 emr-user emr-user 1112 May 01 00:00 revert_all.sh -rwxrwxr-x 1 emr-user emr-user 2042 May 01 00:00 revert.sh 2. \u914d\u7f6e\u56de\u6eda\u8282\u70b9\u4fe1\u606f \u00a7 \u7f16\u8f91 patch \u5305\u4e0b\u7684 hosts \u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684 hostname\uff0c\u5982 master-1-1 \u6216 core-1-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a master-1-1 core-1-1 core-1-2 \u53ef\u5c1d\u8bd5\u4f7f\u7528\u811a\u672c\u83b7\u53d6\u5168\u90e8\u8282\u70b9\u4fe1\u606f\uff0c\u5982\u679c hosts \u83b7\u53d6\u5931\u8d25\uff0c\u9700\u8981\u624b\u52a8\u8865\u5168 cat /usr/local/taihao-executor-all/data/cache/.cluster_context | jq --raw-output '.nodes[].hostname.alias[]' > hosts 3. \u6267\u884c\u56de\u6eda \u00a7 \u901a\u8fc7 revert_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./revert_all.sh \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... master-1-1 >>> updating ... core-1-1 >>> updating ... core-1-2 ### DONE 4. \u786e\u8ba4\u56de\u6eda\u6210\u529f \u00a7 ls -l /opt/apps/JINDOSDK/jindosdk-current/lib \u4ee5\u56de\u6eda\u4e3a 6.2.0 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r--r-- 1 root root 1253740 Apr 24 17:40 jindo-core-6.2.0.jar -rw-r--r-- 1 root root 13110547 Apr 24 17:40 jindo-core-linux-el7-aarch64-6.2.0.jar -rw-r--r-- 1 root root 4432227 Apr 24 17:40 jindo-sdk-6.2.0.jar drwxr-xr-x 2 root root 4096 Apr 24 17:40 native 5. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1 \u00a7 \u8bf4\u660e: \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u91cd\u542f \u3002","title":"EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0aarm\u7248\u672c\uff08\u65b0\u7248\u63a7\u5236\u53f0\uff09"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#emr-jindosdk-arm","text":"","title":"EMR \u65b0\u7248\u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6587\u6863\uff08arm/\u501a\u5929\u73af\u5883\uff09"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#_1","text":"\u82e5\u5df2\u6709\u65b0\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u3002\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u6216\u8005\u9700\u8981\u4f7f\u7528 JindoSDK \u7684\u65b0\u529f\u80fd\uff0c\u5177\u4f53\u67e5\u770b \u7248\u672c\u8bf4\u660e , \u53ef\u4ee5\u6839\u636e\u4e0b\u9762\u7684\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002","title":"\u573a\u666f\u4e00\uff1a\u5347\u7ea7\u5df2\u6709\u65b0\u7248\u96c6\u7fa4"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#1","text":"\u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728emr-user\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528emr-user\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - emr-user cd /home/emr-user/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz \u4e0b\u8f7d JindoSDK \u8f6f\u4ef6\u5305 jindosdk-{VERSION}-{PLATFORM}.tar.gz\uff0c\u653e\u5728\u89e3\u538b\u540e\u7684\u76ee\u5f55\u3002 \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux arm \u5e73\u53f0\u4e3a\u4f8b: cd jindosdk-patches wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux-el7-aarch64.tar.gz ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 emr-user emr-user 575 May 01 00:00 apply_all.sh -rwxrwxr-x 1 emr-user emr-user 4047 May 01 00:00 apply.sh -rw-rw-r-- 1 emr-user emr-user 40 May 01 00:00 hosts -rw-r----- 1 emr-user emr-user xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux-el7-aarch64.tar.gz \u6ce8\u610f \uff1a\u5982\u679c\u4ece4.6.8\u4ee5\u4e0b\u7248\u672c\u5347\u7ea7\u52304.6.9\u4ee5\u4e0a\u62166.x\u7248\u672c\u65f6\uff0c\u7531\u4e8e JindoCommitter \u9ed8\u8ba4\u4f7f\u7528\u7684\u4f5c\u4e1a\u4e34\u65f6\u8def\u5f84\u53d1\u751f\u53d8\u5316\uff0c\u9700\u8981\u5728\u5347\u7ea7\u524d \u5148\u8bbe\u7f6e fs.jdo.committer.allow.concurrent=false \uff08core-site.xml\uff09 \u6216\u5728 Spark \u914d\u7f6e\u4e2d\u8bbe\u7f6e spark.hadoop.fs.jdo.committer.allow.concurrent=false \uff0c\u786e\u4fdd\u5347\u7ea7\u671f\u95f4\u4e0d\u4f1a\u51fa\u73b0\u4e22\u6570\u636e\u7684\u60c5\u51b5\u3002 \u540e\u7eed\u5728\u5305\u542bGATEWAY\u8282\u70b9\u7684\u6240\u6709JindoSDK\u5168\u90e8\u5347\u7ea7\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u62e9\u673a\u53bb\u6389\u8be5\u914d\u7f6e\u3002","title":"1. \u51c6\u5907\u8f6f\u4ef6\u5305\u548c\u5347\u7ea7\u811a\u672c\u3001\u914d\u7f6e"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#2","text":"\u7f16\u8f91 patch \u5305\u4e0b\u7684 hosts \u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684 hostname\uff0c\u5982 master-1-1 \u6216 core-1-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a master-1-1 core-1-1 core-1-2 \u53ef\u5c1d\u8bd5\u4f7f\u7528\u811a\u672c\u83b7\u53d6\u5168\u90e8\u8282\u70b9\u4fe1\u606f\uff0c\u5982\u679c hosts \u83b7\u53d6\u5931\u8d25\uff0c\u9700\u8981\u624b\u52a8\u8865\u5168 cat /usr/local/taihao-executor-all/data/cache/.cluster_context | jq --raw-output '.nodes[].hostname.alias[]' > hosts","title":"2. \u914d\u7f6e\u5347\u7ea7\u8282\u70b9\u4fe1\u606f"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#3","text":"\u901a\u8fc7 apply_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./apply_all.sh $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: ./apply_all.sh 6.7.3 \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... master-1-1 >>> updating ... core-1-1 >>> updating ... core-1-2 ### DONE","title":"3. \u6267\u884c\u5347\u7ea7"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#4","text":"ls -l /opt/apps/JINDOSDK/jindosdk-current/lib \u4ee5\u4ece\u96c6\u7fa4\u9ed8\u8ba4\u7248\u672c 6.2.0 \u5347\u7ea7\u4e3a 6.7.3 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a lrwxrwxrwx 1 root root 64 Apr 12 11:08 jindo-core-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-core-6.7.3.jar lrwxrwxrwx 1 root root 82 Apr 12 11:08 jindo-core-linux-el7-aarch64-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-core-linux-el7-aarch64-6.7.3.jar lrwxrwxrwx 1 root root 63 Apr 12 11:08 jindo-sdk-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-sdk-6.7.3.jar lrwxrwxrwx 1 root root 50 Apr 12 11:08 native -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/native lrwxrwxrwx 1 root root 57 Apr 12 11:08 site-packages -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/site-packages","title":"4. \u786e\u8ba4\u5347\u7ea7\u6210\u529f"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#5","text":"\u8bf4\u660e: \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u91cd\u542f \u3002","title":"5. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#_2","text":"\u82e5\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 JindoSDK, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002","title":"\u573a\u666f\u4e8c\uff1a\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#1_1","text":"\u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindosdk.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux aarch64\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux-el7-aarch64.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux-el7-aarch64.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindosdk.sh -gen 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002","title":"1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#2_1","text":"\u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002","title":"2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#3-emr","text":"\u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#4_1","text":"\u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542f Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002","title":"4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#_3","text":"\u82e5\u65b0\u5efa EMR \u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 JindoSDK, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002","title":"\u573a\u666f\u4e09\uff1a\u65b0\u5efa\u96c6\u7fa4"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#1_2","text":"\u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindosdk.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux aarch64\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux-el7-aarch64.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux-el7-aarch64.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen-full $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindosdk.sh -gen-full 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002","title":"1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#2_2","text":"\u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002","title":"2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#3-emr_1","text":"\u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#4_2","text":"\u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542f Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002","title":"4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#jindosdk","text":"\u82e5\u5df2\u6709\u65b0\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u3002\u5728\u5347\u7ea7\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u9700\u8981\u56de\u6eda\u5230\u96c6\u7fa4\u521d\u59cbJindoSDk\u7248\u672c","title":"\u573a\u666f\u56db\uff1a\u56de\u6eda JindoSDK \u5230\u96c6\u7fa4\u521d\u59cb\u7248\u672c"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#1_3","text":"\u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728emr-user\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528emr-user\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - emr-user cd /home/emr-user/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz cd jindosdk-patches ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 emr-user emr-user 2439 May 01 00:00 apply_all.sh -rwxrwxr-x 1 emr-user emr-user 7315 May 01 00:00 apply.sh -rw-rw-r-- 1 emr-user emr-user 40 May 01 00:00 hosts -rwxrwxr-x 1 emr-user emr-user 1112 May 01 00:00 revert_all.sh -rwxrwxr-x 1 emr-user emr-user 2042 May 01 00:00 revert.sh","title":"1. \u51c6\u5907\u56de\u6eda\u811a\u672c"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#2_3","text":"\u7f16\u8f91 patch \u5305\u4e0b\u7684 hosts \u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684 hostname\uff0c\u5982 master-1-1 \u6216 core-1-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a master-1-1 core-1-1 core-1-2 \u53ef\u5c1d\u8bd5\u4f7f\u7528\u811a\u672c\u83b7\u53d6\u5168\u90e8\u8282\u70b9\u4fe1\u606f\uff0c\u5982\u679c hosts \u83b7\u53d6\u5931\u8d25\uff0c\u9700\u8981\u624b\u52a8\u8865\u5168 cat /usr/local/taihao-executor-all/data/cache/.cluster_context | jq --raw-output '.nodes[].hostname.alias[]' > hosts","title":"2. \u914d\u7f6e\u56de\u6eda\u8282\u70b9\u4fe1\u606f"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#3_1","text":"\u901a\u8fc7 revert_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./revert_all.sh \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... master-1-1 >>> updating ... core-1-1 >>> updating ... core-1-2 ### DONE","title":"3. \u6267\u884c\u56de\u6eda"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#4_3","text":"ls -l /opt/apps/JINDOSDK/jindosdk-current/lib \u4ee5\u56de\u6eda\u4e3a 6.2.0 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r--r-- 1 root root 1253740 Apr 24 17:40 jindo-core-6.2.0.jar -rw-r--r-- 1 root root 13110547 Apr 24 17:40 jindo-core-linux-el7-aarch64-6.2.0.jar -rw-r--r-- 1 root root 4432227 Apr 24 17:40 jindo-sdk-6.2.0.jar drwxr-xr-x 2 root root 4096 Apr 24 17:40 native","title":"4. \u786e\u8ba4\u56de\u6eda\u6210\u529f"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#5_1","text":"\u8bf4\u660e: \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u91cd\u542f \u3002","title":"5. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1"},{"location":"upgrade/emr_upgrade_jindosdk/","text":"EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b\uff08\u65e7\u7248\u63a7\u5236\u53f0\uff09 \u00a7 \u80cc\u666f \u00a7 \u5728\u65e7\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\uff0c\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u6216\u8005\u9700\u8981\u4f7f\u7528 JindoSDK \u7684\u65b0\u529f\u80fd\uff0c\u5177\u4f53\u67e5\u770b \u7248\u672c\u8bf4\u660e , \u53ef\u4ee5\u6839\u636e\u4e0b\u9762\u7684\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002 \u573a\u666f\u4e00\uff1a\u5347\u7ea7\u5df2\u6709\u96c6\u7fa4 \u00a7 1. \u51c6\u5907\u8f6f\u4ef6\u5305\u548c\u5347\u7ea7\u811a\u672c \u00a7 \u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728hadoop\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528hadoop\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - hadoop cd /home/hadoop/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz \u4e0b\u8f7d JindoSDK \u8f6f\u4ef6\u5305 jindosdk-{VERSION}-{PLATFORM}.tar.gz\uff08\u4ee5\u4e0b\u4ee5linux x86\u5e73\u53f0\u4e3a\u4f8b\uff09\uff0c\u653e\u5728\u89e3\u538b\u540e\u7684\u76ee\u5f55\u3002 cd jindosdk-patches wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 hadoop hadoop 1263 May 01 00:00 apply_all.sh -rwxrwxr-x 1 hadoop hadoop 6840 May 01 00:00 apply.sh -rw-rw-r-- 1 hadoop hadoop 40 May 01 00:00 hosts -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz \u6ce8\u610f \uff1a\u5982\u679c\u4ece4.6.8\u4ee5\u4e0b\u7248\u672c\u5347\u7ea7\u52304.6.9\u4ee5\u4e0a\u62166.x\u7248\u672c\u65f6\uff0c\u7531\u4e8e JindoCommitter \u9ed8\u8ba4\u4f7f\u7528\u7684\u4f5c\u4e1a\u4e34\u65f6\u8def\u5f84\u53d1\u751f\u53d8\u5316\uff0c\u9700\u8981\u5728\u5347\u7ea7\u524d \u5148\u8bbe\u7f6e fs.jdo.committer.allow.concurrent=false \uff08core-site.xml\uff09 \u6216\u5728 Spark \u914d\u7f6e\u4e2d\u8bbe\u7f6e spark.hadoop.fs.jdo.committer.allow.concurrent=false \uff0c\u786e\u4fdd\u5347\u7ea7\u671f\u95f4\u4e0d\u4f1a\u51fa\u73b0\u4e22\u6570\u636e\u7684\u60c5\u51b5\u3002 \u540e\u7eed\u5728\u5305\u542bGATEWAY\u8282\u70b9\u7684\u6240\u6709JindoSDK\u5168\u90e8\u5347\u7ea7\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u62e9\u673a\u53bb\u6389\u8be5\u914d\u7f6e\u3002 2. \u914d\u7f6e\u5347\u7ea7\u8282\u70b9\u4fe1\u606f \u00a7 \u7f16\u8f91patch\u5305\u4e0b\u7684hosts\u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684host name\uff0c\u5982emr-header-1\u6216emr-worker-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a emr-header-1 emr-worker-1 emr-worker-2 3. \u6267\u884c\u5347\u7ea7 \u00a7 \u901a\u8fc7apply_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./apply_all.sh $JINDOSDK_VERSION \u5982 ./apply_all.sh 6.7.3 \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... emr-header-1 >>> updating ... emr-worker-1 >>> updating ... emr-worker-2 ### DONE 4. \u786e\u8ba4\u5347\u7ea7\u6210\u529f \u00a7 ls -l /opt/apps/extra-jars/ \u4ee5\u5347\u7ea7\u4e3a 6.7.3 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a drwxr-xr-x 2 root root 4096 Apr 24 15:49 flink -rw-r--r-- 1 hadoop hadoop 189081 Apr 24 15:20 hadoop-lzo-0.4.21-SNAPSHOT.jar -rw-r--r-- 1 root root 16264149 Apr 24 15:49 jindo-core-6.7.3.jar -rw-r--r-- 1 root root 14429862 Apr 24 15:49 jindo-core-linux-el7-aarch64-6.7.3.jar -rw-r--r-- 1 root root 4459297 Apr 24 15:49 jindo-sdk-6.7.3.jar drwxr-xr-x 2 root root 4096 Apr 24 15:49 spark drwxr-xr-x 2 root root 4096 Apr 24 15:49 spark3 5. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1 \u00a7 \u8bf4\u660e \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u64cd\u4f5c > \u91cd\u542f All Components \u3002 \u573a\u666f\u4e8c\uff1a\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4 \u00a7 \u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u53ef\u4ee5\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u64cd\u4f5c\u6b65\u9aa4\u5982\u4e0b\uff1a 1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-6.7.3-linux.tar.gz \u548c bootstrap_jindosdk.sh\u3002 mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen $JINDOSDK_VERSION \u5982 bash bootstrap_jindosdk.sh -gen 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/hadoop/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230patch\u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002 2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u5c06 patch\u5305 \u548c bootstrap\u811a\u672c\u4e0a\u4f20\u5230OSS\u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 oss \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/jindo-patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002 3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u00a7 \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002 4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d \u00a7 \u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542fHive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u573a\u666f\u4e09\uff1a\u65b0\u5efa\u96c6\u7fa4 \u00a7 \u65b0\u5efaEMR\u96c6\u7fa4\u65f6\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\u3002\u5177\u4f53\u64cd\u4f5c\u6b65\u9aa4\u5982\u4e0b\uff1a 1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-6.7.3-linux.tar.gz \u548c bootstrap_jindosdk.sh\u3002 mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen-full $JINDOSDK_VERSION \u5982 bash bootstrap_jindosdk.sh -gen-full 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/hadoop/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230patch\u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002 2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u5c06 patch\u5305 \u548c bootstrap\u811a\u672c\u4e0a\u4f20\u5230OSS\u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 oss \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/jindo-patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002 \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u00a7 \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002 \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d \u00a7 \u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542fHive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u573a\u666f\u56db\uff1a\u56de\u6eda JindoSDK \u5230\u96c6\u7fa4\u521d\u59cb\u7248\u672c \u00a7 1. \u51c6\u5907\u56de\u6eda\u811a\u672c \u00a7 \u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728 hadoop \u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528 hadoop \u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - hadoop cd /home/hadoop/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz cd jindosdk-patches ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 hadoop hadoop 1263 May 01 00:00 apply_all.sh -rwxrwxr-x 1 hadoop hadoop 6840 May 01 00:00 apply.sh -rw-rw-r-- 1 hadoop hadoop 40 May 01 00:00 hosts -rwxrwxr-x 1 hadoop hadoop 1308 May 01 00:00 revert_all.sh -rwxrwxr-x 1 hadoop hadoop 6326 May 01 00:00 revert.sh 2. \u914d\u7f6e\u56de\u6eda\u8282\u70b9\u4fe1\u606f \u00a7 \u7f16\u8f91patch\u5305\u4e0b\u7684hosts\u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684host name\uff0c\u5982emr-header-1\u6216emr-worker-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a emr-header-1 emr-worker-1 emr-worker-2 3. \u6267\u884c\u56de\u6eda \u00a7 \u901a\u8fc7revert_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./revert_all.sh \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... emr-header-1 >>> updating ... emr-worker-1 >>> updating ... emr-worker-2 ### DONE 4. \u786e\u8ba4\u56de\u6eda\u6210\u529f \u00a7 ls -l /opt/apps/extra-jars/ \u4ee5\u56de\u6eda\u4e3a 4.3.0 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a drwxr-xr-x 2 root root 4096 Apr 24 17:37 flink -rw-r--r-- 1 hadoop hadoop 189081 Apr 24 17:21 hadoop-lzo-0.4.21-SNAPSHOT.jar drwxr-xr-x 2 root root 4096 Apr 24 17:37 hive -rw-r--r-- 1 root root 85025668 Apr 24 17:36 jindo-core-4.3.0.jar -rw-r--r-- 1 root root 3165000 Apr 24 17:36 jindo-sdk-4.3.0.jar drwxr-xr-x 2 root root 4096 Apr 24 17:37 native drwxr-xr-x 2 root root 4096 Apr 24 17:37 spark drwxr-xr-x 2 root root 4096 Apr 24 17:37 spark3 5. \u56de\u6eda\u540e\u91cd\u542f\u670d\u52a1 \u00a7 \u8bf4\u660e \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u56de\u6eda\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u64cd\u4f5c > \u91cd\u542f All Components \u3002","title":"EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\uff08\u65e7\u7248\u63a7\u5236\u53f0\uff09"},{"location":"upgrade/emr_upgrade_jindosdk/#emr-jindosdk","text":"","title":"EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b\uff08\u65e7\u7248\u63a7\u5236\u53f0\uff09"},{"location":"upgrade/emr_upgrade_jindosdk/#_1","text":"\u5728\u65e7\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\uff0c\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u6216\u8005\u9700\u8981\u4f7f\u7528 JindoSDK \u7684\u65b0\u529f\u80fd\uff0c\u5177\u4f53\u67e5\u770b \u7248\u672c\u8bf4\u660e , \u53ef\u4ee5\u6839\u636e\u4e0b\u9762\u7684\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002","title":"\u80cc\u666f"},{"location":"upgrade/emr_upgrade_jindosdk/#_2","text":"","title":"\u573a\u666f\u4e00\uff1a\u5347\u7ea7\u5df2\u6709\u96c6\u7fa4"},{"location":"upgrade/emr_upgrade_jindosdk/#1","text":"\u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728hadoop\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528hadoop\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - hadoop cd /home/hadoop/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz \u4e0b\u8f7d JindoSDK \u8f6f\u4ef6\u5305 jindosdk-{VERSION}-{PLATFORM}.tar.gz\uff08\u4ee5\u4e0b\u4ee5linux x86\u5e73\u53f0\u4e3a\u4f8b\uff09\uff0c\u653e\u5728\u89e3\u538b\u540e\u7684\u76ee\u5f55\u3002 cd jindosdk-patches wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 hadoop hadoop 1263 May 01 00:00 apply_all.sh -rwxrwxr-x 1 hadoop hadoop 6840 May 01 00:00 apply.sh -rw-rw-r-- 1 hadoop hadoop 40 May 01 00:00 hosts -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz \u6ce8\u610f \uff1a\u5982\u679c\u4ece4.6.8\u4ee5\u4e0b\u7248\u672c\u5347\u7ea7\u52304.6.9\u4ee5\u4e0a\u62166.x\u7248\u672c\u65f6\uff0c\u7531\u4e8e JindoCommitter \u9ed8\u8ba4\u4f7f\u7528\u7684\u4f5c\u4e1a\u4e34\u65f6\u8def\u5f84\u53d1\u751f\u53d8\u5316\uff0c\u9700\u8981\u5728\u5347\u7ea7\u524d \u5148\u8bbe\u7f6e fs.jdo.committer.allow.concurrent=false \uff08core-site.xml\uff09 \u6216\u5728 Spark \u914d\u7f6e\u4e2d\u8bbe\u7f6e spark.hadoop.fs.jdo.committer.allow.concurrent=false \uff0c\u786e\u4fdd\u5347\u7ea7\u671f\u95f4\u4e0d\u4f1a\u51fa\u73b0\u4e22\u6570\u636e\u7684\u60c5\u51b5\u3002 \u540e\u7eed\u5728\u5305\u542bGATEWAY\u8282\u70b9\u7684\u6240\u6709JindoSDK\u5168\u90e8\u5347\u7ea7\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u62e9\u673a\u53bb\u6389\u8be5\u914d\u7f6e\u3002","title":"1. \u51c6\u5907\u8f6f\u4ef6\u5305\u548c\u5347\u7ea7\u811a\u672c"},{"location":"upgrade/emr_upgrade_jindosdk/#2","text":"\u7f16\u8f91patch\u5305\u4e0b\u7684hosts\u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684host name\uff0c\u5982emr-header-1\u6216emr-worker-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a emr-header-1 emr-worker-1 emr-worker-2","title":"2. \u914d\u7f6e\u5347\u7ea7\u8282\u70b9\u4fe1\u606f"},{"location":"upgrade/emr_upgrade_jindosdk/#3","text":"\u901a\u8fc7apply_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./apply_all.sh $JINDOSDK_VERSION \u5982 ./apply_all.sh 6.7.3 \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... emr-header-1 >>> updating ... emr-worker-1 >>> updating ... emr-worker-2 ### DONE","title":"3. \u6267\u884c\u5347\u7ea7"},{"location":"upgrade/emr_upgrade_jindosdk/#4","text":"ls -l /opt/apps/extra-jars/ \u4ee5\u5347\u7ea7\u4e3a 6.7.3 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a drwxr-xr-x 2 root root 4096 Apr 24 15:49 flink -rw-r--r-- 1 hadoop hadoop 189081 Apr 24 15:20 hadoop-lzo-0.4.21-SNAPSHOT.jar -rw-r--r-- 1 root root 16264149 Apr 24 15:49 jindo-core-6.7.3.jar -rw-r--r-- 1 root root 14429862 Apr 24 15:49 jindo-core-linux-el7-aarch64-6.7.3.jar -rw-r--r-- 1 root root 4459297 Apr 24 15:49 jindo-sdk-6.7.3.jar drwxr-xr-x 2 root root 4096 Apr 24 15:49 spark drwxr-xr-x 2 root root 4096 Apr 24 15:49 spark3","title":"4. \u786e\u8ba4\u5347\u7ea7\u6210\u529f"},{"location":"upgrade/emr_upgrade_jindosdk/#5","text":"\u8bf4\u660e \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u64cd\u4f5c > \u91cd\u542f All Components \u3002","title":"5. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1"},{"location":"upgrade/emr_upgrade_jindosdk/#_3","text":"\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u53ef\u4ee5\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u64cd\u4f5c\u6b65\u9aa4\u5982\u4e0b\uff1a","title":"\u573a\u666f\u4e8c\uff1a\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4"},{"location":"upgrade/emr_upgrade_jindosdk/#1_1","text":"\u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-6.7.3-linux.tar.gz \u548c bootstrap_jindosdk.sh\u3002 mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen $JINDOSDK_VERSION \u5982 bash bootstrap_jindosdk.sh -gen 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/hadoop/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230patch\u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002","title":"1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr_upgrade_jindosdk/#2_1","text":"\u5c06 patch\u5305 \u548c bootstrap\u811a\u672c\u4e0a\u4f20\u5230OSS\u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 oss \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/jindo-patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002","title":"2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr_upgrade_jindosdk/#3-emr","text":"\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c"},{"location":"upgrade/emr_upgrade_jindosdk/#4_1","text":"\u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542fHive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002","title":"4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d"},{"location":"upgrade/emr_upgrade_jindosdk/#_4","text":"\u65b0\u5efaEMR\u96c6\u7fa4\u65f6\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\u3002\u5177\u4f53\u64cd\u4f5c\u6b65\u9aa4\u5982\u4e0b\uff1a","title":"\u573a\u666f\u4e09\uff1a\u65b0\u5efa\u96c6\u7fa4"},{"location":"upgrade/emr_upgrade_jindosdk/#1_2","text":"\u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-6.7.3-linux.tar.gz \u548c bootstrap_jindosdk.sh\u3002 mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen-full $JINDOSDK_VERSION \u5982 bash bootstrap_jindosdk.sh -gen-full 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/hadoop/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230patch\u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002","title":"1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr_upgrade_jindosdk/#2_2","text":"\u5c06 patch\u5305 \u548c bootstrap\u811a\u672c\u4e0a\u4f20\u5230OSS\u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 oss \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/jindo-patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002","title":"2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr_upgrade_jindosdk/#emr","text":"\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c"},{"location":"upgrade/emr_upgrade_jindosdk/#_5","text":"\u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542fHive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002","title":"\u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d"},{"location":"upgrade/emr_upgrade_jindosdk/#jindosdk","text":"","title":"\u573a\u666f\u56db\uff1a\u56de\u6eda JindoSDK \u5230\u96c6\u7fa4\u521d\u59cb\u7248\u672c"},{"location":"upgrade/emr_upgrade_jindosdk/#1_3","text":"\u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728 hadoop \u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528 hadoop \u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - hadoop cd /home/hadoop/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz cd jindosdk-patches ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 hadoop hadoop 1263 May 01 00:00 apply_all.sh -rwxrwxr-x 1 hadoop hadoop 6840 May 01 00:00 apply.sh -rw-rw-r-- 1 hadoop hadoop 40 May 01 00:00 hosts -rwxrwxr-x 1 hadoop hadoop 1308 May 01 00:00 revert_all.sh -rwxrwxr-x 1 hadoop hadoop 6326 May 01 00:00 revert.sh","title":"1. \u51c6\u5907\u56de\u6eda\u811a\u672c"},{"location":"upgrade/emr_upgrade_jindosdk/#2_3","text":"\u7f16\u8f91patch\u5305\u4e0b\u7684hosts\u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684host name\uff0c\u5982emr-header-1\u6216emr-worker-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a emr-header-1 emr-worker-1 emr-worker-2","title":"2. \u914d\u7f6e\u56de\u6eda\u8282\u70b9\u4fe1\u606f"},{"location":"upgrade/emr_upgrade_jindosdk/#3_1","text":"\u901a\u8fc7revert_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./revert_all.sh \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... emr-header-1 >>> updating ... emr-worker-1 >>> updating ... emr-worker-2 ### DONE","title":"3. \u6267\u884c\u56de\u6eda"},{"location":"upgrade/emr_upgrade_jindosdk/#4_2","text":"ls -l /opt/apps/extra-jars/ \u4ee5\u56de\u6eda\u4e3a 4.3.0 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a drwxr-xr-x 2 root root 4096 Apr 24 17:37 flink -rw-r--r-- 1 hadoop hadoop 189081 Apr 24 17:21 hadoop-lzo-0.4.21-SNAPSHOT.jar drwxr-xr-x 2 root root 4096 Apr 24 17:37 hive -rw-r--r-- 1 root root 85025668 Apr 24 17:36 jindo-core-4.3.0.jar -rw-r--r-- 1 root root 3165000 Apr 24 17:36 jindo-sdk-4.3.0.jar drwxr-xr-x 2 root root 4096 Apr 24 17:37 native drwxr-xr-x 2 root root 4096 Apr 24 17:37 spark drwxr-xr-x 2 root root 4096 Apr 24 17:37 spark3","title":"4. \u786e\u8ba4\u56de\u6eda\u6210\u529f"},{"location":"upgrade/emr_upgrade_jindosdk/#5_1","text":"\u8bf4\u660e \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u56de\u6eda\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u64cd\u4f5c > \u91cd\u542f All Components \u3002","title":"5. \u56de\u6eda\u540e\u91cd\u542f\u670d\u52a1"},{"location":"upgrade/emr_upgrade_smartdata/","text":"EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b\uff08\u65e7\u7248\u63a7\u5236\u53f0\uff09 \u00a7 \u80cc\u666f \u00a7 \u5728\u65e7\u7248\u7ba1\u63a7\u5e73\u53f0\u4e2d\u521b\u5efaE-MapReduce EMR-5.5.0/EMR-3.39.0 \u4ee5\u524d\u7248\u672c\u7684\u96c6\u7fa4\uff0c\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u6216\u8005\u9700\u8981\u4f7f\u7528 JindoSDK \u7684\u65b0\u529f\u80fd\uff0c\u5177\u4f53\u67e5\u770b \u7248\u672c\u8bf4\u660e , \u53ef\u4ee5\u6839\u636e\u4e0b\u9762\u7684\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002 \u573a\u666f\u4e00\uff1a\u5347\u7ea7\u5df2\u6709\u96c6\u7fa4 \u00a7 1. \u51c6\u5907\u8f6f\u4ef6\u5305\u548c\u5347\u7ea7\u811a\u672c \u00a7 \u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728 hadoop \u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528 hadoop \u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - hadoop cd /home/hadoop/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz \u4e0b\u8f7d JindoSDK \u8f6f\u4ef6\u5305 jindosdk-{VERSION}-{PLATFORM}.tar.gz\uff08\u4ee5\u4e0b\u4ee5linux x86\u5e73\u53f0\u4e3a\u4f8b\uff09 \uff0c\u653e\u5728\u89e3\u538b\u540e\u7684\u76ee\u5f55\u3002 cd jindosdk-patches wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 hadoop hadoop 1263 May 01 00:00 apply_all.sh -rwxrwxr-x 1 hadoop hadoop 6840 May 01 00:00 apply.sh -rw-rw-r-- 1 hadoop hadoop 40 May 01 00:00 hosts -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rwxrwxr-x 1 hadoop hadoop 1308 May 01 00:00 revert_all.sh -rwxrwxr-x 1 hadoop hadoop 6326 May 01 00:00 revert.sh \u6ce8\u610f \uff1a\u5982\u679c\u4ece4.6.8\u4ee5\u4e0b\u7248\u672c\u5347\u7ea7\u52304.6.9\u4ee5\u4e0a\u62166.x\u7248\u672c\u65f6\uff0c\u7531\u4e8e JindoCommitter \u9ed8\u8ba4\u4f7f\u7528\u7684\u4f5c\u4e1a\u4e34\u65f6\u8def\u5f84\u53d1\u751f\u53d8\u5316\uff0c\u9700\u8981\u5728\u5347\u7ea7\u524d \u5148\u8bbe\u7f6e fs.jdo.committer.allow.concurrent=false \uff08core-site.xml\uff09 \u6216\u5728 Spark \u914d\u7f6e\u4e2d\u8bbe\u7f6e spark.hadoop.fs.jdo.committer.allow.concurrent=false \uff0c\u786e\u4fdd\u5347\u7ea7\u671f\u95f4\u4e0d\u4f1a\u51fa\u73b0\u4e22\u6570\u636e\u7684\u60c5\u51b5\u3002 \u540e\u7eed\u5728\u5305\u542bGATEWAY\u8282\u70b9\u7684\u6240\u6709JindoSDK\u5168\u90e8\u5347\u7ea7\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u62e9\u673a\u53bb\u6389\u8be5\u914d\u7f6e\u3002 2. \u914d\u7f6e\u5347\u7ea7\u8282\u70b9\u4fe1\u606f \u00a7 \u7f16\u8f91patch\u5305\u4e0b\u7684hosts\u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684host name\uff0c\u5982emr-header-1\u6216emr-worker-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a emr-header-1 emr-worker-1 emr-worker-2 3. \u6267\u884c\u5347\u7ea7 \u00a7 \u901a\u8fc7apply_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./apply_all.sh $JINDOSDK_VERSION \u5982 ./apply_all.sh 6.7.3 \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... emr-header-1 >>> updating ... emr-worker-1 >>> updating ... emr-worker-2 ### DONE 4. \u4fee\u6539\u96c6\u7fa4\u914d\u7f6e \u00a7 4.1 \u517c\u5bb9 JindoSDK \u8001\u7248\u672c\u914d\u7f6e\uff08\u5347\u7ea7\u5230 JindoSDK-6.1.1\u7248\u672c\u53ca\u4ee5\u4e0a\u652f\u6301\uff09 \u00a7 a. \u5728SmartData\u670d\u52a1\u7684 \u914d\u7f6e \u9875\u9762\uff0c\u5355\u51fb smartdata-site \u9875\u7b7e\u3002 b. \u5728 smartdata-site \u9875\u9762\uff0c\u5355\u51fb \u81ea\u5b9a\u4e49\u914d\u7f6e \u3002 c. \u5728 \u65b0\u589e\u914d\u7f6e \u9879\u5bf9\u8bdd\u6846\u4e2d\uff0c\u6dfb\u52a0\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 fs.oss.legacy.version \u914d\u7f6e\u9700\u8981\u517c\u5bb9smartdata\u7248\u672c\u7684\u524d\u4e24\u4f4d\u3002\u4f8b\u5982\uff1a3.8 **\u6ce8\uff1a \u4ec5\u6d4b\u8bd5\u4e863.8.x\u7248\u672c\uff0c\u5176\u4ed6\u7248\u672c\u8bf7\u8c28\u614e\u4f7f\u7528\u3002 4.2 \u4fee\u6539 hive/spark \u914d\u7f6e \u00a7 a. \u5bf9\u5e94 \u7ec4\u4ef6(\u5982HIVE) \u670d\u52a1\u7684\u9875\u9762\uff0c\u5355\u51fb \u914d\u7f6e\uff08\u5982hive-site.xml\uff09 \u9875\u7b7e\u3002 b. \u5728 \u914d\u7f6e\uff08\u5982hive-site.xml\uff09 \u9875\u9762\uff0c\u5728 \u914d\u7f6e\u641c\u7d22 \u4e2d\u641c\u7d22 \u53c2\u6570\uff08\u5982hive.exec.post.hooks\uff09 \u3002 c. \u5982\u53c2\u6570value\u5b58\u5728\u9700\u8981\u5220\u9664\u7684\u53c2\u6570\uff0c\u4fee\u6539\u540e\u70b9\u51fb\u4fdd\u5b58\uff0c\u5e76\u66f4\u65b0\u90e8\u7f72\u914d\u7f6e\u3002\u4e0d\u5b58\u5728\u5219\u8df3\u8fc7\u3002 \u7ec4\u4ef6 \u914d\u7f6e \u53c2\u6570 \u63cf\u8ff0 HIVE hive-site.xml hive.exec.post.hooks \u5220\u9664com.aliyun.emr.table.hive.HivePostHook(\u6216com.aliyun.jindotable.hive.HivePostHook)\uff0c\u4fdd\u7559\u5176\u4f59hook\uff0c\u5982com.aliyun.emr.meta.hive.hook.LineageLoggerHook SPARK spark-defaults.conf spark.sql.queryExecutionListeners \u5220\u9664com.aliyun.emr.table.spark.SparkSQLQueryListener(\u6216com.aliyun.jindotable.spark.SparkSQLQueryListener)\uff0c\u4fdd\u7559\u5176\u4f59hook\uff0c\u5982com.aliyun.emr.meta.spark.listener.EMRQueryLogger 5. \u786e\u8ba4\u5347\u7ea7\u6210\u529f \u00a7 ls -l /opt/apps/extra-jars/ \u4ee5\u5347\u7ea7\u4e3a 6.7.3 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a drwxr-xr-x 2 root root 4096 Apr 24 16:19 flink -rw-r--r-- 1 hadoop hadoop 189081 Apr 24 15:33 hadoop-lzo-0.4.21-SNAPSHOT.jar drwxr-xr-x 2 hadoop hadoop 4096 Apr 24 15:33 hive -rw-r--r-- 1 root root 16264149 Apr 24 16:19 jindo-core-6.7.3.jar -rw-r--r-- 1 root root 14429862 Apr 24 16:19 jindo-core-linux-el7-aarch64-6.7.3.jar -rw-r--r-- 1 root root 4459297 Apr 24 16:19 jindo-sdk-6.7.3.jar -rw-r--r-- 1 hadoop hadoop 1438552 Apr 24 15:33 jindotable-common.jar drwxr-xr-x 2 root root 4096 Apr 24 16:19 spark drwxr-xr-x 2 root root 4096 Apr 24 16:19 spark3 6. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1 \u00a7 \u8bf4\u660e \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u64cd\u4f5c > \u91cd\u542f All Components \u3002 7. \u65e5\u5fd7\u7b49\u7ea7\u8c03\u6574 \u00a7 Hadoop\u547d\u4ee4\uff0cSpark\u4f5c\u4e1a\u65e5\u5fd7\u7b49log4j\u65e5\u5fd7\u53ef\u901a\u8fc7\u4ee5\u4e0b\u914d\u7f6e\u8c03\u6574\u3002\u4ee5\u8c03\u6574 hadoop \u547d\u4ee4\u65e5\u5fd7\u914d\u7f6e\u4e3a\u4f8b\uff1a vim /etc/ecm/hadoop-conf/log4j.properties \u6dfb\u52a0\u4ee5\u4e0b\u914d\u7f6e\uff1a log4j.logger.com.aliyun.jindodata=WARN \u5982\u9700\u770b\u5230\u64cd\u4f5c\u65e5\u5fd7\uff0c\u5219\u518d\u6dfb\u52a0\u4ee5\u4e0b\u914d\u7f6e\uff1a log4j.logger.com.aliyun.jindodata=WARN log4j.logger.com.aliyun.jindodata.common.FsStats=INFO \u573a\u666f\u4e8c\uff1a\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4 \u00a7 \u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u53ef\u4ee5\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u64cd\u4f5c\u6b65\u9aa4\u5982\u4e0b\uff1a 1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-6.7.3-linux.tar.gz \u548c bootstrap_jindosdk.sh\u3002 mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen $JINDOSDK_VERSION \u5982 bash bootstrap_jindosdk.sh -gen 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/hadoop/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230patch\u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002 2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u5c06 patch\u5305 \u548c bootstrap\u811a\u672c\u4e0a\u4f20\u5230OSS\u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 oss \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/jindo-patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u200b\u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002 3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u00a7 \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002 4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d \u00a7 \u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542fHive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u573a\u666f\u4e09\uff1a\u65b0\u5efa\u96c6\u7fa4 \u00a7 \u65b0\u5efaEMR\u96c6\u7fa4\u65f6\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\u3002\u5177\u4f53\u64cd\u4f5c\u6b65\u9aa4\u5982\u4e0b\uff1a 1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-6.7.3-linux.tar.gz \u548c bootstrap_jindosdk.sh\u3002 mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen-full $JINDOSDK_VERSION \u5982 bash bootstrap_jindosdk.sh -gen-full 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/hadoop/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230patch\u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002 2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u5c06 patch\u5305 \u548c bootstrap\u811a\u672c\u4e0a\u4f20\u5230OSS\u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 oss \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/jindo-patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u200b\u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002 \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u00a7 \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002 \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d \u00a7 \u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542fHive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u573a\u666f\u56db\uff1a\u56de\u6eda JindoSDK \u5230\u96c6\u7fa4\u521d\u59cb\u7248\u672c \u00a7 1. \u51c6\u5907\u56de\u6eda\u811a\u672c \u00a7 \u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728 hadoop \u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528 hadoop \u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - hadoop cd /home/hadoop/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz cd jindosdk-patches ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 hadoop hadoop 1263 May 01 00:00 apply_all.sh -rwxrwxr-x 1 hadoop hadoop 6840 May 01 00:00 apply.sh -rw-rw-r-- 1 hadoop hadoop 40 May 01 00:00 hosts -rwxrwxr-x 1 hadoop hadoop 1308 May 01 00:00 revert_all.sh -rwxrwxr-x 1 hadoop hadoop 6326 May 01 00:00 revert.sh 2. \u914d\u7f6e\u56de\u6eda\u8282\u70b9\u4fe1\u606f \u00a7 \u7f16\u8f91patch\u5305\u4e0b\u7684hosts\u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684host name\uff0c\u5982emr-header-1\u6216emr-worker-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a emr-header-1 emr-worker-1 emr-worker-2 3. \u6267\u884c\u56de\u6eda \u00a7 \u901a\u8fc7revert_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./revert_all.sh \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... emr-header-1 >>> updating ... emr-worker-1 >>> updating ... emr-worker-2 ### DONE 4. \u786e\u8ba4\u56de\u6eda\u6210\u529f \u00a7 ls -l /opt/apps/extra-jars/ \u4ee5\u56de\u6eda\u4e3a 3.7.2 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a drwxr-xr-x 2 root root 4096 Apr 24 16:22 flink -rw-r--r-- 1 hadoop hadoop 189081 Apr 24 15:33 hadoop-lzo-0.4.21-SNAPSHOT.jar drwxr-xr-x 2 root root 4096 Apr 24 16:22 hive -rw-r--r-- 1 root root 1438552 Apr 24 16:22 jindotable-common.jar drwxr-xr-x 2 root root 4096 Apr 24 16:22 native -rw-r--r-- 1 root root 29389473 Apr 24 16:22 smartdata-jindofs-3.7.2.jar drwxr-xr-x 2 root root 4096 Apr 24 16:22 spark drwxr-xr-x 2 root root 4096 Apr 24 16:22 spark3 5. \u56de\u6eda\u540e\u91cd\u542f\u670d\u52a1 \u00a7 \u8bf4\u660e \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u56de\u6eda\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u64cd\u4f5c > \u91cd\u542f All Components \u3002","title":"EMR-5.5.0/EMR-3.39.0 \u4ee5\u524d\u7248\u672c\uff08\u65e7\u7248\u63a7\u5236\u53f0\uff09"},{"location":"upgrade/emr_upgrade_smartdata/#emr-jindosdk","text":"","title":"EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b\uff08\u65e7\u7248\u63a7\u5236\u53f0\uff09"},{"location":"upgrade/emr_upgrade_smartdata/#_1","text":"\u5728\u65e7\u7248\u7ba1\u63a7\u5e73\u53f0\u4e2d\u521b\u5efaE-MapReduce EMR-5.5.0/EMR-3.39.0 \u4ee5\u524d\u7248\u672c\u7684\u96c6\u7fa4\uff0c\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u6216\u8005\u9700\u8981\u4f7f\u7528 JindoSDK \u7684\u65b0\u529f\u80fd\uff0c\u5177\u4f53\u67e5\u770b \u7248\u672c\u8bf4\u660e , \u53ef\u4ee5\u6839\u636e\u4e0b\u9762\u7684\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002","title":"\u80cc\u666f"},{"location":"upgrade/emr_upgrade_smartdata/#_2","text":"","title":"\u573a\u666f\u4e00\uff1a\u5347\u7ea7\u5df2\u6709\u96c6\u7fa4"},{"location":"upgrade/emr_upgrade_smartdata/#1","text":"\u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728 hadoop \u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528 hadoop \u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - hadoop cd /home/hadoop/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz \u4e0b\u8f7d JindoSDK \u8f6f\u4ef6\u5305 jindosdk-{VERSION}-{PLATFORM}.tar.gz\uff08\u4ee5\u4e0b\u4ee5linux x86\u5e73\u53f0\u4e3a\u4f8b\uff09 \uff0c\u653e\u5728\u89e3\u538b\u540e\u7684\u76ee\u5f55\u3002 cd jindosdk-patches wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 hadoop hadoop 1263 May 01 00:00 apply_all.sh -rwxrwxr-x 1 hadoop hadoop 6840 May 01 00:00 apply.sh -rw-rw-r-- 1 hadoop hadoop 40 May 01 00:00 hosts -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rwxrwxr-x 1 hadoop hadoop 1308 May 01 00:00 revert_all.sh -rwxrwxr-x 1 hadoop hadoop 6326 May 01 00:00 revert.sh \u6ce8\u610f \uff1a\u5982\u679c\u4ece4.6.8\u4ee5\u4e0b\u7248\u672c\u5347\u7ea7\u52304.6.9\u4ee5\u4e0a\u62166.x\u7248\u672c\u65f6\uff0c\u7531\u4e8e JindoCommitter \u9ed8\u8ba4\u4f7f\u7528\u7684\u4f5c\u4e1a\u4e34\u65f6\u8def\u5f84\u53d1\u751f\u53d8\u5316\uff0c\u9700\u8981\u5728\u5347\u7ea7\u524d \u5148\u8bbe\u7f6e fs.jdo.committer.allow.concurrent=false \uff08core-site.xml\uff09 \u6216\u5728 Spark \u914d\u7f6e\u4e2d\u8bbe\u7f6e spark.hadoop.fs.jdo.committer.allow.concurrent=false \uff0c\u786e\u4fdd\u5347\u7ea7\u671f\u95f4\u4e0d\u4f1a\u51fa\u73b0\u4e22\u6570\u636e\u7684\u60c5\u51b5\u3002 \u540e\u7eed\u5728\u5305\u542bGATEWAY\u8282\u70b9\u7684\u6240\u6709JindoSDK\u5168\u90e8\u5347\u7ea7\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u62e9\u673a\u53bb\u6389\u8be5\u914d\u7f6e\u3002","title":"1. \u51c6\u5907\u8f6f\u4ef6\u5305\u548c\u5347\u7ea7\u811a\u672c"},{"location":"upgrade/emr_upgrade_smartdata/#2","text":"\u7f16\u8f91patch\u5305\u4e0b\u7684hosts\u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684host name\uff0c\u5982emr-header-1\u6216emr-worker-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a emr-header-1 emr-worker-1 emr-worker-2","title":"2. \u914d\u7f6e\u5347\u7ea7\u8282\u70b9\u4fe1\u606f"},{"location":"upgrade/emr_upgrade_smartdata/#3","text":"\u901a\u8fc7apply_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./apply_all.sh $JINDOSDK_VERSION \u5982 ./apply_all.sh 6.7.3 \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... emr-header-1 >>> updating ... emr-worker-1 >>> updating ... emr-worker-2 ### DONE","title":"3. \u6267\u884c\u5347\u7ea7"},{"location":"upgrade/emr_upgrade_smartdata/#4","text":"","title":"4. \u4fee\u6539\u96c6\u7fa4\u914d\u7f6e"},{"location":"upgrade/emr_upgrade_smartdata/#41-jindosdk-jindosdk-611","text":"a. \u5728SmartData\u670d\u52a1\u7684 \u914d\u7f6e \u9875\u9762\uff0c\u5355\u51fb smartdata-site \u9875\u7b7e\u3002 b. \u5728 smartdata-site \u9875\u9762\uff0c\u5355\u51fb \u81ea\u5b9a\u4e49\u914d\u7f6e \u3002 c. \u5728 \u65b0\u589e\u914d\u7f6e \u9879\u5bf9\u8bdd\u6846\u4e2d\uff0c\u6dfb\u52a0\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 fs.oss.legacy.version \u914d\u7f6e\u9700\u8981\u517c\u5bb9smartdata\u7248\u672c\u7684\u524d\u4e24\u4f4d\u3002\u4f8b\u5982\uff1a3.8 **\u6ce8\uff1a \u4ec5\u6d4b\u8bd5\u4e863.8.x\u7248\u672c\uff0c\u5176\u4ed6\u7248\u672c\u8bf7\u8c28\u614e\u4f7f\u7528\u3002","title":"4.1 \u517c\u5bb9 JindoSDK \u8001\u7248\u672c\u914d\u7f6e\uff08\u5347\u7ea7\u5230 JindoSDK-6.1.1\u7248\u672c\u53ca\u4ee5\u4e0a\u652f\u6301\uff09"},{"location":"upgrade/emr_upgrade_smartdata/#42-hivespark","text":"a. \u5bf9\u5e94 \u7ec4\u4ef6(\u5982HIVE) \u670d\u52a1\u7684\u9875\u9762\uff0c\u5355\u51fb \u914d\u7f6e\uff08\u5982hive-site.xml\uff09 \u9875\u7b7e\u3002 b. \u5728 \u914d\u7f6e\uff08\u5982hive-site.xml\uff09 \u9875\u9762\uff0c\u5728 \u914d\u7f6e\u641c\u7d22 \u4e2d\u641c\u7d22 \u53c2\u6570\uff08\u5982hive.exec.post.hooks\uff09 \u3002 c. \u5982\u53c2\u6570value\u5b58\u5728\u9700\u8981\u5220\u9664\u7684\u53c2\u6570\uff0c\u4fee\u6539\u540e\u70b9\u51fb\u4fdd\u5b58\uff0c\u5e76\u66f4\u65b0\u90e8\u7f72\u914d\u7f6e\u3002\u4e0d\u5b58\u5728\u5219\u8df3\u8fc7\u3002 \u7ec4\u4ef6 \u914d\u7f6e \u53c2\u6570 \u63cf\u8ff0 HIVE hive-site.xml hive.exec.post.hooks \u5220\u9664com.aliyun.emr.table.hive.HivePostHook(\u6216com.aliyun.jindotable.hive.HivePostHook)\uff0c\u4fdd\u7559\u5176\u4f59hook\uff0c\u5982com.aliyun.emr.meta.hive.hook.LineageLoggerHook SPARK spark-defaults.conf spark.sql.queryExecutionListeners \u5220\u9664com.aliyun.emr.table.spark.SparkSQLQueryListener(\u6216com.aliyun.jindotable.spark.SparkSQLQueryListener)\uff0c\u4fdd\u7559\u5176\u4f59hook\uff0c\u5982com.aliyun.emr.meta.spark.listener.EMRQueryLogger","title":"4.2 \u4fee\u6539 hive/spark \u914d\u7f6e"},{"location":"upgrade/emr_upgrade_smartdata/#5","text":"ls -l /opt/apps/extra-jars/ \u4ee5\u5347\u7ea7\u4e3a 6.7.3 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a drwxr-xr-x 2 root root 4096 Apr 24 16:19 flink -rw-r--r-- 1 hadoop hadoop 189081 Apr 24 15:33 hadoop-lzo-0.4.21-SNAPSHOT.jar drwxr-xr-x 2 hadoop hadoop 4096 Apr 24 15:33 hive -rw-r--r-- 1 root root 16264149 Apr 24 16:19 jindo-core-6.7.3.jar -rw-r--r-- 1 root root 14429862 Apr 24 16:19 jindo-core-linux-el7-aarch64-6.7.3.jar -rw-r--r-- 1 root root 4459297 Apr 24 16:19 jindo-sdk-6.7.3.jar -rw-r--r-- 1 hadoop hadoop 1438552 Apr 24 15:33 jindotable-common.jar drwxr-xr-x 2 root root 4096 Apr 24 16:19 spark drwxr-xr-x 2 root root 4096 Apr 24 16:19 spark3","title":"5. \u786e\u8ba4\u5347\u7ea7\u6210\u529f"},{"location":"upgrade/emr_upgrade_smartdata/#6","text":"\u8bf4\u660e \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u64cd\u4f5c > \u91cd\u542f All Components \u3002","title":"6. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1"},{"location":"upgrade/emr_upgrade_smartdata/#7","text":"Hadoop\u547d\u4ee4\uff0cSpark\u4f5c\u4e1a\u65e5\u5fd7\u7b49log4j\u65e5\u5fd7\u53ef\u901a\u8fc7\u4ee5\u4e0b\u914d\u7f6e\u8c03\u6574\u3002\u4ee5\u8c03\u6574 hadoop \u547d\u4ee4\u65e5\u5fd7\u914d\u7f6e\u4e3a\u4f8b\uff1a vim /etc/ecm/hadoop-conf/log4j.properties \u6dfb\u52a0\u4ee5\u4e0b\u914d\u7f6e\uff1a log4j.logger.com.aliyun.jindodata=WARN \u5982\u9700\u770b\u5230\u64cd\u4f5c\u65e5\u5fd7\uff0c\u5219\u518d\u6dfb\u52a0\u4ee5\u4e0b\u914d\u7f6e\uff1a log4j.logger.com.aliyun.jindodata=WARN log4j.logger.com.aliyun.jindodata.common.FsStats=INFO","title":"7. \u65e5\u5fd7\u7b49\u7ea7\u8c03\u6574"},{"location":"upgrade/emr_upgrade_smartdata/#_3","text":"\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u53ef\u4ee5\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u64cd\u4f5c\u6b65\u9aa4\u5982\u4e0b\uff1a","title":"\u573a\u666f\u4e8c\uff1a\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4"},{"location":"upgrade/emr_upgrade_smartdata/#1_1","text":"\u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-6.7.3-linux.tar.gz \u548c bootstrap_jindosdk.sh\u3002 mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen $JINDOSDK_VERSION \u5982 bash bootstrap_jindosdk.sh -gen 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/hadoop/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230patch\u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002","title":"1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr_upgrade_smartdata/#2_1","text":"\u5c06 patch\u5305 \u548c bootstrap\u811a\u672c\u4e0a\u4f20\u5230OSS\u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 oss \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/jindo-patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u200b\u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002","title":"2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr_upgrade_smartdata/#3-emr","text":"\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c"},{"location":"upgrade/emr_upgrade_smartdata/#4_1","text":"\u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542fHive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002","title":"4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d"},{"location":"upgrade/emr_upgrade_smartdata/#_4","text":"\u65b0\u5efaEMR\u96c6\u7fa4\u65f6\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\u3002\u5177\u4f53\u64cd\u4f5c\u6b65\u9aa4\u5982\u4e0b\uff1a","title":"\u573a\u666f\u4e09\uff1a\u65b0\u5efa\u96c6\u7fa4"},{"location":"upgrade/emr_upgrade_smartdata/#1_2","text":"\u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-6.7.3-linux.tar.gz \u548c bootstrap_jindosdk.sh\u3002 mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen-full $JINDOSDK_VERSION \u5982 bash bootstrap_jindosdk.sh -gen-full 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/hadoop/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230patch\u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002","title":"1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr_upgrade_smartdata/#2_2","text":"\u5c06 patch\u5305 \u548c bootstrap\u811a\u672c\u4e0a\u4f20\u5230OSS\u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 oss \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/jindo-patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u200b\u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002","title":"2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr_upgrade_smartdata/#emr","text":"\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c"},{"location":"upgrade/emr_upgrade_smartdata/#_5","text":"\u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542fHive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002","title":"\u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d"},{"location":"upgrade/emr_upgrade_smartdata/#jindosdk","text":"","title":"\u573a\u666f\u56db\uff1a\u56de\u6eda JindoSDK \u5230\u96c6\u7fa4\u521d\u59cb\u7248\u672c"},{"location":"upgrade/emr_upgrade_smartdata/#1_3","text":"\u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728 hadoop \u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528 hadoop \u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - hadoop cd /home/hadoop/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz cd jindosdk-patches ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 hadoop hadoop 1263 May 01 00:00 apply_all.sh -rwxrwxr-x 1 hadoop hadoop 6840 May 01 00:00 apply.sh -rw-rw-r-- 1 hadoop hadoop 40 May 01 00:00 hosts -rwxrwxr-x 1 hadoop hadoop 1308 May 01 00:00 revert_all.sh -rwxrwxr-x 1 hadoop hadoop 6326 May 01 00:00 revert.sh","title":"1. \u51c6\u5907\u56de\u6eda\u811a\u672c"},{"location":"upgrade/emr_upgrade_smartdata/#2_3","text":"\u7f16\u8f91patch\u5305\u4e0b\u7684hosts\u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684host name\uff0c\u5982emr-header-1\u6216emr-worker-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a emr-header-1 emr-worker-1 emr-worker-2","title":"2. \u914d\u7f6e\u56de\u6eda\u8282\u70b9\u4fe1\u606f"},{"location":"upgrade/emr_upgrade_smartdata/#3_1","text":"\u901a\u8fc7revert_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./revert_all.sh \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... emr-header-1 >>> updating ... emr-worker-1 >>> updating ... emr-worker-2 ### DONE","title":"3. \u6267\u884c\u56de\u6eda"},{"location":"upgrade/emr_upgrade_smartdata/#4_2","text":"ls -l /opt/apps/extra-jars/ \u4ee5\u56de\u6eda\u4e3a 3.7.2 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a drwxr-xr-x 2 root root 4096 Apr 24 16:22 flink -rw-r--r-- 1 hadoop hadoop 189081 Apr 24 15:33 hadoop-lzo-0.4.21-SNAPSHOT.jar drwxr-xr-x 2 root root 4096 Apr 24 16:22 hive -rw-r--r-- 1 root root 1438552 Apr 24 16:22 jindotable-common.jar drwxr-xr-x 2 root root 4096 Apr 24 16:22 native -rw-r--r-- 1 root root 29389473 Apr 24 16:22 smartdata-jindofs-3.7.2.jar drwxr-xr-x 2 root root 4096 Apr 24 16:22 spark drwxr-xr-x 2 root root 4096 Apr 24 16:22 spark3","title":"4. \u786e\u8ba4\u56de\u6eda\u6210\u529f"},{"location":"upgrade/emr_upgrade_smartdata/#5_1","text":"\u8bf4\u660e \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u56de\u6eda\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u64cd\u4f5c > \u91cd\u542f All Components \u3002","title":"5. \u56de\u6eda\u540e\u91cd\u542f\u670d\u52a1"}]}
\ No newline at end of file
+{"config":{"indexing":"full","lang":["en"],"min_search_length":3,"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"JindoData \u4f7f\u7528\u6587\u6863 \u00a7 JindoData \u662f\u963f\u91cc\u4e91\u5f00\u6e90\u5927\u6570\u636e\u56e2\u961f\u81ea\u7814\u7684\u6570\u636e\u6e56\u5b58\u50a8\u52a0\u901f\u5957\u4ef6\uff0c\u9762\u5411\u5927\u6570\u636e\u548c AI \u751f\u6001\uff0c\u4e3a\u963f\u91cc\u4e91\u548c\u4e1a\u754c\u4e3b\u8981\u6570\u636e\u6e56\u5b58\u50a8\u7cfb\u7edf\u63d0\u4f9b\u5168\u65b9\u4f4d\u8bbf\u95ee\u52a0\u901f\u89e3\u51b3\u65b9\u6848\u3002 JindoData \u5957\u4ef6\u57fa\u4e8e\u7edf\u4e00\u67b6\u6784\u548c\u5185\u6838\u5b9e\u73b0\uff0c\u652f\u6301\u901a\u8fc7\u6570\u636e\u6e56\u4e07\u80fd\u7684 SDK\uff08JindoSDK\uff09 \u8bbf\u95ee OSS/OSS-HDFS\u3002JindoSDK \u517c\u5bb9 HCFS \u63a5\u53e3\uff0c\u5bf9\u8c61\u5b58\u50a8\u63a5\u53e3 \u548c POSIX \u63a5\u53e3\u3001\u4e5f\u652f\u6301 Python\u3001Tensorflow \u7b49\uff0c\u62e5\u6709\u5168\u9762\u517c\u5bb9\u7684\u751f\u6001\u5de5\u5177\uff08Jindo Cli\u3001JindoFuse\u3001JindoDistCp\uff09\u548c\u63d2\u4ef6\u652f\u6301\u3002 \u4e0b\u8f7d\u5b89\u88c5 JindoSDK \u00a7 JindoSDK \u662f\u8bbf\u95ee JindoData \u7ec4\u4ef6\u7684\u6807\u51c6\u5ba2\u6237\u7aef\uff0c\u8bf7\u53c2\u8003 JindoSDK \u4e0b\u8f7d \u548c JindoSDK \u5feb\u901f\u5165\u95e8 \u8fdb\u884c\u5b89\u88c5\u548c\u9a8c\u8bc1\u3002 \u5173\u4e8e\u591a\u5e73\u53f0\u652f\u6301\uff0c\u8bf7\u53c2\u8003 JindoSDK \u591a\u5e73\u53f0\u652f\u6301 \u3002 \u5347\u7ea7 JindoSDK \u00a7 JindoSDK \u662f\u6d3b\u8dc3\u66f4\u65b0\u7684\u5ba2\u6237\u7aef\uff0c\u6301\u7eed\u4e3a\u7528\u6237\u5e26\u6765\u963f\u91cc\u4e91EMR\u6570\u636e\u6e56\u7684\u6700\u65b0\u529f\u80fd\u548c\u6700\u4f73\u6027\u80fd\u3002 \u56e0\u6b64\u6211\u4eec\u63a8\u8350\u5ba2\u6237\u4f7f\u7528\u65b0\u7248 JindoSDK\uff0c\u4ee5\u83b7\u53d6\u6301\u7eed\u652f\u6301\u548c\u66f4\u597d\u7684\u4f7f\u7528\u4f53\u9a8c\u3002 \u6211\u4eec\u63d0\u4f9b\u4e86\u4fbf\u6377\u7684\u811a\u672c\u5e2e\u52a9\u7528\u6237\u5347\u7ea7\u96c6\u7fa4\u91cc\u7684 JindoSDK\uff0c\u8bf7\u53c2\u8003 JindoSDK\u5347\u7ea7\u6587\u6863 \u3002 \u5206\u573a\u666f\u4f7f\u7528\u8bf4\u660e \u00a7 \u5728 Hadoop \u751f\u6001\u4f7f\u7528JindoSDK \u5728 AI \u751f\u6001\u4f7f\u7528JindoSDK Jindo Python SDK \u4f7f\u7528 Jindo TensorFlow Connector \u4f7f\u7528 JindoFuse \u4f7f\u7528 Jindo Flink Sink \u4f7f\u7528 \u5728 Fluid \u4e2d\u4f7f\u7528 JindoRuntime JindoData \u9274\u6743\u65b9\u6848 \u6570\u636e\u62f7\u8d1d\u5de5\u5177 JindoDistCp \u6570\u4ed3\u8fc1\u79fb\u5de5\u5177 JindoTable MoveTo \u6570\u4ed3\u5206\u5c42\u5b58\u50a8\u7ba1\u7406\u5de5\u5177 JindoTable SetStorage OSS-HDFS\u670d\u52a1\uff08JindoFS\uff09\u5ba2\u6237\u7aef\u5de5\u5177 \u5e38\u89c1\u95ee\u9898 \u00a7 \u8bf7\u53c2\u89c1 JindoData \u5e38\u89c1\u95ee\u9898 \u5386\u53f2\u7248\u672c \u00a7 \u8bf7\u53c2\u89c1 JindoSDK \u7248\u672c\u8bb0\u5f55 \u3002","title":"\u9996\u9875"},{"location":"#jindodata","text":"JindoData \u662f\u963f\u91cc\u4e91\u5f00\u6e90\u5927\u6570\u636e\u56e2\u961f\u81ea\u7814\u7684\u6570\u636e\u6e56\u5b58\u50a8\u52a0\u901f\u5957\u4ef6\uff0c\u9762\u5411\u5927\u6570\u636e\u548c AI \u751f\u6001\uff0c\u4e3a\u963f\u91cc\u4e91\u548c\u4e1a\u754c\u4e3b\u8981\u6570\u636e\u6e56\u5b58\u50a8\u7cfb\u7edf\u63d0\u4f9b\u5168\u65b9\u4f4d\u8bbf\u95ee\u52a0\u901f\u89e3\u51b3\u65b9\u6848\u3002 JindoData \u5957\u4ef6\u57fa\u4e8e\u7edf\u4e00\u67b6\u6784\u548c\u5185\u6838\u5b9e\u73b0\uff0c\u652f\u6301\u901a\u8fc7\u6570\u636e\u6e56\u4e07\u80fd\u7684 SDK\uff08JindoSDK\uff09 \u8bbf\u95ee OSS/OSS-HDFS\u3002JindoSDK \u517c\u5bb9 HCFS \u63a5\u53e3\uff0c\u5bf9\u8c61\u5b58\u50a8\u63a5\u53e3 \u548c POSIX \u63a5\u53e3\u3001\u4e5f\u652f\u6301 Python\u3001Tensorflow \u7b49\uff0c\u62e5\u6709\u5168\u9762\u517c\u5bb9\u7684\u751f\u6001\u5de5\u5177\uff08Jindo Cli\u3001JindoFuse\u3001JindoDistCp\uff09\u548c\u63d2\u4ef6\u652f\u6301\u3002","title":"JindoData \u4f7f\u7528\u6587\u6863"},{"location":"#jindosdk","text":"JindoSDK \u662f\u8bbf\u95ee JindoData \u7ec4\u4ef6\u7684\u6807\u51c6\u5ba2\u6237\u7aef\uff0c\u8bf7\u53c2\u8003 JindoSDK \u4e0b\u8f7d \u548c JindoSDK \u5feb\u901f\u5165\u95e8 \u8fdb\u884c\u5b89\u88c5\u548c\u9a8c\u8bc1\u3002 \u5173\u4e8e\u591a\u5e73\u53f0\u652f\u6301\uff0c\u8bf7\u53c2\u8003 JindoSDK \u591a\u5e73\u53f0\u652f\u6301 \u3002","title":"\u4e0b\u8f7d\u5b89\u88c5 JindoSDK"},{"location":"#jindosdk_1","text":"JindoSDK \u662f\u6d3b\u8dc3\u66f4\u65b0\u7684\u5ba2\u6237\u7aef\uff0c\u6301\u7eed\u4e3a\u7528\u6237\u5e26\u6765\u963f\u91cc\u4e91EMR\u6570\u636e\u6e56\u7684\u6700\u65b0\u529f\u80fd\u548c\u6700\u4f73\u6027\u80fd\u3002 \u56e0\u6b64\u6211\u4eec\u63a8\u8350\u5ba2\u6237\u4f7f\u7528\u65b0\u7248 JindoSDK\uff0c\u4ee5\u83b7\u53d6\u6301\u7eed\u652f\u6301\u548c\u66f4\u597d\u7684\u4f7f\u7528\u4f53\u9a8c\u3002 \u6211\u4eec\u63d0\u4f9b\u4e86\u4fbf\u6377\u7684\u811a\u672c\u5e2e\u52a9\u7528\u6237\u5347\u7ea7\u96c6\u7fa4\u91cc\u7684 JindoSDK\uff0c\u8bf7\u53c2\u8003 JindoSDK\u5347\u7ea7\u6587\u6863 \u3002","title":"\u5347\u7ea7 JindoSDK"},{"location":"#_1","text":"\u5728 Hadoop \u751f\u6001\u4f7f\u7528JindoSDK \u5728 AI \u751f\u6001\u4f7f\u7528JindoSDK Jindo Python SDK \u4f7f\u7528 Jindo TensorFlow Connector \u4f7f\u7528 JindoFuse \u4f7f\u7528 Jindo Flink Sink \u4f7f\u7528 \u5728 Fluid \u4e2d\u4f7f\u7528 JindoRuntime JindoData \u9274\u6743\u65b9\u6848 \u6570\u636e\u62f7\u8d1d\u5de5\u5177 JindoDistCp \u6570\u4ed3\u8fc1\u79fb\u5de5\u5177 JindoTable MoveTo \u6570\u4ed3\u5206\u5c42\u5b58\u50a8\u7ba1\u7406\u5de5\u5177 JindoTable SetStorage OSS-HDFS\u670d\u52a1\uff08JindoFS\uff09\u5ba2\u6237\u7aef\u5de5\u5177","title":"\u5206\u573a\u666f\u4f7f\u7528\u8bf4\u660e"},{"location":"#_2","text":"\u8bf7\u53c2\u89c1 JindoData \u5e38\u89c1\u95ee\u9898","title":"\u5e38\u89c1\u95ee\u9898"},{"location":"#_3","text":"\u8bf7\u53c2\u89c1 JindoSDK \u7248\u672c\u8bb0\u5f55 \u3002","title":"\u5386\u53f2\u7248\u672c"},{"location":"faq/","text":"JindoData \u5e38\u89c1\u95ee\u9898 \u00a7 \u5e38\u89c1\u95ee\u9898 \u00a7 JindoData \u5957\u4ef6\u548c\u963f\u91cc\u4e91 EMR \u662f\u4ec0\u4e48\u5173\u7cfb\uff1f JindoData \u5957\u4ef6\u662f\u963f\u91cc\u4e91\u5f00\u6e90\u5927\u6570\u636e JindoData \u56e2\u961f\u5f00\u53d1\uff0c\u9762\u5411\u5f00\u6e90\u751f\u6001\u548c\u793e\u533a\u5168\u9762\u5f00\u653e\u548c\u63d0\u4f9b\uff0c\u9664\u4e86 EMR \u4ea7\u54c1\u9ed8\u8ba4\u96c6\u6210\uff0c\u8fd8\u652f\u6301\u963f\u91cc\u4e91\u4e0a\u591a\u79cd\u573a\u666f\u90e8\u7f72\u4f7f\u7528\uff0c\u5305\u62ec ECS \u81ea\u5efa\u96c6\u7fa4\u3001ACK \u4e91\u539f\u751f Fluid \u96c6\u6210\uff0c\u751a\u81f3 IDC \u96c6\u7fa4\u73af\u5883\u3002\u963f\u91cc\u4e91 EMR \u4ea7\u54c1\u9488\u5bf9\u591a\u79cd\u573a\u666f\u63d0\u4f9b\u7684\u591a\u79cd\u96c6\u7fa4\u7c7b\u578b\uff08DataLake\u3001DataFlow\u3001DataScience \u548c OLAP\uff09\u4f1a\u9ed8\u8ba4\u96c6\u6210\u6240\u9700\u8981\u7684 JindoData \u76f8\u5e94\u7ec4\u4ef6\uff0c\u63d0\u4f9b\u66f4\u52a0\u4f18\u5316\u3001\u6613\u7528\u7684\u4f7f\u7528\u4f53\u9a8c\u3002 JindoData \u5957\u4ef6\u548c EMR SmartData \u7ec4\u4ef6\u662f\u4ec0\u4e48\u5173\u7cfb\uff1f JindoData \u5957\u4ef6\u524d\u8eab\u5c31\u662f\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1 SmartData \u7ec4\u4ef6\uff0c\u662f\u5bf9 SmartData \u7684\u67b6\u6784\u5347\u7ea7\uff0c\u66f4\u52a0\u6a21\u5757\u5316\uff0c\u9762\u5411\u793e\u533a\u751f\u6001\u7cfb\u7edf\u66f4\u52a0\u5f00\u653e\u548c\u517c\u5bb9\u3002 JindoData \u5957\u4ef6\u548c JindoFS \u662f\u4ec0\u4e48\u5173\u7cfb\uff1f JindoData \u5957\u4ef6\u57fa\u4e8e\u7edf\u4e00\u7684\u67b6\u6784\u548c\u5185\u6838\u5b9e\u73b0\uff0c\u4e0d\u4ec5\u5305\u62ec JindoFS\uff08\u5b58\u50a8\u7cfb\u7edf\uff09\uff0c\u8fd8\u5305\u542b JindoFSx\uff08\u7f13\u5b58\u7cfb\u7edf\uff09\uff0cJindoSDK \u548c\u751f\u6001\u5de5\u5177\u3001\u63d2\u4ef6\u3002 JindoData \u5957\u4ef6\u548c Jindo \u662f\u4ec0\u4e48\u5173\u7cfb\uff1f Jindo \u7ea6\u7b49\u4e8e JindoData\uff0c\u662f\u540e\u8005\u7684\u7b80\u4fbf\u53eb\u6cd5\u3002 JindoFS \u548c OSS-HDFS \u670d\u52a1\u4e4b\u95f4\u7684\u5173\u7cfb OSS-HDFS \u670d\u52a1\uff08\u6709\u65f6\u4e5f\u63d0 JindoFS \u670d\u52a1\uff09\u662f JindoFS \u5b58\u50a8\u7cfb\u7edf\u5728\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u670d\u52a1\u5316\u90e8\u7f72\u5f62\u6001\uff0c\u548c OSS \u5bf9\u8c61\u5b58\u50a8\u4ea7\u54c1\u6df1\u5ea6\u878d\u5408\uff0c\u4f5c\u4e3a\u4e00\u79cd\u65b0\u7684 bucket \u7c7b\u578b\uff0c\u5f00\u7bb1\u5373\u7528\u3002 JindoFS \u670d\u52a1\u5316\u6210\u4e3a OSS-HDFS \u670d\u52a1\uff0c\u4e3a\u4ec0\u4e48\u53eb HDFS\uff1f JindoFS \u5b58\u50a8\u7cfb\u7edf\u4e8c\u8fdb\u5236\u517c\u5bb9 Apache HDFS\uff0c\u5e76\u4e14\u57fa\u672c\u529f\u80fd\u5b8c\u5168\u5bf9\u9f50\uff0c\u63d0\u4f9b\u4f18\u5316\u7684 HDFS \u4f7f\u7528\u4f53\u9a8c\uff0c\u53ef\u5b9e\u73b0 HDFS \u81ea\u5efa\u96c6\u7fa4\u5e73\u8fc1\u4e0a\u4e91\u4e0a\u963f\u91cc\u4e91 OSS\u3002 \u5386\u53f2\u7248\u672c\u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.12 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.11 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.10 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.9 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.8 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.7 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.6 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.5 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.4 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.3 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.2 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.1 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.0 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.5.1 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.5.0 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.4.0 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.3.0 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.2.0 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.1.0 \u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.0.0 \u5df2\u77e5\u95ee\u9898 \u00a7","title":"\u5e38\u89c1\u95ee\u9898"},{"location":"faq/#jindodata","text":"","title":"JindoData \u5e38\u89c1\u95ee\u9898"},{"location":"faq/#_1","text":"JindoData \u5957\u4ef6\u548c\u963f\u91cc\u4e91 EMR \u662f\u4ec0\u4e48\u5173\u7cfb\uff1f JindoData \u5957\u4ef6\u662f\u963f\u91cc\u4e91\u5f00\u6e90\u5927\u6570\u636e JindoData \u56e2\u961f\u5f00\u53d1\uff0c\u9762\u5411\u5f00\u6e90\u751f\u6001\u548c\u793e\u533a\u5168\u9762\u5f00\u653e\u548c\u63d0\u4f9b\uff0c\u9664\u4e86 EMR \u4ea7\u54c1\u9ed8\u8ba4\u96c6\u6210\uff0c\u8fd8\u652f\u6301\u963f\u91cc\u4e91\u4e0a\u591a\u79cd\u573a\u666f\u90e8\u7f72\u4f7f\u7528\uff0c\u5305\u62ec ECS \u81ea\u5efa\u96c6\u7fa4\u3001ACK \u4e91\u539f\u751f Fluid \u96c6\u6210\uff0c\u751a\u81f3 IDC \u96c6\u7fa4\u73af\u5883\u3002\u963f\u91cc\u4e91 EMR \u4ea7\u54c1\u9488\u5bf9\u591a\u79cd\u573a\u666f\u63d0\u4f9b\u7684\u591a\u79cd\u96c6\u7fa4\u7c7b\u578b\uff08DataLake\u3001DataFlow\u3001DataScience \u548c OLAP\uff09\u4f1a\u9ed8\u8ba4\u96c6\u6210\u6240\u9700\u8981\u7684 JindoData \u76f8\u5e94\u7ec4\u4ef6\uff0c\u63d0\u4f9b\u66f4\u52a0\u4f18\u5316\u3001\u6613\u7528\u7684\u4f7f\u7528\u4f53\u9a8c\u3002 JindoData \u5957\u4ef6\u548c EMR SmartData \u7ec4\u4ef6\u662f\u4ec0\u4e48\u5173\u7cfb\uff1f JindoData \u5957\u4ef6\u524d\u8eab\u5c31\u662f\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1 SmartData \u7ec4\u4ef6\uff0c\u662f\u5bf9 SmartData \u7684\u67b6\u6784\u5347\u7ea7\uff0c\u66f4\u52a0\u6a21\u5757\u5316\uff0c\u9762\u5411\u793e\u533a\u751f\u6001\u7cfb\u7edf\u66f4\u52a0\u5f00\u653e\u548c\u517c\u5bb9\u3002 JindoData \u5957\u4ef6\u548c JindoFS \u662f\u4ec0\u4e48\u5173\u7cfb\uff1f JindoData \u5957\u4ef6\u57fa\u4e8e\u7edf\u4e00\u7684\u67b6\u6784\u548c\u5185\u6838\u5b9e\u73b0\uff0c\u4e0d\u4ec5\u5305\u62ec JindoFS\uff08\u5b58\u50a8\u7cfb\u7edf\uff09\uff0c\u8fd8\u5305\u542b JindoFSx\uff08\u7f13\u5b58\u7cfb\u7edf\uff09\uff0cJindoSDK \u548c\u751f\u6001\u5de5\u5177\u3001\u63d2\u4ef6\u3002 JindoData \u5957\u4ef6\u548c Jindo \u662f\u4ec0\u4e48\u5173\u7cfb\uff1f Jindo \u7ea6\u7b49\u4e8e JindoData\uff0c\u662f\u540e\u8005\u7684\u7b80\u4fbf\u53eb\u6cd5\u3002 JindoFS \u548c OSS-HDFS \u670d\u52a1\u4e4b\u95f4\u7684\u5173\u7cfb OSS-HDFS \u670d\u52a1\uff08\u6709\u65f6\u4e5f\u63d0 JindoFS \u670d\u52a1\uff09\u662f JindoFS \u5b58\u50a8\u7cfb\u7edf\u5728\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u670d\u52a1\u5316\u90e8\u7f72\u5f62\u6001\uff0c\u548c OSS \u5bf9\u8c61\u5b58\u50a8\u4ea7\u54c1\u6df1\u5ea6\u878d\u5408\uff0c\u4f5c\u4e3a\u4e00\u79cd\u65b0\u7684 bucket \u7c7b\u578b\uff0c\u5f00\u7bb1\u5373\u7528\u3002 JindoFS \u670d\u52a1\u5316\u6210\u4e3a OSS-HDFS \u670d\u52a1\uff0c\u4e3a\u4ec0\u4e48\u53eb HDFS\uff1f JindoFS \u5b58\u50a8\u7cfb\u7edf\u4e8c\u8fdb\u5236\u517c\u5bb9 Apache HDFS\uff0c\u5e76\u4e14\u57fa\u672c\u529f\u80fd\u5b8c\u5168\u5bf9\u9f50\uff0c\u63d0\u4f9b\u4f18\u5316\u7684 HDFS \u4f7f\u7528\u4f53\u9a8c\uff0c\u53ef\u5b9e\u73b0 HDFS \u81ea\u5efa\u96c6\u7fa4\u5e73\u8fc1\u4e0a\u4e91\u4e0a\u963f\u91cc\u4e91 OSS\u3002","title":"\u5e38\u89c1\u95ee\u9898"},{"location":"faq/#_2","text":"","title":"\u5386\u53f2\u7248\u672c\u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-4612","text":"","title":"JindoSDK 4.6.12 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-4611","text":"","title":"JindoSDK 4.6.11 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-4610","text":"","title":"JindoSDK 4.6.10 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-469","text":"","title":"JindoSDK 4.6.9 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-468","text":"","title":"JindoSDK 4.6.8 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-467","text":"","title":"JindoSDK 4.6.7 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-466","text":"","title":"JindoSDK 4.6.6 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-465","text":"","title":"JindoSDK 4.6.5 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-464","text":"","title":"JindoSDK 4.6.4 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-463","text":"","title":"JindoSDK 4.6.3 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-462","text":"","title":"JindoSDK 4.6.2 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-461","text":"","title":"JindoSDK 4.6.1 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-460","text":"","title":"JindoSDK 4.6.0 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-451","text":"","title":"JindoSDK 4.5.1 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-450","text":"","title":"JindoSDK 4.5.0 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-440","text":"","title":"JindoSDK 4.4.0 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-430","text":"","title":"JindoSDK 4.3.0 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-420","text":"","title":"JindoSDK 4.2.0 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-410","text":"","title":"JindoSDK 4.1.0 \u5df2\u77e5\u95ee\u9898"},{"location":"faq/#jindosdk-400","text":"","title":"JindoSDK 4.0.0 \u5df2\u77e5\u95ee\u9898"},{"location":"releases/","text":"JindoSDK \u7248\u672c\u8bb0\u5f55 \u00a7 6.7.3\uff0c2024-11-01 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.7.3 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.7.3 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u6253\u5305\u65f6\u5e26\u5165 jindoauth plugin\uff0c\u652f\u6301\u5347\u7ea7 JindoSDK \u540e\u4ecd\u517c\u5bb9\u8001\u7248\u672c EMR-Ranger\uff0c\u5177\u4f53\u53c2\u89c1 \u5347\u7ea7\u6587\u6863 \u3002 nextarch classifier \u4fee\u590d hudi \u8bfb\u5199 log \u65f6\uff0c\u5bf9 AlreadyBeingCreatedException \u5f02\u5e38\u5904\u7406\u3002 nextarch classifier \u4fee\u590d\u5728\u4f7f\u7528 http \u8bf7\u6c42\u66f4\u65b0 STS \u65f6\uff0c\u5076\u73b0\u5361\u4f4f\u95ee\u9898\u3002 nextarch classifier \u4fee\u590d\u5f00\u542f Ranger \u540e\u6301\u7eed\u5199\u5165\u4e00\u4e2a OSS-HDFS \u6587\u4ef6\u8d85\u8fc7 1 \u5c0f\u65f6\u540e\uff0c\u6587\u4ef6\u65e0\u6cd5\u7ee7\u7eed\u5199\u5165\u7684\u95ee\u9898\u3002 6.4.1\uff0c2024-10-23 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.4.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.4.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d\u5076\u73b0 Signer V4 \u7b7e\u540d\u95ee\u9898\uff0c\u5c06\u7b7e\u540d\u65f6\u95f4\u6233\u8f6c\u6362\u51fd\u6570 localtime\u3001gmtime \u66ff\u6362\u4e3a localtime_r\u3001gmtime_r \u907f\u514d\u5e76\u53d1\u95ee\u9898\u3002 \u4fee\u590d libjindosdk_c.so \u8bbf\u95ee OSS-HDFS \u5076\u73b0 crash \u95ee\u9898\uff0c\u53ca\u91cd\u8bd5\u5931\u8d25\u95ee\u9898\u3002 \u4fee\u590d hudi \u8bfb\u5199 log \u65f6\uff0c\u5bf9 AlreadyBeingCreatedException \u5f02\u5e38\u5904\u7406\u3002 \u4fee\u590d JindoOssFileSystem \u7684 Delegation Token Renew \u673a\u5236\u3002 \u4fee\u590d JindoCommitter \u8bbf\u95ee OSS-HDFS \u672a\u6e05\u7406 task \u6b8b\u7559\u7684\u4e34\u65f6\u76ee\u5f55\u3002 \u63d0\u5347 JindoSDK \u5bf9\u4e8e\u4f4e\u4e8e Hadoop 2.8.x \u7248\u672c\u63a5\u53e3\u7684\u517c\u5bb9\u6027\uff0c\u5982 CallerContext \u53ca FsServerDefaults \u3002 \u4f18\u5316 classloader \u52a0\u8f7d\uff0c\u5728\u591a\u4e2a classloader \u52a0\u8f7d jindosdk \u65f6\u7684 so \u6b8b\u7559\u95ee\u9898\uff0c\u5e76\u4e14\u5728\u914d\u7f6e\u4e86 java.io.tmpdir \u65f6\uff0c\u9ed8\u8ba4\u89e3\u538b\u81f3\u914d\u7f6e\u76ee\u5f55\u4e0b\u3002 6.7.2\uff0c2024-10-21 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.7.2 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.7.2 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u4f18\u5316 metrics \u7edf\u8ba1\u4fe1\u606f\u3002 nextarch classifier \u4fee\u590d\u5728\u5199 OSS-HDFS \u573a\u666f flush \u540e\u51fa\u73b0\u5c0f\u5757\u7684\u95ee\u9898\u3002 nextarch classifier \u4fee\u590d\u540e\u53f0 metrics \u7ebf\u7a0b\u5f15\u53d1\u7684\u5076\u73b0 crash \u95ee\u9898\u3002 Pyjindo \u4fee\u590d 6.7.x \u7248\u672c\u62a5 invalid pointer \u95ee\u9898\uff0c6.7.0 \u53ca 6.7.1 \u7248\u672c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u73af\u5883\u53d8\u91cf JINDO_STAT_MEMORY=0 \u7ed5\u8fc7\u3002 6.7.1\uff0c2024-10-15 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.7.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.7.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u4f18\u5316\u9ed8\u8ba4\u5e76\u53d1\u4e0b\u7684\u5199\u6027\u80fd\u3002 \u4fee\u590d JindoCommitter \u8bbf\u95ee OSS-HDFS \u672a\u6e05\u7406 task \u6b8b\u7559\u7684\u4e34\u65f6\u76ee\u5f55\u3002 jindo-fuse \u4fee\u590d\u5728 s3 scheme \u4e0a\u8ffd\u52a0\u5199\u7684\u652f\u6301\u3002 6.7.0\uff0c2024-09-29 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.7.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.7.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u652f\u6301\u7edf\u4e00\u7ba1\u7406\u5185\u5b58\uff0c\u4f7f\u7528 fs.jdo.memory.pool.size.max.mb \u914d\u7f6e\u6700\u5927\u4f7f\u7528\u5185\u5b58\uff0c\u5e76\u652f\u6301\u5185\u5b58\u5b9e\u65f6\u4f7f\u7528 metrics\u3002 nextarch classifier \u652f\u6301 io buffer \u5185\u5b58\u4f7f\u7528\u63a7\u5236\uff0c\u4f7f\u7528 fs.jdo.memory.io.buffer.size.max.ratio \u914d\u7f6e\u6700\u5927\u4f7f\u7528\u5185\u5b58\u6c60\u7684\u6bd4\u4f8b\u3002 nextarch classifier \u652f\u6301 Golang SDK\u3002 nextarch classifier \u652f\u6301 OSS \u5bf9\u8c61\u5b58\u50a8\u6df1\u51b7\u5f52\u6863\u3002 nextarch classifier \u4f18\u5316 pread \u63a5\u53e3\uff0c\u5185\u5b58\u96f6\u62f7\u8d1d\u3002 nextarch classifier \u4f18\u5316 sendfile \u5728\u5199\u5c0f\u6587\u4ef6\u573a\u666f\u7684\u4f7f\u7528\u3002 nextarch classifier \u4f18\u5316 metrics \u6846\u67b6\uff0c\u652f\u6301\u6bcf\u6b21\u8f93\u51fa\u5230\u5355\u72ec\u6587\u4ef6\uff0c\u65b9\u4fbf\u91c7\u96c6\u3002 nextarch classifier \u4f18\u5316 distjob \u6846\u4e2d\uff0c\u6392\u9664 log4j \u76f8\u5173\u4f9d\u8d56\u3002 nextarch classifier \u4f18\u5316 classloader \u52a0\u8f7d\uff0c\u5728\u591a\u4e2a classloader \u52a0\u8f7d jindosdk \u65f6\u7684 so \u6b8b\u7559\u95ee\u9898\uff0c\u5e76\u4e14\u5728\u914d\u7f6e\u4e86 java.io.tmpdir \u65f6\uff0c\u9ed8\u8ba4\u89e3\u538b\u81f3\u914d\u7f6e\u76ee\u5f55\u4e0b\u3002 nextarch classifier \u4fee\u590d OSS-HDFS StoragePolicy \u76f8\u5173\u63a5\u53e3\u3002 nextarch classifier \u4fee\u590d OSS \u5bf9\u8c61\u5b58\u50a8\uff0cListDirectory \u8fd4\u56de\u7ed3\u679c\u7684 Mtime \u5355\u4f4d\u3002 nextarch classifier \u4fee\u590d JindoCommitter \u5728 Cleanup \u9636\u6bb5\uff0c\u4e0d\u5e94\u629b\u51fa\u5f02\u5e38\u3002 nextarch classifier \u4fee\u590d DLF \u76f8\u5173 CredentialProvider \u4f7f\u7528 MagicCommitter \u65f6\u7684\u6743\u9650\u95ee\u9898\u3002 nextarch classifier \u4fee\u590d JindoCache \u7684\u82e5\u5e72\u95ee\u9898\u3002 6.6.3\uff0c2024-09-14 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.6.3 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.6.3 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u4fee\u590d\u5199\u573a\u666f\u5c0f\u6982\u7387\u56e0 sendfile \u51fa\u73b0\u9519\u8bef\u672a\u91cd\u8bd5\uff0c\u5bfc\u81f4\u6587\u4ef6 close \u65f6\u62a5\u5931\u8d25\u7684\u60c5\u51b5\u3002 6.6.2\uff0c2024-09-12 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.6.2 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.6.2 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u8bbf\u95ee OSS \u652f\u6301\u6df1\u51b7\u5f52\u6863 nextarch classifier \u4fee\u590d DLF \u76f8\u5173 CredentialProvider \u4f7f\u7528 MagicCommitter \u65f6\u7684\u6743\u9650\u95ee\u9898\u3002 6.6.1\uff0c2024-09-03 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.6.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.6.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u4f18\u5316\u5728\u591a\u4e2a classloader \u52a0\u8f7d jindosdk \u65f6\u7684 so \u6b8b\u7559\u95ee\u9898\uff0c\u5e76\u4e14\u5728\u914d\u7f6e\u4e86 java.io.tmpdir \u65f6\uff0c\u9ed8\u8ba4\u89e3\u538b\u81f3\u914d\u7f6e\u76ee\u5f55\u4e0b\u3002 nextarch classifier \u4fee\u590d\u8bbf\u95ee JindoCache \u7684\u82e5\u5e72\u95ee\u9898\u3002 nextarch classifier \u4f18\u5316 metrics \u6846\u67b6\u3002 6.6.0\uff0c2024-08-25 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.6.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.6.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u4f18\u5316 io \u6027\u80fd\uff0c\u4f7f\u7528\u534f\u7a0b\u5f02\u6b65\u5316\uff0c\u76f8\u540c\u914d\u7f6e\u4e0b\u652f\u6301\u66f4\u9ad8\u5e76\u53d1\u3002\u5199\u573a\u666f\u4f7f\u7528 sendfile \u5b9e\u73b0\u96f6\u62f7\u8d1d\u4f18\u5316\uff0c\u8282\u7701\u5185\u5b58\uff0c\u5e76\u63d0\u5347\u6027\u80fd\u3002 nextarch classifier \u652f\u6301\u5355\u72ec\u6307\u5b9a io \u8d85\u65f6\u65f6\u95f4\u3002 nextarch classifier \u652f\u6301 metrics \u6846\u67b6\u3002 \u4fee\u590d nextarch classifier \u5bf9 JindoCache \u652f\u6301\u95ee\u9898\u3002 jindo-fuse \u9ed8\u8ba4\u6539\u4e3a\u4f7f\u7528 nextarch \u5b9e\u73b0\u3002 6.5.4\uff0c2024-08-20 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.5.4 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.5.4 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4f18\u5316 flink connector \u4f9d\u8d56\uff0c\u5bf9\u90e8\u5206\u4f9d\u8d56\u8fdb\u884c shade\u3002 6.5.3\uff0c2024-08-15 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.5.3 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.5.3 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d JindoOssFileSystem \u7684 Delegation Token Renew \u673a\u5236\u3002 6.5.2\uff0c2024-08-12 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.5.2 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.5.2 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d getfacl \u7f3a\u5c11 entries \u95ee\u9898\u3002 6.5.1\uff0c2024-08-05 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.5.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.5.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u63d0\u5347 JindoSDK \u5bf9\u4e8e\u4f4e\u4e8e Hadoop 2.8.x \u7248\u672c\u63a5\u53e3\u7684\u517c\u5bb9\u6027\uff0c\u5982 CallerContext \u53ca FsServerDefaults \u3002 \u4f18\u5316 JindoCommitter \u6027\u80fd\u3002 \u4f18\u5316\u5185\u5b58\u4e0d\u8db3\u65f6\u7684\u9884\u8bfb\u7b97\u6cd5\uff0c\u56de\u6536\u4e0d\u4f7f\u7528\u7684\u5185\u5b58\u3002 \u4f18\u5316 plugin \u52a0\u8f7d\u65b9\u5f0f\uff0c fs.jdo.plugin.dir \u652f\u6301\u4ee5\u591a\u8def\u5f84\u52a0\u8f7d\uff0c\u5982 /dir1,/dir2 \u4fee\u590d list \u5bf9\u8c61\u5b58\u50a8\u65f6\uff0c\u8def\u5f84\u4e2d\u5e26\u6709 // \uff0c\u53ef\u80fd\u4f1alist\u51fa\u81ea\u8eab\u7684\u95ee\u9898 \u4fee\u590d jindo-dependence-shaded.jar \u5b89\u5168\u6027\u95ee\u9898\uff0c\u53bb\u9664 log4j \u53ca apache commons text \u4f9d\u8d56 \u4fee\u590d nextarch classifier \u8dd1 hadoop-compat-bench \u82e5\u5e72\u95ee\u9898\u3002 \u4fee\u590d nextarch classifier \u5bf9 RootPolicy \u652f\u6301\u95ee\u9898\u3002 6.5.0\uff0c2024-07-12 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.5.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.5.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d\u5076\u73b0 Signer V4 \u7b7e\u540d\u95ee\u9898\uff0c\u5c06\u7b7e\u540d\u65f6\u95f4\u6233\u8f6c\u6362\u51fd\u6570 localtime\u3001gmtime \u66ff\u6362\u4e3a localtime_r\u3001gmtime_r \u907f\u514d\u5e76\u53d1\u95ee\u9898\u3002 \u4fee\u590d libjindosdk_c.so \u8bbf\u95ee OSS-HDFS \u5076\u73b0 crash \u95ee\u9898\uff0c\u53ca\u91cd\u8bd5\u5931\u8d25\u95ee\u9898\u3002 jindo-core.jar nextarch classifer \u652f\u6301\u65b0\u7248\u5185\u6838\u3002 \u4f18\u5316\u5185\u5b58\u4e0d\u8db3\u65f6\u7684\u9884\u8bfb\u7b97\u6cd5\uff0c\u9632\u6b62\u5185\u5b58\u8fc7\u5ea6\u62a2\u5360\u3002 commiter \u652f\u6301 setXAttr\u3002 6.3.5\uff0c2024-06-28 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.3.5 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.3.5 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d\u5076\u73b0 Signer V4 \u7b7e\u540d\u95ee\u9898\uff0c\u5c06\u7b7e\u540d\u65f6\u95f4\u6233\u8f6c\u6362\u51fd\u6570 localtime\u3001gmtime \u66ff\u6362\u4e3a localtime_r\u3001gmtime_r \u907f\u514d\u5e76\u53d1\u95ee\u9898\u3002 \u4fee\u590d libjindosdk_c.so \u8bbf\u95ee OSS-HDFS \u5076\u73b0 crash \u95ee\u9898\uff0c\u53ca\u91cd\u8bd5\u5931\u8d25\u95ee\u9898\u3002 6.4.0\uff0c2024-05-16 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.4.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.4.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 JindoSDK \u652f\u6301 CallerContext\u3002 JindoSDK \u652f\u6301\u5199\u65f6 flush\u3002 \u4fee\u590d concat \u95ee\u9898\u3002 \u652f\u6301 RDMA \u8bbf\u95ee JindoCache\u3002 \u4f18\u5316\u4e86\u8bfb AppendObject \u5bf9\u8c61\u7684\u6027\u80fd\u3002 6.1.7\uff0c2024-04-12 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.1.7 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.1.7 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d\u82e5\u5e72\u95ee\u9898 JindoSDK \u652f\u6301 JindoCache + JindoAuth \u8bbf\u95ee DLF\u3002 6.3.4\uff0c2024-04-11 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.3.4 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.3.4 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 JindoFuse \u652f\u6301\u6307\u5b9a metrics_ip\u3001metrics_port \u6307\u5b9a promethues \u76d1\u542c ip \u548c \u7aef\u53e3\u3002 \u4fee\u590d JindoOssFileSystem \u7684 Delegation Token Renew \u673a\u5236\u3002 \u4fee\u590d fs.accessPolicies.discovery \u672b\u5c3e\u4e0d\u5e26 / \u65f6\uff0cgetTrashRoot \u4e3a\u7a7a\u62a5\u9519\uff08\u4ec5\u5f71\u54cd 6.3.3 \u7248\u672c\uff09\u3002 \u4fee\u590d listStatusIterator \u4e0d\u652f\u6301 ListObjectV2\u3002\u5373\u914d\u7f6e fs.oss.list.type \u4e3a 2 \uff0c\u53ef\u80fd\u5bfc\u81f4 listStatusIterator \u6b7b\u5faa\u73af\uff08\u9ed8\u8ba4\u914d\u7f6e\u4e0d\u5f71\u54cd\uff09\u3002 6.3.3\uff0c2024-03-20 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.3.3 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.3.3 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u4fee\u590d JindoFuse \u521b\u5efa symlink \u903b\u8f91\uff0c\u652f\u6301 csi plugin \u573a\u666f\u4f7f\u7528 \u652f\u6301\u901a\u8fc7\u8bbe\u7f6e\u7684\u53cd\u5411\u4ee3\u7406\u8bbf\u95ee\u5bf9\u8c61\u5b58\u50a8 6.3.2\uff0c2024-02-26 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.3.2 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.3.2 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u4fee\u590d JindoFuse \u8bfb\u53d6 symlink \u903b\u8f91\uff0csymlink target \u6539\u4e3a\u76f8\u5bf9\u8def\u5f84\u4ee5\u652f\u6301 csi plugin \u4fee\u590d JindoCache \u5bf9 fs.oss.signer.version \u7684\u652f\u6301 \u4fee\u590d jindo CLI \u53c2\u6570 -version 6.3.1\uff0c2024-02-21 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.3.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.3.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u4fee\u590d\u5728 OSS-HDFS \u6587\u4ef6\u4e0a\u8fdb\u884c\u53cd\u5411 seek \u5bfc\u81f4\u9884\u8bfb\u5931\u6548\u7684\u95ee\u9898 \u4fee\u590d\u5728\u7531\u968f\u673a\u5199\u4ea7\u751f\u7684 OSS-HDFS \u6587\u4ef6\u4e0a pread \u9884\u8bfb\u9519\u8bef\u7684\u95ee\u9898 \u4fee\u590d getRealPath \u63a5\u53e3 Root Policy \u8def\u5f84\u6539\u5199\u652f\u6301\u6839\u636e\u5b50\u8def\u5f84\u533a\u5206\u7684\u4e00\u5bf9\u591a\u66ff\u6362 6.3.0\uff0c2024-01-31 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.3.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.3.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u652f\u6301\u4f7f\u7528 V4 \u7b7e\u540d\u8bbf\u95ee OSS/OSS-HDFS \u63d0\u4f9b Python SDK\uff0c\u652f\u6301\u8bbf\u95ee OSS \u548c OSS-HDFS \u4fee\u590d Jindo \u547d\u4ee4\u884c\u5220\u9664 OSS-HDFS \u76ee\u5f55\u65f6\u53d1\u8d77\u8fc7\u591a\u8bf7\u6c42\u7684\u95ee\u9898 JindoTable \u5206\u5c42\u5b58\u50a8\u5de5\u5177\u652f\u6301\u5f52\u6863\u6570\u636e\u76f4\u63a5\u8f6c\u51b7\u5f52\u6863 JindoTable MoveTo \u5de5\u5177\u652f\u6301 Iceberg \u8868\u6574\u4f53\u8fc1\u79fb \u4fee\u590d JindoFuse \u8bfb\u53d6 symlink \u5931\u8d25 6.1.6\uff0c2023-12-28 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.1.6 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.1.6 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u652f\u6301\u4f7f\u7528 V4 \u7b7e\u540d\u8bbf\u95ee OSS/OSS-HDFS 6.2.0\uff0c2023-12-22 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.2.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.2.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u65b0\u589e\u652f\u6301\u5f02\u6b65\u7684 CSDK libjindosdk_c.so \uff0c\u8001\u7248\u672c libjindo-csdk.so \u9010\u6e10\u5f03\u7528\u3002 \u4fee\u590d\u8def\u5f84\u5408\u6cd5\u6027\u68c0\u67e5\uff0c\u652f\u6301\u5e26 * \u7684\u8def\u5f84 \u4f18\u5316 JindoFuse \u8bbf\u95ee oss \u65f6\u6709\u975e\u6cd5\u540d\u79f0\u7684\u8def\u5f84\u65f6\u7684\u62a5\u9519\u4fe1\u606f Jindo \u547d\u4ee4\u884c\u652f\u6301 OSS-HDFS setStoragePolicy \u89e3\u51bb\u76f8\u5173\u7b56\u7565 \u91c7\u7528\u65b0\u7684\u9884\u8bfb\u7b97\u6cd5\uff0c\u987a\u5e8f\u8bfb\u5927\u6587\u4ef6\u7684\u6027\u80fd\u63d0\u9ad8 \u65b0\u9884\u8bfb\u7b97\u6cd5\u53ef\u914d\u7f6e\u53c2\u6570\u5982\u4e0b - fs.oss.read.readahead.prefetcher.version = default \u9884\u8bfb\u5b9e\u73b0\u5207\u6362\u5f00\u5173 > - \u53ef\u9009\u503c\uff1a legacy \u539f\u9884\u8bfb\u7b97\u6cd5, default \u65b0\u9884\u8bfb\u7b97\u6cd5 - fs.oss.read.readahead.prefetch.size.max = 268435456 \u9884\u8bfb\u6700\u5927\u957f\u5ea6\uff08\u5355\u4f4d\uff1abyte\uff09 \u65b0\u9884\u8bfb\u7b97\u6cd5\u53ef\u80fd\u4f7f\u7528\u66f4\u591a\u5185\u5b58\u3002\u82e5\u914d\u7f6e\u4e86\u65b0\u9884\u8bfb\u7b97\u6cd5\u540e\u53d1\u751f\u6027\u80fd\u4e0b\u964d\uff0c\u53ef\u80fd\u662f\u7531\u4e8e\u5185\u5b58\u6c60\u5bb9\u91cf\u4e0d\u8db3\u5bfc\u81f4\u9884\u8bfb\u7684\u5757\u5728\u88ab\u8bbf\u95ee\u5230\u4e4b\u524d\u5c31\u88ab\u63d0\u524d\u9010\u51fa\u3002\u4e3a\u4e86\u907f\u514d\u8be5\u60c5\u51b5\u53d1\u751f\uff0c\u53ef\u4ee5\u8003\u8651\u7f29\u51cf\u6700\u5927\u9884\u8bfb\u957f\u5ea6\uff0c\u6216\u5141\u8bb8\u9884\u8bfb\u4f7f\u7528\u66f4\u591a\u5185\u5b58\u3002\u5185\u5b58\u6c60\u914d\u7f6e\u76f8\u5173\u53c2\u6570\u5982\u4e0b - fs.oss.memory.buffer.size.max.mb = 6144 \u5185\u5b58\u6c60\u603b\u5bb9\u91cf\uff08\u5355\u4f4d\uff1aMB\uff09 - fs.oss.memory.buffer.size.watermark = 0.3 \u5185\u5b58\u6c60\u7528\u4e8e\u9884\u8bfb\u7684\u5bb9\u91cf\u6bd4\u4f8b \u4fee\u590d Hive \u4f7f\u7528 LDAP + Kerberos + OSS-HDFS \u65f6\u7684\u8ba4\u8bc1\u95ee\u9898 6.1.5\uff0c2023-12-19 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.1.5 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.1.5 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 OSS-HDFS \u8bfb\u5199 IO \u4f18\u5316 6.1.4\uff0c2023-12-14 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.1.4 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.1.4 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u652f\u6301 HiveServer2 \u5728 Kerberos & LDAP \u53cc\u8ba4\u8bc1\u573a\u666f\u4e0b\u4f7f\u7528 JindoSDK Java Jni \u5ffd\u7565 AddressSanitizer \u8bef\u62a5 6.1.3\uff0c2023-12-01 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.1.3 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.1.3 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u4fee\u590d dls:// \u524d\u7f00\u8bbf\u95ee jindofs \u7684\u82e5\u5e72\u95ee\u9898\u3002 6.1.2\uff0c2023-11-21 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.1.2 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.1.2 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u901a\u8fc7\u914d\u7f6e fs.oss.list.type=2 \u652f\u6301ListObjectV2\uff0c\u4ee5\u907f\u514d\u8bbf\u95ee\u5f00\u542f\u4e86\u591a\u7248\u672cbucket\u65f6\u7684\u64cd\u4f5c\u8d85\u65f6\u95ee\u9898\u3002 \u652f\u6301\u4f7f\u7528 dls:// \u524d\u7f00\u8bbf\u95ee jindofs\u3002 \u4fee\u590d\u8bfb\u6d41 close \u4f1a\u5361\u4f4f\u7684\u95ee\u9898\u3002 \u4fee\u590d\u5199\u6d41\u51fa\u73b0\u4e00\u6b21 write \u5931\u8d25\u540e\uff0c\u540e\u7eed write \u51fa\u73b0\u7684 NPE \u95ee\u9898\u3002 6.1.1\uff0c2023-10-20 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.1.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.1.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u652f\u6301\u901a\u8fc7\u914d\u7f6e fs.oss.legacy.version \u4e3a 3.8\uff0c\u517c\u5bb9 JindoSDK 3.8.x \u914d\u7f6e 6.1.0\uff0c2023-09-28 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.1.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.1.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 JindoSDK \u652f\u6301 FileSystem close \u540e\u4ecd\u652f\u6301\u4f7f\u7528\u8bfb\u5199\u6d41\u8fdb\u884c\u8bfb\u5199 6.0.1\uff0c2023-08-18 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.0.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.0.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 JindoSDK \u652f\u6301 isFileClosed \u65b9\u6cd5\uff0c\u7528\u4e8e\u5224\u65ad\u8bbf\u95ee OSS-HDFS \u7684\u8bfb\u5199\u6d41\u662f\u5426\u5df2\u5173\u95ed\u3002 JindoSDK \u5bf9 flush \u964d\u9891\u505a\u4e86\u4f18\u5316\uff0c\u5728\u964d\u9891\u914d\u7f6e\u751f\u6548\u65f6\uff0c\u5bf9\u7b2c\u4e00\u6b21\u6709\u6570\u636e\u7684flush\u64cd\u4f5c\u4fdd\u8bc1\u6267\u884c\uff0c\u9632\u6b62flush\u51fa\u4e00\u4e2a\u7a7a\u6587\u4ef6\u3002\u964d\u9891\u914d\u7f6e\uff0c\u53ef\u53c2\u8003 Flume \u4f7f\u7528 JindoSDK \u5199\u5165 OSS 6.0.0\uff0c2023-08-15 \u00a7 \u7248\u672c\u6982\u8981 \u00a7 \u53d1\u5e03 JindoSDK 6.0.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd \u4ecb\u7ecd \u00a7 JindoSDK \u66f4\u65b0 6.0.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \uff0cJindoSDK \u5408\u5e76 jindo-mapreduce (JindoCommitterFactory\u7b49) \u5230 jindo-sdk JindoSDK \u65b0\u589e\u82e5\u5e72\u4fee\u590d\uff0c\u5305\u542b flush \u964d\u9891\uff0c\u4f18\u5316 GC \u56de\u6536\uff0c\u9002\u914d OSS-HDFS \u56de\u6536\u7ad9\u7b49 \u591a\u5e73\u53f0\u652f\u6301 MACOS \u548c \u4e3b\u6d41 linux \u573a\u666f\uff0c\u5176\u4e2d CentOS 6, Ubuntu22, \u963f\u91cc\u4e91\u501a\u5929\u673a\u578b(ARM)\uff0cMacOS (Intel/M1) \u9700\u8981\u4f7f\u7528\u7279\u5b9a\u7684\u6269\u5c55\u5305 \u5347\u7ea7 g++ \u7f16\u8bd1\u5668\u5230 10.4\uff0c\u4f7f\u7528 c++20 \u6807\u51c6 \u652f\u6301 yalangtinglibs \u8bbf\u95ee OSS/OSS-HDFS","title":"\u5386\u53f2\u7248\u672c"},{"location":"releases/#jindosdk","text":"","title":"JindoSDK \u7248\u672c\u8bb0\u5f55"},{"location":"releases/#6732024-11-01","text":"","title":"6.7.3\uff0c2024-11-01"},{"location":"releases/#_1","text":"\u53d1\u5e03 JindoSDK 6.7.3 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_2","text":"JindoSDK \u66f4\u65b0 6.7.3 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u6253\u5305\u65f6\u5e26\u5165 jindoauth plugin\uff0c\u652f\u6301\u5347\u7ea7 JindoSDK \u540e\u4ecd\u517c\u5bb9\u8001\u7248\u672c EMR-Ranger\uff0c\u5177\u4f53\u53c2\u89c1 \u5347\u7ea7\u6587\u6863 \u3002 nextarch classifier \u4fee\u590d hudi \u8bfb\u5199 log \u65f6\uff0c\u5bf9 AlreadyBeingCreatedException \u5f02\u5e38\u5904\u7406\u3002 nextarch classifier \u4fee\u590d\u5728\u4f7f\u7528 http \u8bf7\u6c42\u66f4\u65b0 STS \u65f6\uff0c\u5076\u73b0\u5361\u4f4f\u95ee\u9898\u3002 nextarch classifier \u4fee\u590d\u5f00\u542f Ranger \u540e\u6301\u7eed\u5199\u5165\u4e00\u4e2a OSS-HDFS \u6587\u4ef6\u8d85\u8fc7 1 \u5c0f\u65f6\u540e\uff0c\u6587\u4ef6\u65e0\u6cd5\u7ee7\u7eed\u5199\u5165\u7684\u95ee\u9898\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6412024-10-23","text":"","title":"6.4.1\uff0c2024-10-23"},{"location":"releases/#_3","text":"\u53d1\u5e03 JindoSDK 6.4.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_4","text":"JindoSDK \u66f4\u65b0 6.4.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d\u5076\u73b0 Signer V4 \u7b7e\u540d\u95ee\u9898\uff0c\u5c06\u7b7e\u540d\u65f6\u95f4\u6233\u8f6c\u6362\u51fd\u6570 localtime\u3001gmtime \u66ff\u6362\u4e3a localtime_r\u3001gmtime_r \u907f\u514d\u5e76\u53d1\u95ee\u9898\u3002 \u4fee\u590d libjindosdk_c.so \u8bbf\u95ee OSS-HDFS \u5076\u73b0 crash \u95ee\u9898\uff0c\u53ca\u91cd\u8bd5\u5931\u8d25\u95ee\u9898\u3002 \u4fee\u590d hudi \u8bfb\u5199 log \u65f6\uff0c\u5bf9 AlreadyBeingCreatedException \u5f02\u5e38\u5904\u7406\u3002 \u4fee\u590d JindoOssFileSystem \u7684 Delegation Token Renew \u673a\u5236\u3002 \u4fee\u590d JindoCommitter \u8bbf\u95ee OSS-HDFS \u672a\u6e05\u7406 task \u6b8b\u7559\u7684\u4e34\u65f6\u76ee\u5f55\u3002 \u63d0\u5347 JindoSDK \u5bf9\u4e8e\u4f4e\u4e8e Hadoop 2.8.x \u7248\u672c\u63a5\u53e3\u7684\u517c\u5bb9\u6027\uff0c\u5982 CallerContext \u53ca FsServerDefaults \u3002 \u4f18\u5316 classloader \u52a0\u8f7d\uff0c\u5728\u591a\u4e2a classloader \u52a0\u8f7d jindosdk \u65f6\u7684 so \u6b8b\u7559\u95ee\u9898\uff0c\u5e76\u4e14\u5728\u914d\u7f6e\u4e86 java.io.tmpdir \u65f6\uff0c\u9ed8\u8ba4\u89e3\u538b\u81f3\u914d\u7f6e\u76ee\u5f55\u4e0b\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6722024-10-21","text":"","title":"6.7.2\uff0c2024-10-21"},{"location":"releases/#_5","text":"\u53d1\u5e03 JindoSDK 6.7.2 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_6","text":"JindoSDK \u66f4\u65b0 6.7.2 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u4f18\u5316 metrics \u7edf\u8ba1\u4fe1\u606f\u3002 nextarch classifier \u4fee\u590d\u5728\u5199 OSS-HDFS \u573a\u666f flush \u540e\u51fa\u73b0\u5c0f\u5757\u7684\u95ee\u9898\u3002 nextarch classifier \u4fee\u590d\u540e\u53f0 metrics \u7ebf\u7a0b\u5f15\u53d1\u7684\u5076\u73b0 crash \u95ee\u9898\u3002 Pyjindo \u4fee\u590d 6.7.x \u7248\u672c\u62a5 invalid pointer \u95ee\u9898\uff0c6.7.0 \u53ca 6.7.1 \u7248\u672c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u73af\u5883\u53d8\u91cf JINDO_STAT_MEMORY=0 \u7ed5\u8fc7\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6712024-10-15","text":"","title":"6.7.1\uff0c2024-10-15"},{"location":"releases/#_7","text":"\u53d1\u5e03 JindoSDK 6.7.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_8","text":"JindoSDK \u66f4\u65b0 6.7.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u4f18\u5316\u9ed8\u8ba4\u5e76\u53d1\u4e0b\u7684\u5199\u6027\u80fd\u3002 \u4fee\u590d JindoCommitter \u8bbf\u95ee OSS-HDFS \u672a\u6e05\u7406 task \u6b8b\u7559\u7684\u4e34\u65f6\u76ee\u5f55\u3002 jindo-fuse \u4fee\u590d\u5728 s3 scheme \u4e0a\u8ffd\u52a0\u5199\u7684\u652f\u6301\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6702024-09-29","text":"","title":"6.7.0\uff0c2024-09-29"},{"location":"releases/#_9","text":"\u53d1\u5e03 JindoSDK 6.7.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_10","text":"JindoSDK \u66f4\u65b0 6.7.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u652f\u6301\u7edf\u4e00\u7ba1\u7406\u5185\u5b58\uff0c\u4f7f\u7528 fs.jdo.memory.pool.size.max.mb \u914d\u7f6e\u6700\u5927\u4f7f\u7528\u5185\u5b58\uff0c\u5e76\u652f\u6301\u5185\u5b58\u5b9e\u65f6\u4f7f\u7528 metrics\u3002 nextarch classifier \u652f\u6301 io buffer \u5185\u5b58\u4f7f\u7528\u63a7\u5236\uff0c\u4f7f\u7528 fs.jdo.memory.io.buffer.size.max.ratio \u914d\u7f6e\u6700\u5927\u4f7f\u7528\u5185\u5b58\u6c60\u7684\u6bd4\u4f8b\u3002 nextarch classifier \u652f\u6301 Golang SDK\u3002 nextarch classifier \u652f\u6301 OSS \u5bf9\u8c61\u5b58\u50a8\u6df1\u51b7\u5f52\u6863\u3002 nextarch classifier \u4f18\u5316 pread \u63a5\u53e3\uff0c\u5185\u5b58\u96f6\u62f7\u8d1d\u3002 nextarch classifier \u4f18\u5316 sendfile \u5728\u5199\u5c0f\u6587\u4ef6\u573a\u666f\u7684\u4f7f\u7528\u3002 nextarch classifier \u4f18\u5316 metrics \u6846\u67b6\uff0c\u652f\u6301\u6bcf\u6b21\u8f93\u51fa\u5230\u5355\u72ec\u6587\u4ef6\uff0c\u65b9\u4fbf\u91c7\u96c6\u3002 nextarch classifier \u4f18\u5316 distjob \u6846\u4e2d\uff0c\u6392\u9664 log4j \u76f8\u5173\u4f9d\u8d56\u3002 nextarch classifier \u4f18\u5316 classloader \u52a0\u8f7d\uff0c\u5728\u591a\u4e2a classloader \u52a0\u8f7d jindosdk \u65f6\u7684 so \u6b8b\u7559\u95ee\u9898\uff0c\u5e76\u4e14\u5728\u914d\u7f6e\u4e86 java.io.tmpdir \u65f6\uff0c\u9ed8\u8ba4\u89e3\u538b\u81f3\u914d\u7f6e\u76ee\u5f55\u4e0b\u3002 nextarch classifier \u4fee\u590d OSS-HDFS StoragePolicy \u76f8\u5173\u63a5\u53e3\u3002 nextarch classifier \u4fee\u590d OSS \u5bf9\u8c61\u5b58\u50a8\uff0cListDirectory \u8fd4\u56de\u7ed3\u679c\u7684 Mtime \u5355\u4f4d\u3002 nextarch classifier \u4fee\u590d JindoCommitter \u5728 Cleanup \u9636\u6bb5\uff0c\u4e0d\u5e94\u629b\u51fa\u5f02\u5e38\u3002 nextarch classifier \u4fee\u590d DLF \u76f8\u5173 CredentialProvider \u4f7f\u7528 MagicCommitter \u65f6\u7684\u6743\u9650\u95ee\u9898\u3002 nextarch classifier \u4fee\u590d JindoCache \u7684\u82e5\u5e72\u95ee\u9898\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6632024-09-14","text":"","title":"6.6.3\uff0c2024-09-14"},{"location":"releases/#_11","text":"\u53d1\u5e03 JindoSDK 6.6.3 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_12","text":"JindoSDK \u66f4\u65b0 6.6.3 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u4fee\u590d\u5199\u573a\u666f\u5c0f\u6982\u7387\u56e0 sendfile \u51fa\u73b0\u9519\u8bef\u672a\u91cd\u8bd5\uff0c\u5bfc\u81f4\u6587\u4ef6 close \u65f6\u62a5\u5931\u8d25\u7684\u60c5\u51b5\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6622024-09-12","text":"","title":"6.6.2\uff0c2024-09-12"},{"location":"releases/#_13","text":"\u53d1\u5e03 JindoSDK 6.6.2 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_14","text":"JindoSDK \u66f4\u65b0 6.6.2 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u8bbf\u95ee OSS \u652f\u6301\u6df1\u51b7\u5f52\u6863 nextarch classifier \u4fee\u590d DLF \u76f8\u5173 CredentialProvider \u4f7f\u7528 MagicCommitter \u65f6\u7684\u6743\u9650\u95ee\u9898\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6612024-09-03","text":"","title":"6.6.1\uff0c2024-09-03"},{"location":"releases/#_15","text":"\u53d1\u5e03 JindoSDK 6.6.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_16","text":"JindoSDK \u66f4\u65b0 6.6.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u4f18\u5316\u5728\u591a\u4e2a classloader \u52a0\u8f7d jindosdk \u65f6\u7684 so \u6b8b\u7559\u95ee\u9898\uff0c\u5e76\u4e14\u5728\u914d\u7f6e\u4e86 java.io.tmpdir \u65f6\uff0c\u9ed8\u8ba4\u89e3\u538b\u81f3\u914d\u7f6e\u76ee\u5f55\u4e0b\u3002 nextarch classifier \u4fee\u590d\u8bbf\u95ee JindoCache \u7684\u82e5\u5e72\u95ee\u9898\u3002 nextarch classifier \u4f18\u5316 metrics \u6846\u67b6\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6602024-08-25","text":"","title":"6.6.0\uff0c2024-08-25"},{"location":"releases/#_17","text":"\u53d1\u5e03 JindoSDK 6.6.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_18","text":"JindoSDK \u66f4\u65b0 6.6.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 nextarch classifier \u4f18\u5316 io \u6027\u80fd\uff0c\u4f7f\u7528\u534f\u7a0b\u5f02\u6b65\u5316\uff0c\u76f8\u540c\u914d\u7f6e\u4e0b\u652f\u6301\u66f4\u9ad8\u5e76\u53d1\u3002\u5199\u573a\u666f\u4f7f\u7528 sendfile \u5b9e\u73b0\u96f6\u62f7\u8d1d\u4f18\u5316\uff0c\u8282\u7701\u5185\u5b58\uff0c\u5e76\u63d0\u5347\u6027\u80fd\u3002 nextarch classifier \u652f\u6301\u5355\u72ec\u6307\u5b9a io \u8d85\u65f6\u65f6\u95f4\u3002 nextarch classifier \u652f\u6301 metrics \u6846\u67b6\u3002 \u4fee\u590d nextarch classifier \u5bf9 JindoCache \u652f\u6301\u95ee\u9898\u3002 jindo-fuse \u9ed8\u8ba4\u6539\u4e3a\u4f7f\u7528 nextarch \u5b9e\u73b0\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6542024-08-20","text":"","title":"6.5.4\uff0c2024-08-20"},{"location":"releases/#_19","text":"\u53d1\u5e03 JindoSDK 6.5.4 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_20","text":"JindoSDK \u66f4\u65b0 6.5.4 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4f18\u5316 flink connector \u4f9d\u8d56\uff0c\u5bf9\u90e8\u5206\u4f9d\u8d56\u8fdb\u884c shade\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6532024-08-15","text":"","title":"6.5.3\uff0c2024-08-15"},{"location":"releases/#_21","text":"\u53d1\u5e03 JindoSDK 6.5.3 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_22","text":"JindoSDK \u66f4\u65b0 6.5.3 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d JindoOssFileSystem \u7684 Delegation Token Renew \u673a\u5236\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6522024-08-12","text":"","title":"6.5.2\uff0c2024-08-12"},{"location":"releases/#_23","text":"\u53d1\u5e03 JindoSDK 6.5.2 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_24","text":"JindoSDK \u66f4\u65b0 6.5.2 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d getfacl \u7f3a\u5c11 entries \u95ee\u9898\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6512024-08-05","text":"","title":"6.5.1\uff0c2024-08-05"},{"location":"releases/#_25","text":"\u53d1\u5e03 JindoSDK 6.5.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_26","text":"JindoSDK \u66f4\u65b0 6.5.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u63d0\u5347 JindoSDK \u5bf9\u4e8e\u4f4e\u4e8e Hadoop 2.8.x \u7248\u672c\u63a5\u53e3\u7684\u517c\u5bb9\u6027\uff0c\u5982 CallerContext \u53ca FsServerDefaults \u3002 \u4f18\u5316 JindoCommitter \u6027\u80fd\u3002 \u4f18\u5316\u5185\u5b58\u4e0d\u8db3\u65f6\u7684\u9884\u8bfb\u7b97\u6cd5\uff0c\u56de\u6536\u4e0d\u4f7f\u7528\u7684\u5185\u5b58\u3002 \u4f18\u5316 plugin \u52a0\u8f7d\u65b9\u5f0f\uff0c fs.jdo.plugin.dir \u652f\u6301\u4ee5\u591a\u8def\u5f84\u52a0\u8f7d\uff0c\u5982 /dir1,/dir2 \u4fee\u590d list \u5bf9\u8c61\u5b58\u50a8\u65f6\uff0c\u8def\u5f84\u4e2d\u5e26\u6709 // \uff0c\u53ef\u80fd\u4f1alist\u51fa\u81ea\u8eab\u7684\u95ee\u9898 \u4fee\u590d jindo-dependence-shaded.jar \u5b89\u5168\u6027\u95ee\u9898\uff0c\u53bb\u9664 log4j \u53ca apache commons text \u4f9d\u8d56 \u4fee\u590d nextarch classifier \u8dd1 hadoop-compat-bench \u82e5\u5e72\u95ee\u9898\u3002 \u4fee\u590d nextarch classifier \u5bf9 RootPolicy \u652f\u6301\u95ee\u9898\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6502024-07-12","text":"","title":"6.5.0\uff0c2024-07-12"},{"location":"releases/#_27","text":"\u53d1\u5e03 JindoSDK 6.5.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_28","text":"JindoSDK \u66f4\u65b0 6.5.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d\u5076\u73b0 Signer V4 \u7b7e\u540d\u95ee\u9898\uff0c\u5c06\u7b7e\u540d\u65f6\u95f4\u6233\u8f6c\u6362\u51fd\u6570 localtime\u3001gmtime \u66ff\u6362\u4e3a localtime_r\u3001gmtime_r \u907f\u514d\u5e76\u53d1\u95ee\u9898\u3002 \u4fee\u590d libjindosdk_c.so \u8bbf\u95ee OSS-HDFS \u5076\u73b0 crash \u95ee\u9898\uff0c\u53ca\u91cd\u8bd5\u5931\u8d25\u95ee\u9898\u3002 jindo-core.jar nextarch classifer \u652f\u6301\u65b0\u7248\u5185\u6838\u3002 \u4f18\u5316\u5185\u5b58\u4e0d\u8db3\u65f6\u7684\u9884\u8bfb\u7b97\u6cd5\uff0c\u9632\u6b62\u5185\u5b58\u8fc7\u5ea6\u62a2\u5360\u3002 commiter \u652f\u6301 setXAttr\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6352024-06-28","text":"","title":"6.3.5\uff0c2024-06-28"},{"location":"releases/#_29","text":"\u53d1\u5e03 JindoSDK 6.3.5 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_30","text":"JindoSDK \u66f4\u65b0 6.3.5 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d\u5076\u73b0 Signer V4 \u7b7e\u540d\u95ee\u9898\uff0c\u5c06\u7b7e\u540d\u65f6\u95f4\u6233\u8f6c\u6362\u51fd\u6570 localtime\u3001gmtime \u66ff\u6362\u4e3a localtime_r\u3001gmtime_r \u907f\u514d\u5e76\u53d1\u95ee\u9898\u3002 \u4fee\u590d libjindosdk_c.so \u8bbf\u95ee OSS-HDFS \u5076\u73b0 crash \u95ee\u9898\uff0c\u53ca\u91cd\u8bd5\u5931\u8d25\u95ee\u9898\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6402024-05-16","text":"","title":"6.4.0\uff0c2024-05-16"},{"location":"releases/#_31","text":"\u53d1\u5e03 JindoSDK 6.4.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_32","text":"JindoSDK \u66f4\u65b0 6.4.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 JindoSDK \u652f\u6301 CallerContext\u3002 JindoSDK \u652f\u6301\u5199\u65f6 flush\u3002 \u4fee\u590d concat \u95ee\u9898\u3002 \u652f\u6301 RDMA \u8bbf\u95ee JindoCache\u3002 \u4f18\u5316\u4e86\u8bfb AppendObject \u5bf9\u8c61\u7684\u6027\u80fd\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6172024-04-12","text":"","title":"6.1.7\uff0c2024-04-12"},{"location":"releases/#_33","text":"\u53d1\u5e03 JindoSDK 6.1.7 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_34","text":"JindoSDK \u66f4\u65b0 6.1.7 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 \u4fee\u590d\u82e5\u5e72\u95ee\u9898 JindoSDK \u652f\u6301 JindoCache + JindoAuth \u8bbf\u95ee DLF\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6342024-04-11","text":"","title":"6.3.4\uff0c2024-04-11"},{"location":"releases/#_35","text":"\u53d1\u5e03 JindoSDK 6.3.4 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_36","text":"JindoSDK \u66f4\u65b0 6.3.4 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u3002 JindoFuse \u652f\u6301\u6307\u5b9a metrics_ip\u3001metrics_port \u6307\u5b9a promethues \u76d1\u542c ip \u548c \u7aef\u53e3\u3002 \u4fee\u590d JindoOssFileSystem \u7684 Delegation Token Renew \u673a\u5236\u3002 \u4fee\u590d fs.accessPolicies.discovery \u672b\u5c3e\u4e0d\u5e26 / \u65f6\uff0cgetTrashRoot \u4e3a\u7a7a\u62a5\u9519\uff08\u4ec5\u5f71\u54cd 6.3.3 \u7248\u672c\uff09\u3002 \u4fee\u590d listStatusIterator \u4e0d\u652f\u6301 ListObjectV2\u3002\u5373\u914d\u7f6e fs.oss.list.type \u4e3a 2 \uff0c\u53ef\u80fd\u5bfc\u81f4 listStatusIterator \u6b7b\u5faa\u73af\uff08\u9ed8\u8ba4\u914d\u7f6e\u4e0d\u5f71\u54cd\uff09\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6332024-03-20","text":"","title":"6.3.3\uff0c2024-03-20"},{"location":"releases/#_37","text":"\u53d1\u5e03 JindoSDK 6.3.3 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_38","text":"JindoSDK \u66f4\u65b0 6.3.3 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u4fee\u590d JindoFuse \u521b\u5efa symlink \u903b\u8f91\uff0c\u652f\u6301 csi plugin \u573a\u666f\u4f7f\u7528 \u652f\u6301\u901a\u8fc7\u8bbe\u7f6e\u7684\u53cd\u5411\u4ee3\u7406\u8bbf\u95ee\u5bf9\u8c61\u5b58\u50a8","title":"\u4ecb\u7ecd"},{"location":"releases/#6322024-02-26","text":"","title":"6.3.2\uff0c2024-02-26"},{"location":"releases/#_39","text":"\u53d1\u5e03 JindoSDK 6.3.2 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_40","text":"JindoSDK \u66f4\u65b0 6.3.2 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u4fee\u590d JindoFuse \u8bfb\u53d6 symlink \u903b\u8f91\uff0csymlink target \u6539\u4e3a\u76f8\u5bf9\u8def\u5f84\u4ee5\u652f\u6301 csi plugin \u4fee\u590d JindoCache \u5bf9 fs.oss.signer.version \u7684\u652f\u6301 \u4fee\u590d jindo CLI \u53c2\u6570 -version","title":"\u4ecb\u7ecd"},{"location":"releases/#6312024-02-21","text":"","title":"6.3.1\uff0c2024-02-21"},{"location":"releases/#_41","text":"\u53d1\u5e03 JindoSDK 6.3.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_42","text":"JindoSDK \u66f4\u65b0 6.3.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u4fee\u590d\u5728 OSS-HDFS \u6587\u4ef6\u4e0a\u8fdb\u884c\u53cd\u5411 seek \u5bfc\u81f4\u9884\u8bfb\u5931\u6548\u7684\u95ee\u9898 \u4fee\u590d\u5728\u7531\u968f\u673a\u5199\u4ea7\u751f\u7684 OSS-HDFS \u6587\u4ef6\u4e0a pread \u9884\u8bfb\u9519\u8bef\u7684\u95ee\u9898 \u4fee\u590d getRealPath \u63a5\u53e3 Root Policy \u8def\u5f84\u6539\u5199\u652f\u6301\u6839\u636e\u5b50\u8def\u5f84\u533a\u5206\u7684\u4e00\u5bf9\u591a\u66ff\u6362","title":"\u4ecb\u7ecd"},{"location":"releases/#6302024-01-31","text":"","title":"6.3.0\uff0c2024-01-31"},{"location":"releases/#_43","text":"\u53d1\u5e03 JindoSDK 6.3.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_44","text":"JindoSDK \u66f4\u65b0 6.3.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u652f\u6301\u4f7f\u7528 V4 \u7b7e\u540d\u8bbf\u95ee OSS/OSS-HDFS \u63d0\u4f9b Python SDK\uff0c\u652f\u6301\u8bbf\u95ee OSS \u548c OSS-HDFS \u4fee\u590d Jindo \u547d\u4ee4\u884c\u5220\u9664 OSS-HDFS \u76ee\u5f55\u65f6\u53d1\u8d77\u8fc7\u591a\u8bf7\u6c42\u7684\u95ee\u9898 JindoTable \u5206\u5c42\u5b58\u50a8\u5de5\u5177\u652f\u6301\u5f52\u6863\u6570\u636e\u76f4\u63a5\u8f6c\u51b7\u5f52\u6863 JindoTable MoveTo \u5de5\u5177\u652f\u6301 Iceberg \u8868\u6574\u4f53\u8fc1\u79fb \u4fee\u590d JindoFuse \u8bfb\u53d6 symlink \u5931\u8d25","title":"\u4ecb\u7ecd"},{"location":"releases/#6162023-12-28","text":"","title":"6.1.6\uff0c2023-12-28"},{"location":"releases/#_45","text":"\u53d1\u5e03 JindoSDK 6.1.6 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_46","text":"JindoSDK \u66f4\u65b0 6.1.6 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u652f\u6301\u4f7f\u7528 V4 \u7b7e\u540d\u8bbf\u95ee OSS/OSS-HDFS","title":"\u4ecb\u7ecd"},{"location":"releases/#6202023-12-22","text":"","title":"6.2.0\uff0c2023-12-22"},{"location":"releases/#_47","text":"\u53d1\u5e03 JindoSDK 6.2.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_48","text":"JindoSDK \u66f4\u65b0 6.2.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u65b0\u589e\u652f\u6301\u5f02\u6b65\u7684 CSDK libjindosdk_c.so \uff0c\u8001\u7248\u672c libjindo-csdk.so \u9010\u6e10\u5f03\u7528\u3002 \u4fee\u590d\u8def\u5f84\u5408\u6cd5\u6027\u68c0\u67e5\uff0c\u652f\u6301\u5e26 * \u7684\u8def\u5f84 \u4f18\u5316 JindoFuse \u8bbf\u95ee oss \u65f6\u6709\u975e\u6cd5\u540d\u79f0\u7684\u8def\u5f84\u65f6\u7684\u62a5\u9519\u4fe1\u606f Jindo \u547d\u4ee4\u884c\u652f\u6301 OSS-HDFS setStoragePolicy \u89e3\u51bb\u76f8\u5173\u7b56\u7565 \u91c7\u7528\u65b0\u7684\u9884\u8bfb\u7b97\u6cd5\uff0c\u987a\u5e8f\u8bfb\u5927\u6587\u4ef6\u7684\u6027\u80fd\u63d0\u9ad8 \u65b0\u9884\u8bfb\u7b97\u6cd5\u53ef\u914d\u7f6e\u53c2\u6570\u5982\u4e0b - fs.oss.read.readahead.prefetcher.version = default \u9884\u8bfb\u5b9e\u73b0\u5207\u6362\u5f00\u5173 > - \u53ef\u9009\u503c\uff1a legacy \u539f\u9884\u8bfb\u7b97\u6cd5, default \u65b0\u9884\u8bfb\u7b97\u6cd5 - fs.oss.read.readahead.prefetch.size.max = 268435456 \u9884\u8bfb\u6700\u5927\u957f\u5ea6\uff08\u5355\u4f4d\uff1abyte\uff09 \u65b0\u9884\u8bfb\u7b97\u6cd5\u53ef\u80fd\u4f7f\u7528\u66f4\u591a\u5185\u5b58\u3002\u82e5\u914d\u7f6e\u4e86\u65b0\u9884\u8bfb\u7b97\u6cd5\u540e\u53d1\u751f\u6027\u80fd\u4e0b\u964d\uff0c\u53ef\u80fd\u662f\u7531\u4e8e\u5185\u5b58\u6c60\u5bb9\u91cf\u4e0d\u8db3\u5bfc\u81f4\u9884\u8bfb\u7684\u5757\u5728\u88ab\u8bbf\u95ee\u5230\u4e4b\u524d\u5c31\u88ab\u63d0\u524d\u9010\u51fa\u3002\u4e3a\u4e86\u907f\u514d\u8be5\u60c5\u51b5\u53d1\u751f\uff0c\u53ef\u4ee5\u8003\u8651\u7f29\u51cf\u6700\u5927\u9884\u8bfb\u957f\u5ea6\uff0c\u6216\u5141\u8bb8\u9884\u8bfb\u4f7f\u7528\u66f4\u591a\u5185\u5b58\u3002\u5185\u5b58\u6c60\u914d\u7f6e\u76f8\u5173\u53c2\u6570\u5982\u4e0b - fs.oss.memory.buffer.size.max.mb = 6144 \u5185\u5b58\u6c60\u603b\u5bb9\u91cf\uff08\u5355\u4f4d\uff1aMB\uff09 - fs.oss.memory.buffer.size.watermark = 0.3 \u5185\u5b58\u6c60\u7528\u4e8e\u9884\u8bfb\u7684\u5bb9\u91cf\u6bd4\u4f8b \u4fee\u590d Hive \u4f7f\u7528 LDAP + Kerberos + OSS-HDFS \u65f6\u7684\u8ba4\u8bc1\u95ee\u9898","title":"\u4ecb\u7ecd"},{"location":"releases/#6152023-12-19","text":"","title":"6.1.5\uff0c2023-12-19"},{"location":"releases/#_49","text":"\u53d1\u5e03 JindoSDK 6.1.5 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_50","text":"JindoSDK \u66f4\u65b0 6.1.5 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 OSS-HDFS \u8bfb\u5199 IO \u4f18\u5316","title":"\u4ecb\u7ecd"},{"location":"releases/#6142023-12-14","text":"","title":"6.1.4\uff0c2023-12-14"},{"location":"releases/#_51","text":"\u53d1\u5e03 JindoSDK 6.1.4 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_52","text":"JindoSDK \u66f4\u65b0 6.1.4 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u652f\u6301 HiveServer2 \u5728 Kerberos & LDAP \u53cc\u8ba4\u8bc1\u573a\u666f\u4e0b\u4f7f\u7528 JindoSDK Java Jni \u5ffd\u7565 AddressSanitizer \u8bef\u62a5","title":"\u4ecb\u7ecd"},{"location":"releases/#6132023-12-01","text":"","title":"6.1.3\uff0c2023-12-01"},{"location":"releases/#_53","text":"\u53d1\u5e03 JindoSDK 6.1.3 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_54","text":"JindoSDK \u66f4\u65b0 6.1.3 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u4fee\u590d dls:// \u524d\u7f00\u8bbf\u95ee jindofs \u7684\u82e5\u5e72\u95ee\u9898\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6122023-11-21","text":"","title":"6.1.2\uff0c2023-11-21"},{"location":"releases/#_55","text":"\u53d1\u5e03 JindoSDK 6.1.2 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_56","text":"JindoSDK \u66f4\u65b0 6.1.2 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u901a\u8fc7\u914d\u7f6e fs.oss.list.type=2 \u652f\u6301ListObjectV2\uff0c\u4ee5\u907f\u514d\u8bbf\u95ee\u5f00\u542f\u4e86\u591a\u7248\u672cbucket\u65f6\u7684\u64cd\u4f5c\u8d85\u65f6\u95ee\u9898\u3002 \u652f\u6301\u4f7f\u7528 dls:// \u524d\u7f00\u8bbf\u95ee jindofs\u3002 \u4fee\u590d\u8bfb\u6d41 close \u4f1a\u5361\u4f4f\u7684\u95ee\u9898\u3002 \u4fee\u590d\u5199\u6d41\u51fa\u73b0\u4e00\u6b21 write \u5931\u8d25\u540e\uff0c\u540e\u7eed write \u51fa\u73b0\u7684 NPE \u95ee\u9898\u3002","title":"\u4ecb\u7ecd"},{"location":"releases/#6112023-10-20","text":"","title":"6.1.1\uff0c2023-10-20"},{"location":"releases/#_57","text":"\u53d1\u5e03 JindoSDK 6.1.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_58","text":"JindoSDK \u66f4\u65b0 6.1.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \u652f\u6301\u901a\u8fc7\u914d\u7f6e fs.oss.legacy.version \u4e3a 3.8\uff0c\u517c\u5bb9 JindoSDK 3.8.x \u914d\u7f6e","title":"\u4ecb\u7ecd"},{"location":"releases/#6102023-09-28","text":"","title":"6.1.0\uff0c2023-09-28"},{"location":"releases/#_59","text":"\u53d1\u5e03 JindoSDK 6.1.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_60","text":"JindoSDK \u66f4\u65b0 6.1.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 JindoSDK \u652f\u6301 FileSystem close \u540e\u4ecd\u652f\u6301\u4f7f\u7528\u8bfb\u5199\u6d41\u8fdb\u884c\u8bfb\u5199","title":"\u4ecb\u7ecd"},{"location":"releases/#6012023-08-18","text":"","title":"6.0.1\uff0c2023-08-18"},{"location":"releases/#_61","text":"\u53d1\u5e03 JindoSDK 6.0.1 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_62","text":"JindoSDK \u66f4\u65b0 6.0.1 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 JindoSDK \u652f\u6301 isFileClosed \u65b9\u6cd5\uff0c\u7528\u4e8e\u5224\u65ad\u8bbf\u95ee OSS-HDFS \u7684\u8bfb\u5199\u6d41\u662f\u5426\u5df2\u5173\u95ed\u3002 JindoSDK \u5bf9 flush \u964d\u9891\u505a\u4e86\u4f18\u5316\uff0c\u5728\u964d\u9891\u914d\u7f6e\u751f\u6548\u65f6\uff0c\u5bf9\u7b2c\u4e00\u6b21\u6709\u6570\u636e\u7684flush\u64cd\u4f5c\u4fdd\u8bc1\u6267\u884c\uff0c\u9632\u6b62flush\u51fa\u4e00\u4e2a\u7a7a\u6587\u4ef6\u3002\u964d\u9891\u914d\u7f6e\uff0c\u53ef\u53c2\u8003 Flume \u4f7f\u7528 JindoSDK \u5199\u5165 OSS","title":"\u4ecb\u7ecd"},{"location":"releases/#6002023-08-15","text":"","title":"6.0.0\uff0c2023-08-15"},{"location":"releases/#_63","text":"\u53d1\u5e03 JindoSDK 6.0.0 \u6b63\u5f0f\u7248\u7684\u529f\u80fd","title":"\u7248\u672c\u6982\u8981"},{"location":"releases/#_64","text":"JindoSDK \u66f4\u65b0 6.0.0 \u7684 Maven \u4ed3\u5e93 \u548c \u4e0b\u8f7d\u5730\u5740 \uff0cJindoSDK \u5408\u5e76 jindo-mapreduce (JindoCommitterFactory\u7b49) \u5230 jindo-sdk JindoSDK \u65b0\u589e\u82e5\u5e72\u4fee\u590d\uff0c\u5305\u542b flush \u964d\u9891\uff0c\u4f18\u5316 GC \u56de\u6536\uff0c\u9002\u914d OSS-HDFS \u56de\u6536\u7ad9\u7b49 \u591a\u5e73\u53f0\u652f\u6301 MACOS \u548c \u4e3b\u6d41 linux \u573a\u666f\uff0c\u5176\u4e2d CentOS 6, Ubuntu22, \u963f\u91cc\u4e91\u501a\u5929\u673a\u578b(ARM)\uff0cMacOS (Intel/M1) \u9700\u8981\u4f7f\u7528\u7279\u5b9a\u7684\u6269\u5c55\u5305 \u5347\u7ea7 g++ \u7f16\u8bd1\u5668\u5230 10.4\uff0c\u4f7f\u7528 c++20 \u6807\u51c6 \u652f\u6301 yalangtinglibs \u8bbf\u95ee OSS/OSS-HDFS","title":"\u4ecb\u7ecd"},{"location":"backup/jindodata_QA/","text":"JindoData \u5e38\u89c1\u95ee\u9898 \u00a7 OSS-HDFS \u670d\u52a1 \u00a7 \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u663e\u793a\u5185\u5bb9\u4e0e\u5b9e\u9645\u4e0d\u7b26 \u00a7 \u73b0\u8c61 \u00a7 \u8bbf\u95ee oss://<bucket>.oss-dls.aliyuncs.com/ \u663e\u793a\u5185\u5bb9\u4e0e\u5b9e\u9645\u4e0d\u7b26\u3002 \u89e3\u51b3\u529e\u6cd5 \u00a7 \u5347\u7ea7 JindoSDK \u5230 4.x \u7248\u672c\u3002 OSS SDK \u65b9\u5f0f \u00a7 JindoSDK \u8bbf\u95ee OSS \u51fa\u73b0 CopyNotExecuted \u95ee\u9898\u6392\u67e5 \u00a7 \u73b0\u8c61 \u00a7 \u51fa\u73b0\u62a5\u9519 java.io.IOException: ErrorCode : 25201 , ErrorMsg: OSS Op Error. [ErrorMessage]: Error in xxxxxxxxxx/.hive-staging_hive_xxxxxxxxxxx/_tmp.-ext-10002/xxxxx to xxxxxxxxxx/.hive-staging_hive_xxxxxxxxxxx/_tmp.-ext-10002.moved/xxxx, err: CopyNotExcecuted; \u89e3\u51b3\u529e\u6cd5 \u00a7 CopyNotExecuted \u662f OSS \u670d\u52a1\u7aef\u8fd4\u56de\u7684\u5f02\u5e38\uff0c\u4e00\u822c\u662f OSS BatchCopy \u670d\u52a1\u7aef\u8fc7\u8f7d\u8fd4\u56de\u5f02\u5e38\u3002\u65e9\u671f\u7248\u672c OSS \u672a\u5b9a\u4e49 BatchCopy \u5f02\u5e38\u65f6\u5982\u4f55\u5904\u7406\uff0c\u9700\u8981\u5173\u95ed BatchCopy \u624d\u80fd\u5728\u8001\u7248\u672c\u4e0a\u89e3\u51b3\u3002\u5728EMR\u63a7\u5236\u53f0\uff0c\u4fee\u6539 smartdata-site \u7684 fs.jfs.cache.batch-copy.size \u4e3a 0\uff0c\u5c31\u53ef\u4ee5\u5173\u95ed batch copy\uff0c\u5e76\u4f9d\u6b21\u4fdd\u5b58->\u90e8\u7f72-> \u52fe\u9009\u540c\u6b65\u66f4\u65b0 Gateway","title":"JindoData \u5e38\u89c1\u95ee\u9898"},{"location":"backup/jindodata_QA/#jindodata","text":"","title":"JindoData \u5e38\u89c1\u95ee\u9898"},{"location":"backup/jindodata_QA/#oss-hdfs","text":"","title":"OSS-HDFS \u670d\u52a1"},{"location":"backup/jindodata_QA/#oss-hdfs_1","text":"","title":"\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u663e\u793a\u5185\u5bb9\u4e0e\u5b9e\u9645\u4e0d\u7b26"},{"location":"backup/jindodata_QA/#_1","text":"\u8bbf\u95ee oss://<bucket>.oss-dls.aliyuncs.com/ \u663e\u793a\u5185\u5bb9\u4e0e\u5b9e\u9645\u4e0d\u7b26\u3002","title":"\u73b0\u8c61"},{"location":"backup/jindodata_QA/#_2","text":"\u5347\u7ea7 JindoSDK \u5230 4.x \u7248\u672c\u3002","title":"\u89e3\u51b3\u529e\u6cd5"},{"location":"backup/jindodata_QA/#oss-sdk","text":"","title":"OSS SDK \u65b9\u5f0f"},{"location":"backup/jindodata_QA/#jindosdk-oss-copynotexecuted","text":"","title":"JindoSDK \u8bbf\u95ee OSS \u51fa\u73b0 CopyNotExecuted \u95ee\u9898\u6392\u67e5"},{"location":"backup/jindodata_QA/#_3","text":"\u51fa\u73b0\u62a5\u9519 java.io.IOException: ErrorCode : 25201 , ErrorMsg: OSS Op Error. [ErrorMessage]: Error in xxxxxxxxxx/.hive-staging_hive_xxxxxxxxxxx/_tmp.-ext-10002/xxxxx to xxxxxxxxxx/.hive-staging_hive_xxxxxxxxxxx/_tmp.-ext-10002.moved/xxxx, err: CopyNotExcecuted;","title":"\u73b0\u8c61"},{"location":"backup/jindodata_QA/#_4","text":"CopyNotExecuted \u662f OSS \u670d\u52a1\u7aef\u8fd4\u56de\u7684\u5f02\u5e38\uff0c\u4e00\u822c\u662f OSS BatchCopy \u670d\u52a1\u7aef\u8fc7\u8f7d\u8fd4\u56de\u5f02\u5e38\u3002\u65e9\u671f\u7248\u672c OSS \u672a\u5b9a\u4e49 BatchCopy \u5f02\u5e38\u65f6\u5982\u4f55\u5904\u7406\uff0c\u9700\u8981\u5173\u95ed BatchCopy \u624d\u80fd\u5728\u8001\u7248\u672c\u4e0a\u89e3\u51b3\u3002\u5728EMR\u63a7\u5236\u53f0\uff0c\u4fee\u6539 smartdata-site \u7684 fs.jfs.cache.batch-copy.size \u4e3a 0\uff0c\u5c31\u53ef\u4ee5\u5173\u95ed batch copy\uff0c\u5e76\u4f9d\u6b21\u4fdd\u5b58->\u90e8\u7f72-> \u52fe\u9009\u540c\u6b65\u66f4\u65b0 Gateway","title":"\u89e3\u51b3\u529e\u6cd5"},{"location":"backup/known-issues/","text":"\u5df2\u77e5\u95ee\u9898 \u00a7 JindoSDK 4.6.1 \u5728 EMR \u96c6\u7fa4\u4f7f\u7528\u514d\u5bc6\u8bbf\u95ee OSS-HDFS \u51fa\u73b0\u7b49\u5f85 Token \u66f4\u65b0\uff0c\u5bfc\u81f4\u90e8\u5206\u4f5c\u4e1a\u5361\u4f4f\u7684\u95ee\u9898\uff0c\u5982\u9700\u89e3\u51b3\u6b64\u95ee\u9898\uff0c\u4f7f\u7528\u56fa\u5b9a AK \u6216\u8005\u66f4\u65b0\u5230 4.6.4 \u53ca\u4ee5\u4e0a\u7248\u672c\uff0cEMR\u96c6\u7fa4\u5347\u7ea7\u53ef\u53c2\u8003 EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b \uff1b","title":"\u5df2\u77e5\u95ee\u9898"},{"location":"backup/known-issues/#_1","text":"JindoSDK 4.6.1 \u5728 EMR \u96c6\u7fa4\u4f7f\u7528\u514d\u5bc6\u8bbf\u95ee OSS-HDFS \u51fa\u73b0\u7b49\u5f85 Token \u66f4\u65b0\uff0c\u5bfc\u81f4\u90e8\u5206\u4f5c\u4e1a\u5361\u4f4f\u7684\u95ee\u9898\uff0c\u5982\u9700\u89e3\u51b3\u6b64\u95ee\u9898\uff0c\u4f7f\u7528\u56fa\u5b9a AK \u6216\u8005\u66f4\u65b0\u5230 4.6.4 \u53ca\u4ee5\u4e0a\u7248\u672c\uff0cEMR\u96c6\u7fa4\u5347\u7ea7\u53ef\u53c2\u8003 EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b \uff1b","title":"\u5df2\u77e5\u95ee\u9898"},{"location":"backup/release-notes/","text":"JindoData 4.6.12 \u7248\u672c\u8bf4\u660e \u00a7 \u7248\u672c\u6982\u8981 \u00a7 JindoData 4.6.12 \u7248\u672c\u4f18\u5316\u4e86\u5bf9 Hadoop \u793e\u533a\u4f7f\u7528\u6587\u4ef6\u8bfb\u5199\u6d41\u7684\u517c\u5bb9\u6027\u3002 \u4fee\u590d\u4ecb\u7ecd \u00a7 JindoSDK \u5728 FileSystem close \u540e\u4ecd\u652f\u6301\u4f7f\u7528\u8bfb\u5199\u6d41\u8fdb\u884c\u8bfb\u5199\u3002","title":"JindoData 4.6.12 \u7248\u672c\u8bf4\u660e"},{"location":"backup/release-notes/#jindodata-4612","text":"","title":"JindoData 4.6.12 \u7248\u672c\u8bf4\u660e"},{"location":"backup/release-notes/#_1","text":"JindoData 4.6.12 \u7248\u672c\u4f18\u5316\u4e86\u5bf9 Hadoop \u793e\u533a\u4f7f\u7528\u6587\u4ef6\u8bfb\u5199\u6d41\u7684\u517c\u5bb9\u6027\u3002","title":"\u7248\u672c\u6982\u8981"},{"location":"backup/release-notes/#_2","text":"JindoSDK \u5728 FileSystem close \u540e\u4ecd\u652f\u6301\u4f7f\u7528\u8bfb\u5199\u6d41\u8fdb\u884c\u8bfb\u5199\u3002","title":"\u4fee\u590d\u4ecb\u7ecd"},{"location":"backup/jindofs/jindo_dls_quickstart/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8 \u00a7 OSS-HDFS \u670d\u52a1\u662f\u963f\u91cc\u4e91\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9HDFS\u63a5\u53e3, \u652f\u6301\u76ee\u5f55\u4ee5\u53ca\u76ee\u5f55\u5c42\u7ea7\uff0cJindoSDK \u4e3a Apache Hadoop\u7684\u8ba1\u7b97\u5206\u6790\u5e94\u7528\uff08\u4f8b\u5982MapReduce\u3001Hive\u3001Spark\u3001Flink\u7b49\uff09\u63d0\u4f9b\u4e86\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u529f\u80fd\u3002\u5728\u7528\u6237\u73b0\u6709\u7684 Hadoop \u73af\u5883\u3001Hadoop \u96c6\u7fa4\u6216\u8005 Hadoop \u5ba2\u6237\u7aef\uff0c\u901a\u8fc7\u4fee\u6539 core-site.xml\uff0c\u5982\u4f55\u5bf9\u63a5\u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff0c\u53ef\u4ee5\u5feb\u901f\u67e5\u770b\u672c\u6587\u6863\u3002 1. \u670d\u52a1\u5f00\u901a \u00a7 \u8be6\u60c5\u53c2\u8003 \u5f00\u901a\u5e76\u6388\u6743\u8bbf\u95ee OSS-HDFS \u670d\u52a1 2. \u83b7\u53d6HDFS\u670d\u52a1\u57df\u540d \u00a7 \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u5728OSS\u7ba1\u7406\u63a7\u5236\u53f0\u7684\u6982\u89c8\u9875\u9762\uff0c\u590d\u5236HDFS\u670d\u52a1\u7684\u57df\u540d\u3002 3. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 4. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 5. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002 \u57fa\u672c\u64cd\u4f5c\u793a\u4f8b \u00a7 OSS-HDFS \u670d\u52a1\u521b\u5efa\u4ee5\u53ca\u914d\u7f6e\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7 hdfs dfs \u547d\u4ee4\u8fdb\u884c\u76f8\u5173\u6587\u4ef6/\u76ee\u5f55\u64cd\u4f5c \u65b0\u5efa\u76ee\u5f55 \u00a7 \u5728 OSS-HDFS \u670d\u52a1\u4e0a\u521b\u5efa\u76ee\u5f55\u3002 \u7528\u4f8b: hdfs dfs -mkdir oss://<Bucket>.<Endpoint>/Test/subdir [root@emr-header-1 ~]# hdfs dfs -mkdir oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test Found 1 items drwxr-x--x - root supergroup 0 2021-12-01 20:19 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir \u65b0\u5efa\u6587\u4ef6 \u00a7 \u5229\u7528 hdfs dfs -put \u547d\u4ee4\u4e0a\u4f20\u672c\u5730\u6587\u4ef6\u5230 OSS-HDFS \u670d\u52a1\u3002 \u7528\u4f8b\uff1a hdfs dfs -put <localfile> oss://<Bucket>.<Endpoint>/Test [root@emr-header-1 ~]# hdfs dfs -put /etc/hosts oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/ [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test Found 2 items -rw-r----- 1 root supergroup 5824 2021-12-01 20:24 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/hosts drwxr-x--x - root supergroup 0 2021-12-01 20:19 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir \u67e5\u770b\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u4fe1\u606f \u00a7 \u5728\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u521b\u5efa\u5b8c\u4e4b\u540e\uff0c\u53ef\u4ee5\u67e5\u770b\u6307\u5b9a\u8def\u5f84\u4e0b\u7684\u6587\u4ef6/\u76ee\u5f55\u4fe1\u606f\u3002hdfs dfs \u6ca1\u6709\u8fdb\u5165\u67d0\u4e2a\u76ee\u5f55\u4e0b\u7684\u6982\u5ff5\u3002\u5728\u67e5\u770b\u76ee\u5f55\u548c\u6587\u4ef6\u7684\u4fe1\u606f\u7684\u65f6\u5019\u9700\u8981\u7ed9\u51fa\u6587\u4ef6/\u76ee\u5f55\u7684\u7edd\u5bf9\u8def\u5f84\u3002 \u6307\u4ee4\uff1als \u7528\u4f8b\uff1a hdfs dfs -ls oss://<Bucket>.<Endpoint>/Test [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test Found 2 items -rw-r----- 1 root supergroup 5824 2021-12-01 20:24 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/hosts drwxr-x--x - root supergroup 0 2021-12-01 20:19 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir \u67e5\u770b\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u7684\u5927\u5c0f \u00a7 \u67e5\u770b\u5df2\u6709\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u7684\u5927\u5c0f \u7528\u4f8b\uff1a hdfs dfs -du oss://<Bucket>.<Endpoint>/Test [root@emr-header-1 ~]# hdfs dfs -du oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test 5824 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/hosts 0 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir \u67e5\u770b\u6587\u4ef6\u7684\u5185\u5bb9 \u00a7 \u6709\u65f6\u5019\u6211\u4eec\u9700\u8981\u67e5\u770b\u4e00\u4e0b\u5728 OSS-HDFS \u670d\u52a1\u6587\u4ef6\u7684\u5185\u5bb9\u3002hdfs dfs \u547d\u4ee4\u652f\u6301\u6211\u4eec\u5c06\u6587\u4ef6\u5185\u5bb9\u6253\u5370\u5728\u5c4f\u5e55\u4e0a\u3002\uff08\u8bf7\u6ce8\u610f\uff0c\u6587\u4ef6\u5185\u5bb9\u5c06\u4f1a\u4ee5\u7eaf\u6587\u672c\u5f62\u5f0f\u6253\u5370\u51fa\u6765\uff0c\u5982\u679c\u6587\u4ef6\u8fdb\u884c\u4e86\u7279\u5b9a\u683c\u5f0f\u7684\u7f16\u7801\uff0c\u8bf7\u4f7f\u7528 HDFS \u7684 JavaAPI \u5c06\u6587\u4ef6\u5185\u5bb9\u8bfb\u53d6\u5e76\u8fdb\u884c\u76f8\u5e94\u7684\u89e3\u7801\u83b7\u53d6\u6587\u4ef6\u5185\u5bb9\uff09 \u7528\u4f8b\uff1a hdfs dfs -cat oss://<Bucket>.<Endpoint>/Test/helloworld.txt [root@emr-header-1 ~]# hdfs dfs -cat oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/helloworld.txt hello world! \u590d\u5236\u76ee\u5f55/\u6587\u4ef6 \u00a7 \u6709\u65f6\u5019\u6211\u4eec\u9700\u8981\u5c06 OSS-HDFS \u670d\u52a1\u7684\u4e00\u4e2a\u6587\u4ef6/\u76ee\u5f55\u62f7\u8d1d\u5230\u53e6\u4e00\u4e2a\u4f4d\u7f6e\uff0c\u5e76\u4e14\u4fdd\u6301\u6e90\u6587\u4ef6\u548c\u76ee\u5f55\u7ed3\u6784\u548c\u5185\u5bb9\u4e0d\u53d8\u3002 \u7528\u4f8b\uff1a hdfs dfs -cp oss://<Bucket>.<Endpoint>/Test/subdir oss://<Bucket>.<Endpoint>/TestTarget/sudir2 [root@emr-header-1 ~]# hdfs dfs -cp oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/TestTarget/subdir1 [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/TestTarget/ Found 1 items drwxr-x--x - root supergroup 0 2021-12-01 20:37 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/TestTarget/subdir1 \u79fb\u52a8\u76ee\u5f55/\u6587\u4ef6 \u00a7 \u5728\u5f88\u591a\u5927\u6570\u636e\u5904\u7406\u7684\u4f8b\u5b50\u4e2d\uff0c\u6211\u4eec\u4f1a\u5c06\u6587\u4ef6\u5199\u5165\u4e00\u4e2a\u4e34\u65f6\u76ee\u5f55\uff0c\u7136\u540e\u5c06\u8be5\u76ee\u5f55\u79fb\u52a8\u5230\u53e6\u4e00\u4e2a\u4f4d\u7f6e\u4f5c\u4e3a\u6700\u7ec8\u7ed3\u679c\u3002\u6e90\u6587\u4ef6\u548c\u76ee\u5f55\u7ed3\u6784\u548c\u5185\u5bb9\u4e0d\u505a\u4fdd\u7559\u3002\u4e0b\u9762\u7684\u547d\u4ee4\u53ef\u4ee5\u5b8c\u6210\u8fd9\u4e9b\u64cd\u4f5c\u3002 \u7528\u4f8b\uff1a hdfs dfs -mv oss://<Bucket>.<Endpoint>/Test/subdir oss://<Bucket>.<Endpoint>/Test/subdir1 [root@emr-header-1 ~]# hdfs dfs -mv oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/newdir [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test Found 3 items -rw-r----- 1 root supergroup 13 2021-12-01 20:33 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/helloworld.txt -rw-r----- 1 root supergroup 5824 2021-12-01 20:24 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/hosts drwxr-x--x - root supergroup 0 2021-12-01 20:19 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/newdir \u4e0b\u8f7d\u6587\u4ef6\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf \u00a7 \u67d0\u4e9b\u60c5\u51b5\u4e0b\uff0c\u6211\u4eec\u9700\u8981\u5c06 OSS-HDFS \u670d\u52a1\u4e0a\u4e2d\u7684\u67d0\u4e9b\u6587\u4ef6\u4e0b\u8f7d\u5230\u672c\u5730\uff0c\u518d\u8fdb\u884c\u5904\u7406\u6216\u8005\u67e5\u770b\u5185\u5bb9\u3002\u8fd9\u4e2a\u53ef\u4ee5\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u5b8c\u6210\u3002 \u7528\u4f8b\uff1a hdfs dfs -get oss://<Bucket>.<Endpoint>/Test/helloworld.txt <localpath> [root@emr-header-1 ~]# hdfs dfs -get oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/helloworld.txt /tmp/ [root@emr-header-1 ~]# ll /tmp/helloworld.txt -rw-r----- 1 root root 13 12\u6708 1 20:44 /tmp/helloworld.txt \u5220\u9664\u76ee\u5f55/\u6587\u4ef6 \u00a7 \u5728\u5f88\u591a\u60c5\u51b5\u4e0b\uff0c\u6211\u4eec\u5728\u5b8c\u6210\u5de5\u4f5c\u540e\uff0c\u9700\u8981\u5220\u9664\u5728 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u67d0\u4e9b\u4e34\u65f6\u6587\u4ef6\u6216\u8005\u5e9f\u5f03\u6587\u4ef6\u3002\u8fd9\u4e9b\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u547d\u4ee4\u5b8c\u6210\u3002 \u7528\u4f8b\uff1a hdfs dfs -rm oss://<Bucket>.<Endpoint>/Test/helloworld.txt [root@emr-header-1 ~]# hdfs dfs -rm oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/helloworld.txt 21/12/01 20:46:44 INFO fs.TrashPolicyDefault: Moved: 'oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/helloworld.txt' to trash at: oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/user/root/.Trash/Current/Test/helloworld.txt [root@emr-header-1 ~]# hdfs dfs -rm -r oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/newdir 21/12/01 20:47:16 INFO fs.TrashPolicyDefault: Moved: 'oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/newdir' to trash at: oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/user/root/.Trash/Current/Test/newdir [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/ Found 1 items -rw-r----- 1 root supergroup 5824 2021-12-01 20:24 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/hosts","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8"},{"location":"backup/jindofs/jindo_dls_quickstart/#oss-hdfs-jindofs","text":"OSS-HDFS \u670d\u52a1\u662f\u963f\u91cc\u4e91\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9HDFS\u63a5\u53e3, \u652f\u6301\u76ee\u5f55\u4ee5\u53ca\u76ee\u5f55\u5c42\u7ea7\uff0cJindoSDK \u4e3a Apache Hadoop\u7684\u8ba1\u7b97\u5206\u6790\u5e94\u7528\uff08\u4f8b\u5982MapReduce\u3001Hive\u3001Spark\u3001Flink\u7b49\uff09\u63d0\u4f9b\u4e86\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u529f\u80fd\u3002\u5728\u7528\u6237\u73b0\u6709\u7684 Hadoop \u73af\u5883\u3001Hadoop \u96c6\u7fa4\u6216\u8005 Hadoop \u5ba2\u6237\u7aef\uff0c\u901a\u8fc7\u4fee\u6539 core-site.xml\uff0c\u5982\u4f55\u5bf9\u63a5\u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff0c\u53ef\u4ee5\u5feb\u901f\u67e5\u770b\u672c\u6587\u6863\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8"},{"location":"backup/jindofs/jindo_dls_quickstart/#1","text":"\u8be6\u60c5\u53c2\u8003 \u5f00\u901a\u5e76\u6388\u6743\u8bbf\u95ee OSS-HDFS \u670d\u52a1","title":"1. \u670d\u52a1\u5f00\u901a"},{"location":"backup/jindofs/jindo_dls_quickstart/#2-hdfs","text":"\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u5728OSS\u7ba1\u7406\u63a7\u5236\u53f0\u7684\u6982\u89c8\u9875\u9762\uff0c\u590d\u5236HDFS\u670d\u52a1\u7684\u57df\u540d\u3002","title":"2. \u83b7\u53d6HDFS\u670d\u52a1\u57df\u540d"},{"location":"backup/jindofs/jindo_dls_quickstart/#3-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"3. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindofs/jindo_dls_quickstart/#4-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"4. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofs/jindo_dls_quickstart/#5-oss-hdfs-access-key","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002","title":"5. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"backup/jindofs/jindo_dls_quickstart/#_1","text":"OSS-HDFS \u670d\u52a1\u521b\u5efa\u4ee5\u53ca\u914d\u7f6e\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7 hdfs dfs \u547d\u4ee4\u8fdb\u884c\u76f8\u5173\u6587\u4ef6/\u76ee\u5f55\u64cd\u4f5c","title":"\u57fa\u672c\u64cd\u4f5c\u793a\u4f8b"},{"location":"backup/jindofs/jindo_dls_quickstart/#_2","text":"\u5728 OSS-HDFS \u670d\u52a1\u4e0a\u521b\u5efa\u76ee\u5f55\u3002 \u7528\u4f8b: hdfs dfs -mkdir oss://<Bucket>.<Endpoint>/Test/subdir [root@emr-header-1 ~]# hdfs dfs -mkdir oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test Found 1 items drwxr-x--x - root supergroup 0 2021-12-01 20:19 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir","title":"\u65b0\u5efa\u76ee\u5f55"},{"location":"backup/jindofs/jindo_dls_quickstart/#_3","text":"\u5229\u7528 hdfs dfs -put \u547d\u4ee4\u4e0a\u4f20\u672c\u5730\u6587\u4ef6\u5230 OSS-HDFS \u670d\u52a1\u3002 \u7528\u4f8b\uff1a hdfs dfs -put <localfile> oss://<Bucket>.<Endpoint>/Test [root@emr-header-1 ~]# hdfs dfs -put /etc/hosts oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/ [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test Found 2 items -rw-r----- 1 root supergroup 5824 2021-12-01 20:24 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/hosts drwxr-x--x - root supergroup 0 2021-12-01 20:19 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir","title":"\u65b0\u5efa\u6587\u4ef6"},{"location":"backup/jindofs/jindo_dls_quickstart/#_4","text":"\u5728\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u521b\u5efa\u5b8c\u4e4b\u540e\uff0c\u53ef\u4ee5\u67e5\u770b\u6307\u5b9a\u8def\u5f84\u4e0b\u7684\u6587\u4ef6/\u76ee\u5f55\u4fe1\u606f\u3002hdfs dfs \u6ca1\u6709\u8fdb\u5165\u67d0\u4e2a\u76ee\u5f55\u4e0b\u7684\u6982\u5ff5\u3002\u5728\u67e5\u770b\u76ee\u5f55\u548c\u6587\u4ef6\u7684\u4fe1\u606f\u7684\u65f6\u5019\u9700\u8981\u7ed9\u51fa\u6587\u4ef6/\u76ee\u5f55\u7684\u7edd\u5bf9\u8def\u5f84\u3002 \u6307\u4ee4\uff1als \u7528\u4f8b\uff1a hdfs dfs -ls oss://<Bucket>.<Endpoint>/Test [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test Found 2 items -rw-r----- 1 root supergroup 5824 2021-12-01 20:24 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/hosts drwxr-x--x - root supergroup 0 2021-12-01 20:19 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir","title":"\u67e5\u770b\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u4fe1\u606f"},{"location":"backup/jindofs/jindo_dls_quickstart/#_5","text":"\u67e5\u770b\u5df2\u6709\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u7684\u5927\u5c0f \u7528\u4f8b\uff1a hdfs dfs -du oss://<Bucket>.<Endpoint>/Test [root@emr-header-1 ~]# hdfs dfs -du oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test 5824 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/hosts 0 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir","title":"\u67e5\u770b\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u7684\u5927\u5c0f"},{"location":"backup/jindofs/jindo_dls_quickstart/#_6","text":"\u6709\u65f6\u5019\u6211\u4eec\u9700\u8981\u67e5\u770b\u4e00\u4e0b\u5728 OSS-HDFS \u670d\u52a1\u6587\u4ef6\u7684\u5185\u5bb9\u3002hdfs dfs \u547d\u4ee4\u652f\u6301\u6211\u4eec\u5c06\u6587\u4ef6\u5185\u5bb9\u6253\u5370\u5728\u5c4f\u5e55\u4e0a\u3002\uff08\u8bf7\u6ce8\u610f\uff0c\u6587\u4ef6\u5185\u5bb9\u5c06\u4f1a\u4ee5\u7eaf\u6587\u672c\u5f62\u5f0f\u6253\u5370\u51fa\u6765\uff0c\u5982\u679c\u6587\u4ef6\u8fdb\u884c\u4e86\u7279\u5b9a\u683c\u5f0f\u7684\u7f16\u7801\uff0c\u8bf7\u4f7f\u7528 HDFS \u7684 JavaAPI \u5c06\u6587\u4ef6\u5185\u5bb9\u8bfb\u53d6\u5e76\u8fdb\u884c\u76f8\u5e94\u7684\u89e3\u7801\u83b7\u53d6\u6587\u4ef6\u5185\u5bb9\uff09 \u7528\u4f8b\uff1a hdfs dfs -cat oss://<Bucket>.<Endpoint>/Test/helloworld.txt [root@emr-header-1 ~]# hdfs dfs -cat oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/helloworld.txt hello world!","title":"\u67e5\u770b\u6587\u4ef6\u7684\u5185\u5bb9"},{"location":"backup/jindofs/jindo_dls_quickstart/#_7","text":"\u6709\u65f6\u5019\u6211\u4eec\u9700\u8981\u5c06 OSS-HDFS \u670d\u52a1\u7684\u4e00\u4e2a\u6587\u4ef6/\u76ee\u5f55\u62f7\u8d1d\u5230\u53e6\u4e00\u4e2a\u4f4d\u7f6e\uff0c\u5e76\u4e14\u4fdd\u6301\u6e90\u6587\u4ef6\u548c\u76ee\u5f55\u7ed3\u6784\u548c\u5185\u5bb9\u4e0d\u53d8\u3002 \u7528\u4f8b\uff1a hdfs dfs -cp oss://<Bucket>.<Endpoint>/Test/subdir oss://<Bucket>.<Endpoint>/TestTarget/sudir2 [root@emr-header-1 ~]# hdfs dfs -cp oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/TestTarget/subdir1 [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/TestTarget/ Found 1 items drwxr-x--x - root supergroup 0 2021-12-01 20:37 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/TestTarget/subdir1","title":"\u590d\u5236\u76ee\u5f55/\u6587\u4ef6"},{"location":"backup/jindofs/jindo_dls_quickstart/#_8","text":"\u5728\u5f88\u591a\u5927\u6570\u636e\u5904\u7406\u7684\u4f8b\u5b50\u4e2d\uff0c\u6211\u4eec\u4f1a\u5c06\u6587\u4ef6\u5199\u5165\u4e00\u4e2a\u4e34\u65f6\u76ee\u5f55\uff0c\u7136\u540e\u5c06\u8be5\u76ee\u5f55\u79fb\u52a8\u5230\u53e6\u4e00\u4e2a\u4f4d\u7f6e\u4f5c\u4e3a\u6700\u7ec8\u7ed3\u679c\u3002\u6e90\u6587\u4ef6\u548c\u76ee\u5f55\u7ed3\u6784\u548c\u5185\u5bb9\u4e0d\u505a\u4fdd\u7559\u3002\u4e0b\u9762\u7684\u547d\u4ee4\u53ef\u4ee5\u5b8c\u6210\u8fd9\u4e9b\u64cd\u4f5c\u3002 \u7528\u4f8b\uff1a hdfs dfs -mv oss://<Bucket>.<Endpoint>/Test/subdir oss://<Bucket>.<Endpoint>/Test/subdir1 [root@emr-header-1 ~]# hdfs dfs -mv oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/subdir oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/newdir [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test Found 3 items -rw-r----- 1 root supergroup 13 2021-12-01 20:33 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/helloworld.txt -rw-r----- 1 root supergroup 5824 2021-12-01 20:24 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/hosts drwxr-x--x - root supergroup 0 2021-12-01 20:19 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/newdir","title":"\u79fb\u52a8\u76ee\u5f55/\u6587\u4ef6"},{"location":"backup/jindofs/jindo_dls_quickstart/#_9","text":"\u67d0\u4e9b\u60c5\u51b5\u4e0b\uff0c\u6211\u4eec\u9700\u8981\u5c06 OSS-HDFS \u670d\u52a1\u4e0a\u4e2d\u7684\u67d0\u4e9b\u6587\u4ef6\u4e0b\u8f7d\u5230\u672c\u5730\uff0c\u518d\u8fdb\u884c\u5904\u7406\u6216\u8005\u67e5\u770b\u5185\u5bb9\u3002\u8fd9\u4e2a\u53ef\u4ee5\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u5b8c\u6210\u3002 \u7528\u4f8b\uff1a hdfs dfs -get oss://<Bucket>.<Endpoint>/Test/helloworld.txt <localpath> [root@emr-header-1 ~]# hdfs dfs -get oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/helloworld.txt /tmp/ [root@emr-header-1 ~]# ll /tmp/helloworld.txt -rw-r----- 1 root root 13 12\u6708 1 20:44 /tmp/helloworld.txt","title":"\u4e0b\u8f7d\u6587\u4ef6\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf"},{"location":"backup/jindofs/jindo_dls_quickstart/#_10","text":"\u5728\u5f88\u591a\u60c5\u51b5\u4e0b\uff0c\u6211\u4eec\u5728\u5b8c\u6210\u5de5\u4f5c\u540e\uff0c\u9700\u8981\u5220\u9664\u5728 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u67d0\u4e9b\u4e34\u65f6\u6587\u4ef6\u6216\u8005\u5e9f\u5f03\u6587\u4ef6\u3002\u8fd9\u4e9b\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u547d\u4ee4\u5b8c\u6210\u3002 \u7528\u4f8b\uff1a hdfs dfs -rm oss://<Bucket>.<Endpoint>/Test/helloworld.txt [root@emr-header-1 ~]# hdfs dfs -rm oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/helloworld.txt 21/12/01 20:46:44 INFO fs.TrashPolicyDefault: Moved: 'oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/helloworld.txt' to trash at: oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/user/root/.Trash/Current/Test/helloworld.txt [root@emr-header-1 ~]# hdfs dfs -rm -r oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/newdir 21/12/01 20:47:16 INFO fs.TrashPolicyDefault: Moved: 'oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/newdir' to trash at: oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/user/root/.Trash/Current/Test/newdir [root@emr-header-1 ~]# hdfs dfs -ls oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/ Found 1 items -rw-r----- 1 root supergroup 5824 2021-12-01 20:24 oss://dls-chenshi-test.cn-xxx.oss-dls.aliyuncs.com/Test/hosts","title":"\u5220\u9664\u76ee\u5f55/\u6587\u4ef6"},{"location":"backup/jindofs/jindo_util_intro/","text":"jindo \u547d\u4ee4\u4f7f\u7528\u65b9\u6cd5 \u00a7 (\u4ece 4.5.0 \u5f00\u59cb\u652f\u6301) \u4e0b\u8f7d jindo \u547d\u4ee4\u884c\u5de5\u5177\u5305\uff08\u5f53\u524d\u53ea\u652f\u6301 Linux \u73af\u5883\u4f7f\u7528\uff09 \u00a7 1\u3001\u4e0b\u8f7d jindosdk-4.6.12.tar.gz 2\u3001\u89e3\u538b tar \u5305 tar -zxvf jindosdk-4.6.12.tar.gz 3\u3001\u5728 /jindosdk-4.6.12/bin/ \u76ee\u5f55\u4e0b\u627e\u5230 jindo-util \u4e8c\u8fdb\u5236\u6587\u4ef6 chmod 700 jindo-util mv jindo-util jindo 4\u3001\u67e5\u770b jindo \u6240\u6709\u652f\u6301\u7684\u547d\u4ee4\u548c\u4f7f\u7528\u65b9\u6cd5 # ./jindo fs -help Usage: jindo fs [-appendToFile <localsrc> ... <dst>] [-archive [-i] [-c] <path>] [-cat <src> ...] [-count [-p] [-r] <path> ...] [-cp [-f] <src> ... <dst>] [-checksum <src> ...] [-chgrp [-R] GROUP PATH...] [-chmod [-R] <MODE[,MODE]... | OCTALMODE> PATH...] [-chown [-R] [OWNER][:[GROUP]] PATH...] [-copyFromLocal [-f] <localsrc> ... <dst>] [-copyToLocal [-f] <src> ... <localdst>] [-createSnapshot <snapshotDir> [<snapshotName>]] [-checkStoragePolicy -path <path>] [-deleteSnapshot <snapshotDir> <snapshotName>] [-du [-s] [-p] [-r] <path> ...] [-decommission -hosts <hosts>] [-formatCache] [-get [-f] <src> ... <localdst>] [-getfacl [-R] <path>] [-getfattr [-R] {-n name | -d} <path>] [-getStoragePolicy -path <path>] [-ls [-R] <path>] [-listPolicies] [-load [-meta] [-data] [-s] [-m] [-R] [-replica value] <path>] [-mkdir <path>] [-mv <src> ... <dst>] [-moveFromLocal <localsrc> ... <dst>] [-moveToLocal <src> <localdst>] [-metaDiff <path>] [-put [-f] <localsrc> ... <dst>] [-rm [-f] [-r|-R] <src> ...] [-rmdir <src> ...] [-renameSnapshot <snapshotDir> <oldName> <newName>] [-restore [-days value] <path>] [-stat [format] <path> ...] [-setStoragePolicy -path <path> -policy <policy>] [-setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>]] [-setfattr {-n name [-v value] | -x name} <path>] [-sync [-update] [-thread thread_num] <localsrc> <dst> [-touchz <path> ...] [-truncate [-w] <length> <path> ...] [-unarchive [-i] [-a] <path>] [-uncache <path>] [-unsetStoragePolicy -path <path>] jindo admin [-mount <mountpoint> <path>] [-report] [-reportMetrics] [-umount <mountpoint>] jindo admin [-addProxyUser -dlsUri <path> -proxyUser <value> -users|-groups <value> -hosts <value>] [-allowSnapshot -dlsUri <path>] [-addUserGroupsMapping -dlsUri <path> -user <value> -groups <value>] [-disallowSnapshot -dlsUri <path>] [-deleteProxyUser -dlsUri <path> -proxyUser <value>] [-deleteUserGroupsMapping -dlsUri <path> -user <value>] [-dumpFile -dlsUri <path>] [-dumpBlockFiles -in <path> -out <path>] [-listProxyUsers -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-listUserGroupsMappings -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-snapshotDiff -dlsUri <path> -fromSnapshot <value> -toSnapshot <value>] jindo benchmark [-io -read -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-memCheck]] [-io -pread -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-seekTimes <value>] [-memCheck]] [-io -write -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-memCheck]] [-meta -mkdir -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-memCheck]] [-meta -listDirectory -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]] [-meta -getFileStatus -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]] [-meta -renameFile -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]] \u4f7f\u7528 jindo \u547d\u4ee4\u8bbf\u95ee\u6807\u51c6 OSS \u6216\u8005 OSS-HDFS \u00a7 \u5168\u5c40\u914d\u7f6e \u00a7 1\u3001\u65b0\u5efa\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg [common] logger.dir = /tmp/jindo-util/ logger.sync = false logger.consolelogger = false logger.level = 0 logger.verbose = 0 logger.cleaner.enable = true hadoopConf.enable = false [jindosdk] fs.oss.endpoint = <ENDPOINT> fs.oss.accessKeyId = <ACCESS_KEYID> fs.oss.accessKeySecret = <ACCESS_KEYSECRET> < ENDPOINT>: \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 endpoint : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeyId : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeySecret 2\u3001\u6dfb\u52a0\u73af\u5883\u53d8\u91cf export JINDOSDK_CONF_DIR=<JINDOSDK_CFG_DIR> : jindosdk.cfg \u914d\u7f6e\u6587\u4ef6\u6240\u5728\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u6bd4\u5982 /etc/ 3\u3001\u4f7f\u7528 jindo \u547d\u4ee4\u8bbf\u95ee OSS ./jindo fs -ls oss://<bucket>/<dir> bucket \u7ea7\u522b\u914d\u7f6e \u00a7 1\u3001\u65b0\u5efa\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg [common] logger.dir = /tmp/jindo-util/ logger.sync = false logger.consolelogger = false logger.level = 0 logger.verbose = 0 logger.cleaner.enable = true hadoopConf.enable = false [jindosdk] fs.oss.bucket.<BUCKET>.accessKeyId = <ACCESS_KEYID> fs.oss.bucket.<BUCKET>.accessKeySecret = <ACCESS_KEYSECRET> fs.oss.bucket.<BUCKET>.endpoint = <ENDPOINT> < ENDPOINT>: \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 endpoint : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeyId : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeySecret 2\u3001\u6dfb\u52a0\u73af\u5883\u53d8\u91cf export JINDOSDK_CONF_DIR=<JINDOSDK_CFG_DIR> : jindosdk.cfg \u914d\u7f6e\u6587\u4ef6\u6240\u5728\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u6bd4\u5982 /etc/ 3\u3001\u4f7f\u7528 jindo \u547d\u4ee4\u8bbf\u95ee OSS-HDFS \u670d\u52a1 ./jindo fs -ls oss://<bucket>/<dir> \u4f7f\u7528 jindo sync \u540c\u6b65\u6570\u636e \u00a7 1\u3001\u65b0\u5efa\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg [common] logger.dir = /tmp/jindo-util/ logger.sync = false logger.consolelogger = false logger.level = 0 logger.verbose = 0 logger.cleaner.enable = true hadoopConf.enable = false [jindosdk] fs.oss.endpoint = <ENDPOINT> fs.oss.accessKeyId = <ACCESS_KEYID> fs.oss.accessKeySecret = <ACCESS_KEYSECRET> < ENDPOINT>: \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 endpoint : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeyId : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeySecret 2\u3001\u6dfb\u52a0\u73af\u5883\u53d8\u91cf export JINDOSDK_CONF_DIR=<JINDOSDK_CFG_DIR> : jindosdk.cfg \u914d\u7f6e\u6587\u4ef6\u6240\u5728\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u6bd4\u5982 /etc/ 3\u3001\u4f7f\u7528 jindo \u547d\u4ee4\u8fdb\u884c\u6570\u636e\u4f20\u8f93 ./jindo fs -sync -thread 10 /local/dir/ oss://<bucket>/<dir> thread: \u4f7f\u7528\u7ebf\u7a0b\u6570\u91cf 4\u3001\u4f7f\u7528\u7aef\u70b9\u7eed\u4f20\u529f\u80fd ./jindo fs -sync -update -thread 10 /local/dir/ oss://<bucket>/<dir> \u56e0\u4e3a\u9700\u8981\u505a\u6587\u4ef6\u6bd4\u8f83\uff0c\u4f7f\u7528\u65ad\u70b9\u7eed\u4f20\u53ef\u80fd\u5bf9\u4f20\u8f93\u6027\u80fd\u6709\u4e00\u5b9a\u7684\u5f71\u54cd","title":"jindo \u547d\u4ee4\u4f7f\u7528\u65b9\u6cd5"},{"location":"backup/jindofs/jindo_util_intro/#jindo","text":"(\u4ece 4.5.0 \u5f00\u59cb\u652f\u6301)","title":"jindo \u547d\u4ee4\u4f7f\u7528\u65b9\u6cd5"},{"location":"backup/jindofs/jindo_util_intro/#jindo-linux","text":"1\u3001\u4e0b\u8f7d jindosdk-4.6.12.tar.gz 2\u3001\u89e3\u538b tar \u5305 tar -zxvf jindosdk-4.6.12.tar.gz 3\u3001\u5728 /jindosdk-4.6.12/bin/ \u76ee\u5f55\u4e0b\u627e\u5230 jindo-util \u4e8c\u8fdb\u5236\u6587\u4ef6 chmod 700 jindo-util mv jindo-util jindo 4\u3001\u67e5\u770b jindo \u6240\u6709\u652f\u6301\u7684\u547d\u4ee4\u548c\u4f7f\u7528\u65b9\u6cd5 # ./jindo fs -help Usage: jindo fs [-appendToFile <localsrc> ... <dst>] [-archive [-i] [-c] <path>] [-cat <src> ...] [-count [-p] [-r] <path> ...] [-cp [-f] <src> ... <dst>] [-checksum <src> ...] [-chgrp [-R] GROUP PATH...] [-chmod [-R] <MODE[,MODE]... | OCTALMODE> PATH...] [-chown [-R] [OWNER][:[GROUP]] PATH...] [-copyFromLocal [-f] <localsrc> ... <dst>] [-copyToLocal [-f] <src> ... <localdst>] [-createSnapshot <snapshotDir> [<snapshotName>]] [-checkStoragePolicy -path <path>] [-deleteSnapshot <snapshotDir> <snapshotName>] [-du [-s] [-p] [-r] <path> ...] [-decommission -hosts <hosts>] [-formatCache] [-get [-f] <src> ... <localdst>] [-getfacl [-R] <path>] [-getfattr [-R] {-n name | -d} <path>] [-getStoragePolicy -path <path>] [-ls [-R] <path>] [-listPolicies] [-load [-meta] [-data] [-s] [-m] [-R] [-replica value] <path>] [-mkdir <path>] [-mv <src> ... <dst>] [-moveFromLocal <localsrc> ... <dst>] [-moveToLocal <src> <localdst>] [-metaDiff <path>] [-put [-f] <localsrc> ... <dst>] [-rm [-f] [-r|-R] <src> ...] [-rmdir <src> ...] [-renameSnapshot <snapshotDir> <oldName> <newName>] [-restore [-days value] <path>] [-stat [format] <path> ...] [-setStoragePolicy -path <path> -policy <policy>] [-setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>]] [-setfattr {-n name [-v value] | -x name} <path>] [-sync [-update] [-thread thread_num] <localsrc> <dst> [-touchz <path> ...] [-truncate [-w] <length> <path> ...] [-unarchive [-i] [-a] <path>] [-uncache <path>] [-unsetStoragePolicy -path <path>] jindo admin [-mount <mountpoint> <path>] [-report] [-reportMetrics] [-umount <mountpoint>] jindo admin [-addProxyUser -dlsUri <path> -proxyUser <value> -users|-groups <value> -hosts <value>] [-allowSnapshot -dlsUri <path>] [-addUserGroupsMapping -dlsUri <path> -user <value> -groups <value>] [-disallowSnapshot -dlsUri <path>] [-deleteProxyUser -dlsUri <path> -proxyUser <value>] [-deleteUserGroupsMapping -dlsUri <path> -user <value>] [-dumpFile -dlsUri <path>] [-dumpBlockFiles -in <path> -out <path>] [-listProxyUsers -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-listUserGroupsMappings -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-snapshotDiff -dlsUri <path> -fromSnapshot <value> -toSnapshot <value>] jindo benchmark [-io -read -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-memCheck]] [-io -pread -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-seekTimes <value>] [-memCheck]] [-io -write -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-memCheck]] [-meta -mkdir -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-memCheck]] [-meta -listDirectory -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]] [-meta -getFileStatus -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]] [-meta -renameFile -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]]","title":"\u4e0b\u8f7d jindo \u547d\u4ee4\u884c\u5de5\u5177\u5305\uff08\u5f53\u524d\u53ea\u652f\u6301 Linux \u73af\u5883\u4f7f\u7528\uff09"},{"location":"backup/jindofs/jindo_util_intro/#jindo-oss-oss-hdfs","text":"","title":"\u4f7f\u7528 jindo \u547d\u4ee4\u8bbf\u95ee\u6807\u51c6 OSS \u6216\u8005 OSS-HDFS"},{"location":"backup/jindofs/jindo_util_intro/#_1","text":"1\u3001\u65b0\u5efa\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg [common] logger.dir = /tmp/jindo-util/ logger.sync = false logger.consolelogger = false logger.level = 0 logger.verbose = 0 logger.cleaner.enable = true hadoopConf.enable = false [jindosdk] fs.oss.endpoint = <ENDPOINT> fs.oss.accessKeyId = <ACCESS_KEYID> fs.oss.accessKeySecret = <ACCESS_KEYSECRET> < ENDPOINT>: \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 endpoint : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeyId : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeySecret 2\u3001\u6dfb\u52a0\u73af\u5883\u53d8\u91cf export JINDOSDK_CONF_DIR=<JINDOSDK_CFG_DIR> : jindosdk.cfg \u914d\u7f6e\u6587\u4ef6\u6240\u5728\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u6bd4\u5982 /etc/ 3\u3001\u4f7f\u7528 jindo \u547d\u4ee4\u8bbf\u95ee OSS ./jindo fs -ls oss://<bucket>/<dir>","title":"\u5168\u5c40\u914d\u7f6e"},{"location":"backup/jindofs/jindo_util_intro/#bucket","text":"1\u3001\u65b0\u5efa\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg [common] logger.dir = /tmp/jindo-util/ logger.sync = false logger.consolelogger = false logger.level = 0 logger.verbose = 0 logger.cleaner.enable = true hadoopConf.enable = false [jindosdk] fs.oss.bucket.<BUCKET>.accessKeyId = <ACCESS_KEYID> fs.oss.bucket.<BUCKET>.accessKeySecret = <ACCESS_KEYSECRET> fs.oss.bucket.<BUCKET>.endpoint = <ENDPOINT> < ENDPOINT>: \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 endpoint : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeyId : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeySecret 2\u3001\u6dfb\u52a0\u73af\u5883\u53d8\u91cf export JINDOSDK_CONF_DIR=<JINDOSDK_CFG_DIR> : jindosdk.cfg \u914d\u7f6e\u6587\u4ef6\u6240\u5728\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u6bd4\u5982 /etc/ 3\u3001\u4f7f\u7528 jindo \u547d\u4ee4\u8bbf\u95ee OSS-HDFS \u670d\u52a1 ./jindo fs -ls oss://<bucket>/<dir>","title":"bucket \u7ea7\u522b\u914d\u7f6e"},{"location":"backup/jindofs/jindo_util_intro/#jindo-sync","text":"1\u3001\u65b0\u5efa\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg [common] logger.dir = /tmp/jindo-util/ logger.sync = false logger.consolelogger = false logger.level = 0 logger.verbose = 0 logger.cleaner.enable = true hadoopConf.enable = false [jindosdk] fs.oss.endpoint = <ENDPOINT> fs.oss.accessKeyId = <ACCESS_KEYID> fs.oss.accessKeySecret = <ACCESS_KEYSECRET> < ENDPOINT>: \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 endpoint : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeyId : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u6216\u8005 OSS-HDFS \u7684 AccessKeySecret 2\u3001\u6dfb\u52a0\u73af\u5883\u53d8\u91cf export JINDOSDK_CONF_DIR=<JINDOSDK_CFG_DIR> : jindosdk.cfg \u914d\u7f6e\u6587\u4ef6\u6240\u5728\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u6bd4\u5982 /etc/ 3\u3001\u4f7f\u7528 jindo \u547d\u4ee4\u8fdb\u884c\u6570\u636e\u4f20\u8f93 ./jindo fs -sync -thread 10 /local/dir/ oss://<bucket>/<dir> thread: \u4f7f\u7528\u7ebf\u7a0b\u6570\u91cf 4\u3001\u4f7f\u7528\u7aef\u70b9\u7eed\u4f20\u529f\u80fd ./jindo fs -sync -update -thread 10 /local/dir/ oss://<bucket>/<dir> \u56e0\u4e3a\u9700\u8981\u505a\u6587\u4ef6\u6bd4\u8f83\uff0c\u4f7f\u7528\u65ad\u70b9\u7eed\u4f20\u53ef\u80fd\u5bf9\u4f20\u8f93\u6027\u80fd\u6709\u4e00\u5b9a\u7684\u5f71\u54cd","title":"\u4f7f\u7528 jindo sync \u540c\u6b65\u6570\u636e"},{"location":"backup/jindofs/jindofs_dls_checksum/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c \u00a7 (\u4ece 4.6.1 \u5f00\u59cb\u652f\u6301) \u901a\u8fc7 JindoDistCp \u4ece HDFS \u8fc1\u79fb\u5230 OSS-HDFS \u7684\u6587\u4ef6\uff0c\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u6821\u9a8c\u6587\u4ef6\u5185\u5bb9\u3002 \u6b65\u9aa41. \u00a7 \u8ba1\u7b97\u8f93\u51fa\u901a\u8fc7 JindoDistCp \u8fc1\u79fb\u7684\u76ee\u6807\u76ee\u5f55\u7684\u6587\u4ef6 checksum\u3002 \u547d\u4ee4\u683c\u5f0f \u00a7 jindo distjob -checksum --src <src> --dest <dest> --blockSize <blockSize> --recalculate \u53c2\u6570\u8bf4\u660e\uff1a --src <src> \uff1a\u5f85\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c\u7684 OSS-HDFS \u7684\u8def\u5f84\uff08\u5fc5\u586b\uff09\u3002\u6ce8\uff1asrc \u53ea\u80fd\u4e3a OSS-HDFS \u6216 HDFS \u8def\u5f84\u3002 --dest <dest> \uff1a\u8f93\u51fa checksum \u6587\u4ef6\u7684\u76ee\u6807\u8def\u5f84\uff08\u5fc5\u586b\uff09\u3002 --blockSize <blockSize> \uff1a\u6e90\u6587\u4ef6\u5199\u5165\u7684 blockSize\uff08\u53ef\u9009)\uff0c\u9ed8\u8ba4\u4e3a 134217728\u3002 --recalculate \uff1a\u662f\u5426\u8bfb\u53d6\u6587\u4ef6\u91cd\u65b0\u8ba1\u7b97 checksum\uff08\u53ef\u9009)\uff0c\u9ed8\u8ba4\u4e0d\u4f1a\u91cd\u65b0\u8ba1\u7b97\uff0c\u4ece OSS-HDFS \u8bfb\u53d6\u5199\u5165\u65f6\u8ba1\u7b97\u7684 checksum\u3002\u6ce8\uff1a\u8be5\u53c2\u6570\u53ea\u652f\u6301 --src \u4e3a OSS-HDFS\u3002 \u793a\u4f8b \u00a7 \u5047\u8bbe\u5728 OSS-HDFS \u7684 bucket \u540d\u4e3a dlsbucket\uff0c\u9700\u8981\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c\u7684\u76ee\u5f55\u4e3a oss://dlsbucket/test\uff0c\u8f93\u51fa checksum \u6587\u4ef6\u5230 OSS \u76ee\u5f55 oss://ossbucket/test-dls\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u547d\u4ee4\u4e3a\uff1a jindo distjob -checksum --src oss://dlsbucket/test --dest oss://ossbucket/test-dls --recalculate \u6b65\u9aa42. \u00a7 \u8ba1\u7b97\u8f93\u51fa JindoDistCp \u8fc1\u79fb\u7684\u6e90\u76ee\u5f55\u7684\u6587\u4ef6 checksum\u3002 \u793a\u4f8b \u00a7 \u5047\u8bbe JindoDistCp \u7684\u6e90\u76ee\u5f55\u4e3a hdfs:///test\uff0c\u8f93\u51fa checksum \u6587\u4ef6\u5230 OSS \u76ee\u5f55 oss://ossbucket/test-hdfs\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u547d\u4ee4\u4e3a\uff1a jindo distjob -checksum --src hdfs:///test --dest oss://ossbucket/test-hdfs \u6b65\u9aa43. \u00a7 \u901a\u8fc7 JindoDistCp --diff \u5bf9\u6bd4\u6b65\u9aa41\u548c\u6b65\u9aa42\u8f93\u51fa\u7684\u6587\u4ef6 checksum \u76ee\u5f55 \u793a\u4f8b \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src oss://ossbucket/test-hdfs --dest oss://ossbucket/test-dls --diff","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c"},{"location":"backup/jindofs/jindofs_dls_checksum/#oss-hdfs-jindofs","text":"(\u4ece 4.6.1 \u5f00\u59cb\u652f\u6301) \u901a\u8fc7 JindoDistCp \u4ece HDFS \u8fc1\u79fb\u5230 OSS-HDFS \u7684\u6587\u4ef6\uff0c\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u6821\u9a8c\u6587\u4ef6\u5185\u5bb9\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c"},{"location":"backup/jindofs/jindofs_dls_checksum/#1","text":"\u8ba1\u7b97\u8f93\u51fa\u901a\u8fc7 JindoDistCp \u8fc1\u79fb\u7684\u76ee\u6807\u76ee\u5f55\u7684\u6587\u4ef6 checksum\u3002","title":"\u6b65\u9aa41."},{"location":"backup/jindofs/jindofs_dls_checksum/#_1","text":"jindo distjob -checksum --src <src> --dest <dest> --blockSize <blockSize> --recalculate \u53c2\u6570\u8bf4\u660e\uff1a --src <src> \uff1a\u5f85\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c\u7684 OSS-HDFS \u7684\u8def\u5f84\uff08\u5fc5\u586b\uff09\u3002\u6ce8\uff1asrc \u53ea\u80fd\u4e3a OSS-HDFS \u6216 HDFS \u8def\u5f84\u3002 --dest <dest> \uff1a\u8f93\u51fa checksum \u6587\u4ef6\u7684\u76ee\u6807\u8def\u5f84\uff08\u5fc5\u586b\uff09\u3002 --blockSize <blockSize> \uff1a\u6e90\u6587\u4ef6\u5199\u5165\u7684 blockSize\uff08\u53ef\u9009)\uff0c\u9ed8\u8ba4\u4e3a 134217728\u3002 --recalculate \uff1a\u662f\u5426\u8bfb\u53d6\u6587\u4ef6\u91cd\u65b0\u8ba1\u7b97 checksum\uff08\u53ef\u9009)\uff0c\u9ed8\u8ba4\u4e0d\u4f1a\u91cd\u65b0\u8ba1\u7b97\uff0c\u4ece OSS-HDFS \u8bfb\u53d6\u5199\u5165\u65f6\u8ba1\u7b97\u7684 checksum\u3002\u6ce8\uff1a\u8be5\u53c2\u6570\u53ea\u652f\u6301 --src \u4e3a OSS-HDFS\u3002","title":"\u547d\u4ee4\u683c\u5f0f"},{"location":"backup/jindofs/jindofs_dls_checksum/#_2","text":"\u5047\u8bbe\u5728 OSS-HDFS \u7684 bucket \u540d\u4e3a dlsbucket\uff0c\u9700\u8981\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c\u7684\u76ee\u5f55\u4e3a oss://dlsbucket/test\uff0c\u8f93\u51fa checksum \u6587\u4ef6\u5230 OSS \u76ee\u5f55 oss://ossbucket/test-dls\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u547d\u4ee4\u4e3a\uff1a jindo distjob -checksum --src oss://dlsbucket/test --dest oss://ossbucket/test-dls --recalculate","title":"\u793a\u4f8b"},{"location":"backup/jindofs/jindofs_dls_checksum/#2","text":"\u8ba1\u7b97\u8f93\u51fa JindoDistCp \u8fc1\u79fb\u7684\u6e90\u76ee\u5f55\u7684\u6587\u4ef6 checksum\u3002","title":"\u6b65\u9aa42."},{"location":"backup/jindofs/jindofs_dls_checksum/#_3","text":"\u5047\u8bbe JindoDistCp \u7684\u6e90\u76ee\u5f55\u4e3a hdfs:///test\uff0c\u8f93\u51fa checksum \u6587\u4ef6\u5230 OSS \u76ee\u5f55 oss://ossbucket/test-hdfs\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u547d\u4ee4\u4e3a\uff1a jindo distjob -checksum --src hdfs:///test --dest oss://ossbucket/test-hdfs","title":"\u793a\u4f8b"},{"location":"backup/jindofs/jindofs_dls_checksum/#3","text":"\u901a\u8fc7 JindoDistCp --diff \u5bf9\u6bd4\u6b65\u9aa41\u548c\u6b65\u9aa42\u8f93\u51fa\u7684\u6587\u4ef6 checksum \u76ee\u5f55","title":"\u6b65\u9aa43."},{"location":"backup/jindofs/jindofs_dls_checksum/#_4","text":"hadoop jar jindo-distcp-tool-${version}.jar --src oss://ossbucket/test-hdfs --dest oss://ossbucket/test-dls --diff","title":"\u793a\u4f8b"},{"location":"backup/jindofs/jindofs_dls_dump_inventory_howto/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5143\u6570\u636e\u5bfc\u51fa\u4f7f\u7528\u8bf4\u660e \u00a7 (\u4ece 4.6.0 \u5f00\u59cb\u652f\u6301) \u4ecb\u7ecd \u00a7 \u4f7f\u7528\u5143\u6570\u636e\u5bfc\u51fa\u529f\u80fd\uff0c\u53ef\u4ee5\u5c06\u5f53\u524d OSS-HDFS bucket \u4e0b\u7684\u6587\u4ef6\u5143\u6570\u636e\u6e05\u5355\u5bfc\u51fa\u5230 OSS \u4e0a\u7684\uff0c\u683c\u5f0f\u4e3a json \u6587\u4ef6\uff0c\u65b9\u4fbf\u7528\u6237\u5bf9\u5143\u6570\u636e\u8fdb\u884c\u7edf\u8ba1\u5206\u6790 \u914d\u7f6e jindo \u547d\u4ee4\u884c\u5de5\u5177\uff0c\u914d\u7f6e\u5bf9\u5e94 OSS-HDFS bucket \u7684\u8bbf\u95ee\u5bc6\u94a5\uff0c\u53c2\u8003 jindo \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u8bf4\u660e \u6267\u884c\u5bfc\u51fa\u547d\u4ee4 jindo admin -dumpInventory oss://<hdfs_bucket>/ \u6b64\u65f6\u53ef\u4ee5\u89c2\u5bdf\u5230\u8f93\u51fa\u8def\u5f84 ============Dump Inventory============= Job Id: 2ce40fba-5704-45c4-8720-d92a891d5cfd Data Location: oss://<oss_bucket>/.dlsdata/.sysinfo/meta_analyze/inventory/1666584461201.2ce40fba-5704-45c4-8720-d92a891d5cfd ..................................................................................................................... FINISHED. \u8be5\u547d\u4ee4\u4e3a\u963b\u585e\u547d\u4ee4\uff0c\u8bf7\u8010\u5fc3\u7b49\u5f8510\u79d2\u949f~10\u5206\u949f\uff08\u6839\u636e\u5143\u6570\u636e\u91cf\u5927\u5c0f\uff09\uff0c\u77e5\u9053\u6700\u540e\u8f93\u51fa FINISHED \u8868\u793a\u5bfc\u51fa\u6210\u529f\u3002 \u4e0b\u8f7d\u7ed3\u679c\u6587\u4ef6 \u4f7f\u7528 jindo \u547d\u4ee4\u884c\u5de5\u5177 \uff08\u914d\u7f6e\u666e\u901aOSS\u7684key\u975eOSS-HDFS\uff09 \u3001\u6216 ossutil \u3001\u6216\u4f7f\u7528Hadoop fs \u547d\u4ee4\u3001\u6216\u5728OSS\u63a7\u5236\u53f0\u9875\u9762\uff0c\u4e0b\u8f7d\u7ed3\u679c\u6587\u4ef6\u3002 ossutil cp oss://<oss_bucket>/.dlsdata/.sysinfo/meta_analyze/inventory/1666584461201.2ce40fba-5704-45c4-8720-d92a891d5cfd ./ \u4e0b\u8f7d\u5230\u672c\u5730\uff0c\u4f7f\u7528vi/vim\u6253\u5f00\u5373\u53ef\u3002 \u793a\u4f8b\u7ed3\u679c\u53c2\u8003 {\"id\":16385,\"path\":\"/\",\"type\":\"directory\",\"size\":0,\"user\":\"admin\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1666581702933,\"permission\":511,\"state\":1} {\"id\":6246684106789500068,\"path\":\"/dls-1000326249\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500069,\"path\":\"/dls-1000326249/benchmark\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500070,\"path\":\"/dls-1000326249/benchmark/n1\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500071,\"path\":\"/dls-1000326249/benchmark/n1/490747449\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660895613953,\"permission\":511,\"state\":0}","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5143\u6570\u636e\u5bfc\u51fa\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindofs_dls_dump_inventory_howto/#oss-hdfs-jindofs","text":"(\u4ece 4.6.0 \u5f00\u59cb\u652f\u6301)","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5143\u6570\u636e\u5bfc\u51fa\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindofs_dls_dump_inventory_howto/#_1","text":"\u4f7f\u7528\u5143\u6570\u636e\u5bfc\u51fa\u529f\u80fd\uff0c\u53ef\u4ee5\u5c06\u5f53\u524d OSS-HDFS bucket \u4e0b\u7684\u6587\u4ef6\u5143\u6570\u636e\u6e05\u5355\u5bfc\u51fa\u5230 OSS \u4e0a\u7684\uff0c\u683c\u5f0f\u4e3a json \u6587\u4ef6\uff0c\u65b9\u4fbf\u7528\u6237\u5bf9\u5143\u6570\u636e\u8fdb\u884c\u7edf\u8ba1\u5206\u6790 \u914d\u7f6e jindo \u547d\u4ee4\u884c\u5de5\u5177\uff0c\u914d\u7f6e\u5bf9\u5e94 OSS-HDFS bucket \u7684\u8bbf\u95ee\u5bc6\u94a5\uff0c\u53c2\u8003 jindo \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u8bf4\u660e \u6267\u884c\u5bfc\u51fa\u547d\u4ee4 jindo admin -dumpInventory oss://<hdfs_bucket>/ \u6b64\u65f6\u53ef\u4ee5\u89c2\u5bdf\u5230\u8f93\u51fa\u8def\u5f84 ============Dump Inventory============= Job Id: 2ce40fba-5704-45c4-8720-d92a891d5cfd Data Location: oss://<oss_bucket>/.dlsdata/.sysinfo/meta_analyze/inventory/1666584461201.2ce40fba-5704-45c4-8720-d92a891d5cfd ..................................................................................................................... FINISHED. \u8be5\u547d\u4ee4\u4e3a\u963b\u585e\u547d\u4ee4\uff0c\u8bf7\u8010\u5fc3\u7b49\u5f8510\u79d2\u949f~10\u5206\u949f\uff08\u6839\u636e\u5143\u6570\u636e\u91cf\u5927\u5c0f\uff09\uff0c\u77e5\u9053\u6700\u540e\u8f93\u51fa FINISHED \u8868\u793a\u5bfc\u51fa\u6210\u529f\u3002 \u4e0b\u8f7d\u7ed3\u679c\u6587\u4ef6 \u4f7f\u7528 jindo \u547d\u4ee4\u884c\u5de5\u5177 \uff08\u914d\u7f6e\u666e\u901aOSS\u7684key\u975eOSS-HDFS\uff09 \u3001\u6216 ossutil \u3001\u6216\u4f7f\u7528Hadoop fs \u547d\u4ee4\u3001\u6216\u5728OSS\u63a7\u5236\u53f0\u9875\u9762\uff0c\u4e0b\u8f7d\u7ed3\u679c\u6587\u4ef6\u3002 ossutil cp oss://<oss_bucket>/.dlsdata/.sysinfo/meta_analyze/inventory/1666584461201.2ce40fba-5704-45c4-8720-d92a891d5cfd ./ \u4e0b\u8f7d\u5230\u672c\u5730\uff0c\u4f7f\u7528vi/vim\u6253\u5f00\u5373\u53ef\u3002 \u793a\u4f8b\u7ed3\u679c\u53c2\u8003 {\"id\":16385,\"path\":\"/\",\"type\":\"directory\",\"size\":0,\"user\":\"admin\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1666581702933,\"permission\":511,\"state\":1} {\"id\":6246684106789500068,\"path\":\"/dls-1000326249\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500069,\"path\":\"/dls-1000326249/benchmark\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500070,\"path\":\"/dls-1000326249/benchmark/n1\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500071,\"path\":\"/dls-1000326249/benchmark/n1/490747449\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660895613953,\"permission\":511,\"state\":0}","title":"\u4ecb\u7ecd"},{"location":"backup/jindofs/jindofs_dls_hsm_howto/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5206\u5c42\u5b58\u50a8\u4f7f\u7528\u8bf4\u660e \u00a7 (\u4ece 4.6.2 \u5f00\u59cb\u652f\u6301) \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u662f OSS \u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9HDFS\u63a5\u53e3, \u652f\u6301\u76ee\u5f55\u4ee5\u53ca\u76ee\u5f55\u5c42\u7ea7\uff0c\u901a\u8fc7 Jindo SDK 4.x \u53ef\u4ee5\u517c\u5bb9\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u3002\u5173\u4e8e\u7528\u6237\u5982\u4f55\u521b\u5efa\u548c\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u7684\u57fa\u672c\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 \u94fe\u63a5 \u3002 \u672c\u6587\u4e3b\u8981\u4ecb\u7ecd\u5728 OSS-HDFS \u670d\u52a1\u4e2d\u4f7f\u7528\u5206\u5c42\u5b58\u50a8\u529f\u80fd\u7684\u4e00\u4e9b\u5e38\u89c1\u64cd\u4f5c\u3002 \u4e3a\u4e86\u65b9\u4fbf\u4e0b\u9762\u547d\u4ee4\u7684\u4ecb\u7ecd\uff0c\u6211\u4eec\u5047\u8bbe oss://oss-dfs-test \u8fd9\u4e2aBucket\u5df2\u7ecf\u5f00\u542f OSS-HDFS \u670d\u52a1\u3002\u4e0b\u9762\u6240\u6709\u7684\u4f8b\u5b50\u90fd\u4f1a\u9488\u5bf9\u8fd9\u4e2aBucket\u8fdb\u884c\u64cd\u4f5c\u3002 \u5206\u5c42\u5b58\u50a8\u7b80\u4ecb \u00a7 \u6570\u636e\u6e56\u5b58\u50a8\u7684\u6570\u636e\u91cf\u5e38\u5e38\u4f1a\u968f\u7740\u65f6\u95f4\u800c\u4e0d\u65ad\u589e\u957f\uff0c\u4f46\u5728\u6570\u636e\u6e56\u5185\u7684\u6570\u636e\u5e76\u975e\u540c\u6837\u9891\u7e41\u8bbf\u95ee\u3002\u5bf9\u4e8e\u8f83\u8001\u7684\u6570\u636e\uff0c\u6bd4\u5982\u8d85\u8fc7\u4e00\u5e74\u7684\u6570\u636e\uff0c\u4e1a\u52a1\u4e0a\u53ef\u80fd\u51e0\u4e4e\u4e0d\u518d\u8bbf\u95ee\uff0c\u4f46\u4e3a\u4fdd\u8bc1\u5408\u89c4\u3001\u5b58\u6863\u7b49\u76ee\u7684\uff0c\u8fd9\u4e9b\u6570\u636e\u4ecd\u7136\u9700\u8981\u5b58\u50a8\u4ee5\u5907\u4e0d\u65f6\u4e4b\u9700\u3002\u56e0\u6b64\uff0c\u5206\u5c42\u5b58\u50a8\u7684\u9700\u6c42\u5c31\u51fa\u73b0\u4e86\uff1a\u5bf9\u4e8e\u8f83\u5c11\u8bbf\u95ee\u7684\u6570\u636e\uff0c\u9009\u62e9\u8f83\u4f4e\u6210\u672c\u7684\u5b58\u50a8\u4ecb\u8d28\uff08\u901a\u5e38\u4f1a\u6709\u66f4\u9ad8\u7684\u8bbf\u95ee\u6210\u672c\uff09\u8fdb\u884c\u5b58\u50a8\uff0c\u53ef\u4ee5\u964d\u4f4e\u603b\u5b58\u50a8\u6210\u672c\u3002 \u6807\u51c6 OSS Bucket \u652f\u6301\u591a\u79cd\u5206\u5c42\u5b58\u50a8\u7c7b\u578b\uff0c\u5305\u62ec\u6807\u51c6\u3001\u4f4e\u9891\u3001\u5f52\u6863\u3001\u51b7\u5f52\u6863\u56db\u79cd\uff0c\u800c JindoFS \u670d\u52a1\u4f7f\u7528\u6807\u51c6 OSS \u4f5c\u4e3a\u6570\u636e\u5757\u7684\u5b58\u50a8\u540e\u7aef\uff0c\u56e0\u6b64 JindoFS \u5206\u5c42\u5b58\u50a8\u529f\u80fd\u5373\u8bbe\u7f6e\u540e\u7aef\u6570\u636e\u5757\u5728\u6807\u51c6 OSS \u4e0a\u6240\u4f7f\u7528\u7684\u5b58\u50a8\u7c7b\u578b\u3002 \u4f7f\u7528\u5206\u5c42\u5b58\u50a8 \u00a7 \u5728\u4f7f\u7528\u76f8\u5173\u529f\u80fd\u524d\uff0c\u5e94\u786e\u4fdd JindoSDK \u5df2\u7ecf\u6b63\u786e\u914d\u7f6e\uff0c\u53ef\u4ee5\u6b63\u5e38\u8bbf\u95ee JindoFS \u670d\u52a1\u3002JindoFS \u670d\u52a1\u7684\u5206\u5c42\u5b58\u50a8\u529f\u80fd\u4e0e HCFS API \u517c\u5bb9\u3002\u76f8\u5173 API \u5982\u4e0b\u6240\u793a\uff1a public abstract class FileSystem extends Configured implements Closeable { // ...... public void setStoragePolicy(Path src, String policyName) throws IOException; public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException; public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies() throws IOException; } JindoFS \u4e5f\u652f\u6301\u4f7f\u7528\u4e0b\u5217\u547d\u4ee4\u64cd\u4f5c\u5b58\u50a8\u7b56\u7565\uff1a jindo fs -listPolicies jindo fs -setStoragePolicy -path <path> -policy <policy> jindo fs -getStoragePolicy -path <path> getAllStoragePolicies() \u63a5\u53e3\u548c listPolicies \u547d\u4ee4\u8fd4\u56de\u5f53\u524d\u652f\u6301\u7684\u5b58\u50a8\u7b56\u7565\uff0c\u76ee\u524d\u652f\u6301\u7684\u5b58\u50a8\u7b56\u7565\u6709 CLOUD_STD \uff08\u6807\u51c6\u5b58\u50a8\uff09\u3001 CLOUD_IA \uff08\u4f4e\u9891\u5b58\u50a8\uff09\u3001 CLOUD_AR \uff08\u5f52\u6863\uff09\u548c CLOUD_COLD_AR \uff08\u51b7\u5f52\u6863\uff09\u3002 setStoragePolicy() \u63a5\u53e3\u548c setStoragePolicy \u547d\u4ee4\u53ef\u4ee5\u4e3a\u5177\u4f53\u7684\u8def\u5f84\u8bbe\u7f6e\u5b58\u50a8\u7b56\u7565\uff0cJindoFS \u670d\u52a1\u4f1a\u5728\u540e\u53f0\u6839\u636e\u5b58\u50a8\u7b56\u7565\u6539\u53d8\u5b58\u50a8\u5bf9\u8c61\u7684\u5b58\u50a8\u7ea7\u522b\u3002\u4f8b\u5982\uff0c\u8981\u5c06 oss://oss-dfs-test/dir1 \u8bbe\u7f6e\u4e3a\u5f52\u6863\u7c7b\u578b\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a jindo fs -setStoragePolicy -path oss://oss-dfs-test/dir1 -policy CLOUD_AR \u6ce8\u610f\u6587\u4ef6\u6216\u76ee\u5f55\u7684\u9ed8\u8ba4\u5b58\u50a8\u7b56\u7565\u4e3a\u7a7a\uff0c\u663e\u793a\u4e3a UNSPECIFIED \uff0c\u5f53\u5bf9\u8c61\u5b58\u50a8\u7b56\u7565\u4e3a UNSPECIFIED \u65f6\uff0c\u5bf9\u8c61\u5b9e\u9645\u7684\u5b58\u50a8\u7b56\u7565\u5c06\u7531\u6700\u8fd1\u7684\u8bbe\u7f6e\u4e86\u5b58\u50a8\u7b56\u7565\u7684\u7236\u8282\u70b9\u51b3\u5b9a\u3002 \u4f8b\u5982\u5047\u8bbe oss://oss-dfs-test/dir2 \u7684\u5b58\u50a8\u7b56\u7565\u4e3a CLOUD_STD \uff0c\u5219 oss://oss-dfs-test/dir2/subdir2/subsubdir2 \u7684\u5b58\u50a8\u7b56\u7565\u4e5f\u4e3a CLOUD_STD \uff0c \u9664\u975e\u7528\u6237\u5bf9 oss://oss-dfs-test/dir2/subdir2/ \u6216 oss://oss-dfs-test/dir2/subdir2/subsubdir2 \u5355\u72ec\u8bbe\u7f6e\u8fc7\u5b58\u50a8\u7b56\u7565\u3002 \u7528\u6237\u53ef\u4ee5\u901a\u8fc7 getStoragePolicy() \u63a5\u53e3\u548c getStoragePolicy \u547d\u4ee4\u67e5\u8be2\u67d0\u4e00\u8def\u5f84\u6700\u7ec8\u751f\u6548\u7684\u5b58\u50a8\u7b56\u7565\uff0c\u5982\uff1a jindo fs -getStoragePolicy -path oss://oss-dfs-test/dir1/file1 \u5982\u679c\u6700\u7ec8\u751f\u6548\u7684\u5b58\u50a8\u7b56\u7565\u4e3a UNSPECIFIED \uff08\u5373 getStoragePolicy() \u63a5\u53e3\u8fd4\u56de null \uff09\uff0c\u5219\u5bf9\u8c61\u9ed8\u8ba4\u4f7f\u7528\u6807\u51c6\u5b58\u50a8\u3002 \u76ee\u5f55\u5d4c\u5957 \u00a7 \u76ee\u524d JindoFS \u4e0d\u652f\u6301\u5d4c\u5957\u7b56\u7565 , \u4f8b\u5982\u5bf9 oss://oss-dfs-test/warehouse/dwd.db/dt=20220101 \u4f7f\u7528\u4e86 CLOUD_AR \u7b56\u7565, \u518d\u5bf9 oss://oss-dfs-test/warehouse/dwd.db \u7236\u76ee\u5f55\u4f7f\u7528 CLOUD_STD \u7b56\u7565, \u662f\u4e0d\u4f1a\u6267\u884c\u89e3\u5f52\u6863\u64cd\u4f5c\u7684. \u56e0\u6b64\u5efa\u8bae \u53ea\u5bf9\u4e4b\u524d\u8bbe\u7f6e\u8fc7policy\u7684\u76ee\u5f55\u4fee\u6539policy, \u4e0d\u8981\u518d\u4fee\u6539\u5176\u4e0a\u4e0b\u7ea7\u76ee\u5f55\u7684policy. \u5404\u79cd\u573a\u666f\u4e0b Policy \u4e92\u8f6c\u7684\u4f8b\u5b50 \u00a7 \u6807\u51c6 \u8f6c \u4f4e\u9891IA jindo fs -setStoragePolicy -path <path> -policy CLOUD_IA \u6807\u51c6 \u8f6c \u5f52\u6863CLOUD_AR jindo fs -setStoragePolicy -path <path> -policy CLOUD_AR \u6807\u51c6 \u8f6c \u51b7\u5f52\u6863CLOUD_COLD_AR jindo fs -setStoragePolicy -path <path> -policy CLOUD_COLD_AR \u4f4e\u9891IA \u8f6c \u5f52\u6863CLOUD_AR jindo fs -setStoragePolicy -path <path> -policy CLOUD_AR \u4f4e\u9891IA \u8f6c \u51b7\u5f52\u6863CLOUD_COLD_AR jindo fs -setStoragePolicy -path <path> -policy CLOUD_COLD_AR \u4f4e\u9891IA \u8f6c \u6807\u51c6 jindo fs -setStoragePolicy -path <path> -policy CLOUD_STD \u5f52\u6863 \u8f6c \u51b7\u5f52\u6863 \u4e0d\u652f\u6301 \u51b7\u5f52\u6863 \u8f6c \u5f52\u6863 \u4e0d\u652f\u6301 \u5f52\u6863 \u8f6c \u6807\u51c6 jindo fs -setStoragePolicy -path <path> -policy CLOUD_STD \u51b7\u5f52\u6863 \u8f6c \u6807\u51c6 jindo fs -setStoragePolicy -path <path> -policy CLOUD_STD \u8bbe\u7f6eOSS\u751f\u547d\u5468\u671f \u00a7 \u5728\u4f7f\u7528\u5206\u5c42\u5b58\u50a8\u529f\u80fd\u5c06OSS-HDFS\u7684\u6570\u636e\u8fdb\u884c\u5f52\u6863\u7684\u65f6\u5019\uff0c\u9700\u8981\u4f9d\u8d56\u4e8eOSS\u751f\u547d\u5468\u671f\u529f\u80fd\uff0c\u901a\u8fc7\u4e0b\u9762\u547d\u4ee4 jindo fs -setStoragePolicy -path oss://oss-dfs-test/dir1 -policy CLOUD_AR \u8fdb\u884c\u6570\u636e\u5f52\u6863\u64cd\u4f5c\uff0c\u8be5\u64cd\u4f5c\u53ea\u4f1a\u5c06\u5bf9\u5e94\u76ee\u5f55\u4e0b\u7684\u6570\u636e\u8fdb\u884cTag\u6807\u8bb0, Tag\u5bf9\u5e94\u7684\u952e\u503c\u5bf9\u4e3atransition-storage-class:Archive\uff0c\u9700\u8981\u8bbe\u7f6e\u751f\u547d\u5468\u671f\u8fdb\u884c\u6570\u636e\u7684\u8f6c\u6362\uff0c\u8bbe\u7f6e\u6309\u524d\u7f00\u5339\u914d\u7b56\u7565\u89c4\u5219\uff0c\u524d\u7f00\u4e3a.dlsdata, \u5177\u4f53\u5f52\u6863\u7684\u751f\u547d\u5468\u671f\u7b56\u7565\u8bbe\u7f6e\u5982\u4e0b\uff1a \u4e0a\u8ff0\u751f\u547d\u5468\u671f\u7b56\u7565\u8bbe\u7f6e\u5b8c\u6bd5\u540e\uff0c\u7528\u6237\u9700\u8981\u7b49\u5f85\u6539OSS\u751f\u547d\u5468\u671f\u7b56\u7565\u88ab\u8c03\u5ea6\u6267\u884c\u624d\u80fd\u5b8c\u6210\u6570\u636e\u7684\u5f52\u6863\u3002\u5982\u4f55\u8bbe\u7f6e\u751f\u547d\u5468\u671f\u53ef\u4ee5\u53c2\u8003 \u6587\u6863 \u3002 \u6ce8\u610f\u4e8b\u9879 \u00a7 \u5982\u679c\u9700\u8981\u4f7f\u7528\u5f52\u6863/\u51b7\u5f52\u6863\u7684\u5b58\u50a8\u7b56\u7565\uff0c\u9700\u8981\u5728 OSS \u63a7\u5236\u53f0\u4e0a\u914d\u7f6e\u76f8\u5e94\u7684\u751f\u547d\u5468\u671f\u7b56\u7565\uff0c\u5426\u5219\u5b58\u50a8\u7684\u6570\u636e\u5757\u65e0\u6cd5\u771f\u6b63\u8f6c\u4e3a\u5f52\u6863/\u51b7\u5f52\u6863\u7c7b\u578b\u3002 \u5728 JindoFS \u4e0a\u67e5\u8be2\u5230\u7684\u5b58\u50a8\u72b6\u6001\u4ec5\u8868\u793a JindoFS \u5df2\u6309\u9884\u8bbe\u7684\u751f\u547d\u5468\u671f\u7b56\u7565\u4e0b\u53d1\u5230\u5bf9\u5e94\u7684 OSS \u6570\u636e\u5757\uff0c\u4e0d\u8868\u793a\u6570\u636e\u5757\u5b9e\u9645\u7684\u5b58\u50a8\u7c7b\u578b\u5df2\u8f6c\u53d8\u5b8c\u6210\u3002 OSS \u7684\u751f\u547d\u5468\u671f\u7ba1\u7406\u8fd0\u884c\u5468\u671f\u8f83\u957f\uff0c\u8bbe\u7f6e\u5f52\u6863\u5b58\u50a8\u7b56\u7565\u7684\u5bf9\u8c61\u53ef\u80fd\u9700\u8981\u6700\u957f24\u5c0f\u65f6\u5de6\u53f3\u624d\u4f1a\u771f\u6b63\u8f6c\u4e3a\u5f52\u6863/\u51b7\u5f52\u6863\u7c7b\u578b\u3002 \u5f52\u6863/\u51b7\u5f52\u6863\u7c7b\u578b\u7684\u6570\u636e\u65e0\u6cd5\u8bbf\u95ee\uff0c\u9700\u8981\u8f6c\u56de\u6807\u51c6\u5b58\u50a8\u65b9\u53ef\u8bbf\u95ee\u3002\u8f6c\u56de\u6807\u51c6\u5b58\u50a8\u53ef\u80fd\u9700\u8981\u4e00\u5b9a\u65f6\u95f4\uff0c\u53ef\u4ee5\u901a\u8fc7 checkStoragePolicy \u547d\u4ee4\u67e5\u8be2\u6267\u884c\u72b6\u6001\u3002 \u53d6\u56de\u5f52\u6863/\u51b7\u5f52\u6863\u6570\u636e\u4f1a\u4ea7\u751f\u4e00\u4e9b\u989d\u5916\u8d39\u7528\uff0c\u5e94\u5f53\u5c3d\u91cf\u907f\u514d\u5c06\u8fd8\u9700\u8bbf\u95ee\u7684\u6570\u636e\u8f6c\u4e3a\u5f52\u6863/\u51b7\u5f52\u6863\u7c7b\u578b\u3002 \u5bf9\u4e8e\u5b58\u50a8\u7b56\u7565\u4e0d\u4e00\u81f4\u7684\u76ee\u5f55\u95f4\u7684\u6587\u4ef6 rename \u64cd\u4f5c\uff0crename \u7684\u540c\u65f6\u4f1a\u81ea\u52a8\u751f\u6210\u76f8\u5e94\u7684\u540e\u53f0\u4efb\u52a1\uff0c\u4f7f rename \u540e\u7684\u6587\u4ef6\u6700\u7ec8\u7b26\u5408\u76ee\u6807\u8def\u5f84\u7684\u5b58\u50a8\u7b56\u7565\u3002 \u5f53\u524d\u7248\u672c\u4e0d\u652f\u6301\u5728\u8bbe\u7f6e\u4e3a\u4f4e\u9891/\u5f52\u6863/\u51b7\u5f52\u6863\u5b58\u50a8\u7b56\u7565\u7684\u76ee\u5f55\u4e0b\u521b\u5efa\u6587\u4ef6\u3002\u82e5\u8981\u5728\u4f4e\u9891/\u5f52\u6863/\u51b7\u5f52\u6863\u7684\u76ee\u5f55\u4e0b\u6dfb\u52a0\u6587\u4ef6\uff0c\u53ef\u4ee5\u5728\u6807\u51c6\u76ee\u5f55\u4e0b\u521b\u5efa\u5e76\u5173\u95ed\u6587\u4ef6\u540e\uff0crename \u5230\u76ee\u6807\u7684\u4f4e\u9891/\u5f52\u6863/\u51b7\u5f52\u6863\u76ee\u5f55\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5206\u5c42\u5b58\u50a8\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindofs_dls_hsm_howto/#oss-hdfs-jindofs","text":"(\u4ece 4.6.2 \u5f00\u59cb\u652f\u6301) \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u662f OSS \u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9HDFS\u63a5\u53e3, \u652f\u6301\u76ee\u5f55\u4ee5\u53ca\u76ee\u5f55\u5c42\u7ea7\uff0c\u901a\u8fc7 Jindo SDK 4.x \u53ef\u4ee5\u517c\u5bb9\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u3002\u5173\u4e8e\u7528\u6237\u5982\u4f55\u521b\u5efa\u548c\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u7684\u57fa\u672c\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 \u94fe\u63a5 \u3002 \u672c\u6587\u4e3b\u8981\u4ecb\u7ecd\u5728 OSS-HDFS \u670d\u52a1\u4e2d\u4f7f\u7528\u5206\u5c42\u5b58\u50a8\u529f\u80fd\u7684\u4e00\u4e9b\u5e38\u89c1\u64cd\u4f5c\u3002 \u4e3a\u4e86\u65b9\u4fbf\u4e0b\u9762\u547d\u4ee4\u7684\u4ecb\u7ecd\uff0c\u6211\u4eec\u5047\u8bbe oss://oss-dfs-test \u8fd9\u4e2aBucket\u5df2\u7ecf\u5f00\u542f OSS-HDFS \u670d\u52a1\u3002\u4e0b\u9762\u6240\u6709\u7684\u4f8b\u5b50\u90fd\u4f1a\u9488\u5bf9\u8fd9\u4e2aBucket\u8fdb\u884c\u64cd\u4f5c\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5206\u5c42\u5b58\u50a8\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindofs_dls_hsm_howto/#_1","text":"\u6570\u636e\u6e56\u5b58\u50a8\u7684\u6570\u636e\u91cf\u5e38\u5e38\u4f1a\u968f\u7740\u65f6\u95f4\u800c\u4e0d\u65ad\u589e\u957f\uff0c\u4f46\u5728\u6570\u636e\u6e56\u5185\u7684\u6570\u636e\u5e76\u975e\u540c\u6837\u9891\u7e41\u8bbf\u95ee\u3002\u5bf9\u4e8e\u8f83\u8001\u7684\u6570\u636e\uff0c\u6bd4\u5982\u8d85\u8fc7\u4e00\u5e74\u7684\u6570\u636e\uff0c\u4e1a\u52a1\u4e0a\u53ef\u80fd\u51e0\u4e4e\u4e0d\u518d\u8bbf\u95ee\uff0c\u4f46\u4e3a\u4fdd\u8bc1\u5408\u89c4\u3001\u5b58\u6863\u7b49\u76ee\u7684\uff0c\u8fd9\u4e9b\u6570\u636e\u4ecd\u7136\u9700\u8981\u5b58\u50a8\u4ee5\u5907\u4e0d\u65f6\u4e4b\u9700\u3002\u56e0\u6b64\uff0c\u5206\u5c42\u5b58\u50a8\u7684\u9700\u6c42\u5c31\u51fa\u73b0\u4e86\uff1a\u5bf9\u4e8e\u8f83\u5c11\u8bbf\u95ee\u7684\u6570\u636e\uff0c\u9009\u62e9\u8f83\u4f4e\u6210\u672c\u7684\u5b58\u50a8\u4ecb\u8d28\uff08\u901a\u5e38\u4f1a\u6709\u66f4\u9ad8\u7684\u8bbf\u95ee\u6210\u672c\uff09\u8fdb\u884c\u5b58\u50a8\uff0c\u53ef\u4ee5\u964d\u4f4e\u603b\u5b58\u50a8\u6210\u672c\u3002 \u6807\u51c6 OSS Bucket \u652f\u6301\u591a\u79cd\u5206\u5c42\u5b58\u50a8\u7c7b\u578b\uff0c\u5305\u62ec\u6807\u51c6\u3001\u4f4e\u9891\u3001\u5f52\u6863\u3001\u51b7\u5f52\u6863\u56db\u79cd\uff0c\u800c JindoFS \u670d\u52a1\u4f7f\u7528\u6807\u51c6 OSS \u4f5c\u4e3a\u6570\u636e\u5757\u7684\u5b58\u50a8\u540e\u7aef\uff0c\u56e0\u6b64 JindoFS \u5206\u5c42\u5b58\u50a8\u529f\u80fd\u5373\u8bbe\u7f6e\u540e\u7aef\u6570\u636e\u5757\u5728\u6807\u51c6 OSS \u4e0a\u6240\u4f7f\u7528\u7684\u5b58\u50a8\u7c7b\u578b\u3002","title":"\u5206\u5c42\u5b58\u50a8\u7b80\u4ecb"},{"location":"backup/jindofs/jindofs_dls_hsm_howto/#_2","text":"\u5728\u4f7f\u7528\u76f8\u5173\u529f\u80fd\u524d\uff0c\u5e94\u786e\u4fdd JindoSDK \u5df2\u7ecf\u6b63\u786e\u914d\u7f6e\uff0c\u53ef\u4ee5\u6b63\u5e38\u8bbf\u95ee JindoFS \u670d\u52a1\u3002JindoFS \u670d\u52a1\u7684\u5206\u5c42\u5b58\u50a8\u529f\u80fd\u4e0e HCFS API \u517c\u5bb9\u3002\u76f8\u5173 API \u5982\u4e0b\u6240\u793a\uff1a public abstract class FileSystem extends Configured implements Closeable { // ...... public void setStoragePolicy(Path src, String policyName) throws IOException; public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException; public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies() throws IOException; } JindoFS \u4e5f\u652f\u6301\u4f7f\u7528\u4e0b\u5217\u547d\u4ee4\u64cd\u4f5c\u5b58\u50a8\u7b56\u7565\uff1a jindo fs -listPolicies jindo fs -setStoragePolicy -path <path> -policy <policy> jindo fs -getStoragePolicy -path <path> getAllStoragePolicies() \u63a5\u53e3\u548c listPolicies \u547d\u4ee4\u8fd4\u56de\u5f53\u524d\u652f\u6301\u7684\u5b58\u50a8\u7b56\u7565\uff0c\u76ee\u524d\u652f\u6301\u7684\u5b58\u50a8\u7b56\u7565\u6709 CLOUD_STD \uff08\u6807\u51c6\u5b58\u50a8\uff09\u3001 CLOUD_IA \uff08\u4f4e\u9891\u5b58\u50a8\uff09\u3001 CLOUD_AR \uff08\u5f52\u6863\uff09\u548c CLOUD_COLD_AR \uff08\u51b7\u5f52\u6863\uff09\u3002 setStoragePolicy() \u63a5\u53e3\u548c setStoragePolicy \u547d\u4ee4\u53ef\u4ee5\u4e3a\u5177\u4f53\u7684\u8def\u5f84\u8bbe\u7f6e\u5b58\u50a8\u7b56\u7565\uff0cJindoFS \u670d\u52a1\u4f1a\u5728\u540e\u53f0\u6839\u636e\u5b58\u50a8\u7b56\u7565\u6539\u53d8\u5b58\u50a8\u5bf9\u8c61\u7684\u5b58\u50a8\u7ea7\u522b\u3002\u4f8b\u5982\uff0c\u8981\u5c06 oss://oss-dfs-test/dir1 \u8bbe\u7f6e\u4e3a\u5f52\u6863\u7c7b\u578b\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a jindo fs -setStoragePolicy -path oss://oss-dfs-test/dir1 -policy CLOUD_AR \u6ce8\u610f\u6587\u4ef6\u6216\u76ee\u5f55\u7684\u9ed8\u8ba4\u5b58\u50a8\u7b56\u7565\u4e3a\u7a7a\uff0c\u663e\u793a\u4e3a UNSPECIFIED \uff0c\u5f53\u5bf9\u8c61\u5b58\u50a8\u7b56\u7565\u4e3a UNSPECIFIED \u65f6\uff0c\u5bf9\u8c61\u5b9e\u9645\u7684\u5b58\u50a8\u7b56\u7565\u5c06\u7531\u6700\u8fd1\u7684\u8bbe\u7f6e\u4e86\u5b58\u50a8\u7b56\u7565\u7684\u7236\u8282\u70b9\u51b3\u5b9a\u3002 \u4f8b\u5982\u5047\u8bbe oss://oss-dfs-test/dir2 \u7684\u5b58\u50a8\u7b56\u7565\u4e3a CLOUD_STD \uff0c\u5219 oss://oss-dfs-test/dir2/subdir2/subsubdir2 \u7684\u5b58\u50a8\u7b56\u7565\u4e5f\u4e3a CLOUD_STD \uff0c \u9664\u975e\u7528\u6237\u5bf9 oss://oss-dfs-test/dir2/subdir2/ \u6216 oss://oss-dfs-test/dir2/subdir2/subsubdir2 \u5355\u72ec\u8bbe\u7f6e\u8fc7\u5b58\u50a8\u7b56\u7565\u3002 \u7528\u6237\u53ef\u4ee5\u901a\u8fc7 getStoragePolicy() \u63a5\u53e3\u548c getStoragePolicy \u547d\u4ee4\u67e5\u8be2\u67d0\u4e00\u8def\u5f84\u6700\u7ec8\u751f\u6548\u7684\u5b58\u50a8\u7b56\u7565\uff0c\u5982\uff1a jindo fs -getStoragePolicy -path oss://oss-dfs-test/dir1/file1 \u5982\u679c\u6700\u7ec8\u751f\u6548\u7684\u5b58\u50a8\u7b56\u7565\u4e3a UNSPECIFIED \uff08\u5373 getStoragePolicy() \u63a5\u53e3\u8fd4\u56de null \uff09\uff0c\u5219\u5bf9\u8c61\u9ed8\u8ba4\u4f7f\u7528\u6807\u51c6\u5b58\u50a8\u3002","title":"\u4f7f\u7528\u5206\u5c42\u5b58\u50a8"},{"location":"backup/jindofs/jindofs_dls_hsm_howto/#_3","text":"\u76ee\u524d JindoFS \u4e0d\u652f\u6301\u5d4c\u5957\u7b56\u7565 , \u4f8b\u5982\u5bf9 oss://oss-dfs-test/warehouse/dwd.db/dt=20220101 \u4f7f\u7528\u4e86 CLOUD_AR \u7b56\u7565, \u518d\u5bf9 oss://oss-dfs-test/warehouse/dwd.db \u7236\u76ee\u5f55\u4f7f\u7528 CLOUD_STD \u7b56\u7565, \u662f\u4e0d\u4f1a\u6267\u884c\u89e3\u5f52\u6863\u64cd\u4f5c\u7684. \u56e0\u6b64\u5efa\u8bae \u53ea\u5bf9\u4e4b\u524d\u8bbe\u7f6e\u8fc7policy\u7684\u76ee\u5f55\u4fee\u6539policy, \u4e0d\u8981\u518d\u4fee\u6539\u5176\u4e0a\u4e0b\u7ea7\u76ee\u5f55\u7684policy.","title":"\u76ee\u5f55\u5d4c\u5957"},{"location":"backup/jindofs/jindofs_dls_hsm_howto/#policy","text":"\u6807\u51c6 \u8f6c \u4f4e\u9891IA jindo fs -setStoragePolicy -path <path> -policy CLOUD_IA \u6807\u51c6 \u8f6c \u5f52\u6863CLOUD_AR jindo fs -setStoragePolicy -path <path> -policy CLOUD_AR \u6807\u51c6 \u8f6c \u51b7\u5f52\u6863CLOUD_COLD_AR jindo fs -setStoragePolicy -path <path> -policy CLOUD_COLD_AR \u4f4e\u9891IA \u8f6c \u5f52\u6863CLOUD_AR jindo fs -setStoragePolicy -path <path> -policy CLOUD_AR \u4f4e\u9891IA \u8f6c \u51b7\u5f52\u6863CLOUD_COLD_AR jindo fs -setStoragePolicy -path <path> -policy CLOUD_COLD_AR \u4f4e\u9891IA \u8f6c \u6807\u51c6 jindo fs -setStoragePolicy -path <path> -policy CLOUD_STD \u5f52\u6863 \u8f6c \u51b7\u5f52\u6863 \u4e0d\u652f\u6301 \u51b7\u5f52\u6863 \u8f6c \u5f52\u6863 \u4e0d\u652f\u6301 \u5f52\u6863 \u8f6c \u6807\u51c6 jindo fs -setStoragePolicy -path <path> -policy CLOUD_STD \u51b7\u5f52\u6863 \u8f6c \u6807\u51c6 jindo fs -setStoragePolicy -path <path> -policy CLOUD_STD","title":"\u5404\u79cd\u573a\u666f\u4e0b Policy \u4e92\u8f6c\u7684\u4f8b\u5b50"},{"location":"backup/jindofs/jindofs_dls_hsm_howto/#oss","text":"\u5728\u4f7f\u7528\u5206\u5c42\u5b58\u50a8\u529f\u80fd\u5c06OSS-HDFS\u7684\u6570\u636e\u8fdb\u884c\u5f52\u6863\u7684\u65f6\u5019\uff0c\u9700\u8981\u4f9d\u8d56\u4e8eOSS\u751f\u547d\u5468\u671f\u529f\u80fd\uff0c\u901a\u8fc7\u4e0b\u9762\u547d\u4ee4 jindo fs -setStoragePolicy -path oss://oss-dfs-test/dir1 -policy CLOUD_AR \u8fdb\u884c\u6570\u636e\u5f52\u6863\u64cd\u4f5c\uff0c\u8be5\u64cd\u4f5c\u53ea\u4f1a\u5c06\u5bf9\u5e94\u76ee\u5f55\u4e0b\u7684\u6570\u636e\u8fdb\u884cTag\u6807\u8bb0, Tag\u5bf9\u5e94\u7684\u952e\u503c\u5bf9\u4e3atransition-storage-class:Archive\uff0c\u9700\u8981\u8bbe\u7f6e\u751f\u547d\u5468\u671f\u8fdb\u884c\u6570\u636e\u7684\u8f6c\u6362\uff0c\u8bbe\u7f6e\u6309\u524d\u7f00\u5339\u914d\u7b56\u7565\u89c4\u5219\uff0c\u524d\u7f00\u4e3a.dlsdata, \u5177\u4f53\u5f52\u6863\u7684\u751f\u547d\u5468\u671f\u7b56\u7565\u8bbe\u7f6e\u5982\u4e0b\uff1a \u4e0a\u8ff0\u751f\u547d\u5468\u671f\u7b56\u7565\u8bbe\u7f6e\u5b8c\u6bd5\u540e\uff0c\u7528\u6237\u9700\u8981\u7b49\u5f85\u6539OSS\u751f\u547d\u5468\u671f\u7b56\u7565\u88ab\u8c03\u5ea6\u6267\u884c\u624d\u80fd\u5b8c\u6210\u6570\u636e\u7684\u5f52\u6863\u3002\u5982\u4f55\u8bbe\u7f6e\u751f\u547d\u5468\u671f\u53ef\u4ee5\u53c2\u8003 \u6587\u6863 \u3002","title":"\u8bbe\u7f6eOSS\u751f\u547d\u5468\u671f"},{"location":"backup/jindofs/jindofs_dls_hsm_howto/#_4","text":"\u5982\u679c\u9700\u8981\u4f7f\u7528\u5f52\u6863/\u51b7\u5f52\u6863\u7684\u5b58\u50a8\u7b56\u7565\uff0c\u9700\u8981\u5728 OSS \u63a7\u5236\u53f0\u4e0a\u914d\u7f6e\u76f8\u5e94\u7684\u751f\u547d\u5468\u671f\u7b56\u7565\uff0c\u5426\u5219\u5b58\u50a8\u7684\u6570\u636e\u5757\u65e0\u6cd5\u771f\u6b63\u8f6c\u4e3a\u5f52\u6863/\u51b7\u5f52\u6863\u7c7b\u578b\u3002 \u5728 JindoFS \u4e0a\u67e5\u8be2\u5230\u7684\u5b58\u50a8\u72b6\u6001\u4ec5\u8868\u793a JindoFS \u5df2\u6309\u9884\u8bbe\u7684\u751f\u547d\u5468\u671f\u7b56\u7565\u4e0b\u53d1\u5230\u5bf9\u5e94\u7684 OSS \u6570\u636e\u5757\uff0c\u4e0d\u8868\u793a\u6570\u636e\u5757\u5b9e\u9645\u7684\u5b58\u50a8\u7c7b\u578b\u5df2\u8f6c\u53d8\u5b8c\u6210\u3002 OSS \u7684\u751f\u547d\u5468\u671f\u7ba1\u7406\u8fd0\u884c\u5468\u671f\u8f83\u957f\uff0c\u8bbe\u7f6e\u5f52\u6863\u5b58\u50a8\u7b56\u7565\u7684\u5bf9\u8c61\u53ef\u80fd\u9700\u8981\u6700\u957f24\u5c0f\u65f6\u5de6\u53f3\u624d\u4f1a\u771f\u6b63\u8f6c\u4e3a\u5f52\u6863/\u51b7\u5f52\u6863\u7c7b\u578b\u3002 \u5f52\u6863/\u51b7\u5f52\u6863\u7c7b\u578b\u7684\u6570\u636e\u65e0\u6cd5\u8bbf\u95ee\uff0c\u9700\u8981\u8f6c\u56de\u6807\u51c6\u5b58\u50a8\u65b9\u53ef\u8bbf\u95ee\u3002\u8f6c\u56de\u6807\u51c6\u5b58\u50a8\u53ef\u80fd\u9700\u8981\u4e00\u5b9a\u65f6\u95f4\uff0c\u53ef\u4ee5\u901a\u8fc7 checkStoragePolicy \u547d\u4ee4\u67e5\u8be2\u6267\u884c\u72b6\u6001\u3002 \u53d6\u56de\u5f52\u6863/\u51b7\u5f52\u6863\u6570\u636e\u4f1a\u4ea7\u751f\u4e00\u4e9b\u989d\u5916\u8d39\u7528\uff0c\u5e94\u5f53\u5c3d\u91cf\u907f\u514d\u5c06\u8fd8\u9700\u8bbf\u95ee\u7684\u6570\u636e\u8f6c\u4e3a\u5f52\u6863/\u51b7\u5f52\u6863\u7c7b\u578b\u3002 \u5bf9\u4e8e\u5b58\u50a8\u7b56\u7565\u4e0d\u4e00\u81f4\u7684\u76ee\u5f55\u95f4\u7684\u6587\u4ef6 rename \u64cd\u4f5c\uff0crename \u7684\u540c\u65f6\u4f1a\u81ea\u52a8\u751f\u6210\u76f8\u5e94\u7684\u540e\u53f0\u4efb\u52a1\uff0c\u4f7f rename \u540e\u7684\u6587\u4ef6\u6700\u7ec8\u7b26\u5408\u76ee\u6807\u8def\u5f84\u7684\u5b58\u50a8\u7b56\u7565\u3002 \u5f53\u524d\u7248\u672c\u4e0d\u652f\u6301\u5728\u8bbe\u7f6e\u4e3a\u4f4e\u9891/\u5f52\u6863/\u51b7\u5f52\u6863\u5b58\u50a8\u7b56\u7565\u7684\u76ee\u5f55\u4e0b\u521b\u5efa\u6587\u4ef6\u3002\u82e5\u8981\u5728\u4f4e\u9891/\u5f52\u6863/\u51b7\u5f52\u6863\u7684\u76ee\u5f55\u4e0b\u6dfb\u52a0\u6587\u4ef6\uff0c\u53ef\u4ee5\u5728\u6807\u51c6\u76ee\u5f55\u4e0b\u521b\u5efa\u5e76\u5173\u95ed\u6587\u4ef6\u540e\uff0crename \u5230\u76ee\u6807\u7684\u4f4e\u9891/\u5f52\u6863/\u51b7\u5f52\u6863\u76ee\u5f55\u3002","title":"\u6ce8\u610f\u4e8b\u9879"},{"location":"backup/jindofs/jindofs_dls_migration_import/","text":"\u534a\u6258\u7ba1 JindoFS \u8fc1\u79fb\u5230\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 (\u4ece 4.5.0 \u5f00\u59cb\u652f\u6301) 0.\u80cc\u666f \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u662f OSS \u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9HDFS\u63a5\u53e3, \u652f\u6301\u76ee\u5f55\u4ee5\u53ca\u76ee\u5f55\u5c42\u7ea7\u3002\u5982\u679c\u8981\u4eceJindoFS\u534a\u6258\u7ba1\u96c6\u7fa4\u8fc1\u79fb\u5230OSS-HDFS\u670d\u52a1\u3002\u9996\u5148\u9700\u8981\u786e\u4fdd\u5728\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u5bf9\u5e94\u7684OSS Bucket\u4e0a\u5f00\u901aOSS-HDFS\u670d\u52a1\uff0c\u540c\u65f6\u9700\u8981\u5728\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u5f00\u542fAuditLog\u3002 \u6b65\u9aa41. \u5168\u91cf\u5bfc\u5165 \u00a7 \u5168\u91cf\u5bfc\u5165\u6a21\u5f0f\u4f1a\u8d1f\u8d23\u5c06\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u4e2d\u7684\u4e00\u4e2a\u76ee\u5f55\u4e2d\u7684\u5143\u6570\u636e\u4e00\u6b21\u6027\u5168\u91cf\u8fc1\u79fb\u5bfc\u5165JindoFS\u670d\u52a1\u4e2d\u7684\u67d0\u4e00\u4e2a\u76ee\u5f55\u4e2d\u3002\u76ee\u524d\u53ea\u652f\u6301\u5bfc\u5165\u5230JindoFS\u670d\u52a1\u4e2d\u7684\u4e00\u7ea7\u5b50\u76ee\u5f55\u3002 \u547d\u4ee4\u683c\u5f0f \u00a7 jindo distjob -migrateImport -srcPath <srcPath> -destPath <desPth> -backendLoc <backendLoc> \u53c2\u6570\u8bf4\u660e\uff1a -srcPath \uff1a\u5f85\u8fc1\u79fb\u7684\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u7684\u6e90\u8def\u5f84\uff08\u5fc5\u586b\uff09 -destPath \uff1a\u5bfc\u5165\u7684JindoFS\u670d\u52a1\u7684\u76ee\u6807\u8def\u5f84\uff08\u5fc5\u586b\uff09 -backendLoc \uff1a \u534a\u6258\u7ba1\u96c6\u7fa4\u7684\u6e90\u6570\u636e\u5757\u5bf9\u5e94\u7684OSS\u8def\u5f84\uff08\u5fc5\u586b\uff09 \u793a\u4f8b \u00a7 \u5047\u5982\u5728\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u4e2d\u6709\u4e00\u4e2a\u76ee\u5f55 jfs://my-cluster/foo\u3002\u8981\u5c06\u8fd9\u4e2a\u76ee\u5f55\u8fc1\u79fb\u5230JindoFS\u670d\u52a1\u5316\u96c6\u7fa4\u4e2d\u7684bar\u76ee\u5f55\u4e2d\u3002\u5047\u8bbeJindoFS\u670d\u52a1\u5316\u7684bucket\u540d\u4e3adlsbucket\uff0c\u5219\u76ee\u6807\u76ee\u5f55\u4e3aoss://dlsbucket/bar\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u5168\u91cf\u5bfc\u5165\u7684\u547d\u4ee4\u4e3a\uff1a jindo distjob -migrateImport -srcPath jfs://my-cluster/foo -destPath oss://dlsbucket/bar/ \u6b65\u9aa42. \u751f\u6210Change Log \u00a7 \u5982\u679c\u9700\u8981\u589e\u91cf\u8fc1\u79fb\u5bfc\u5165\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u5230JindoFS\u670d\u52a1\u3002\u9700\u8981\u5148\u8fd0\u884cjindo\u5de5\u5177\u53ef\u4ee5\u5c06\u534a\u6258\u7ba1JindoFS\u7684AuditLog\u8f6c\u6362\u6210\u5bf9\u5e94\u76ee\u5f55\u7684\u53d8\u66f4\u65e5\u5fd7\uff08Change Log\uff09\u3002\u5177\u4f53\u547d\u4ee4\u683c\u5f0f\u4e3a\uff1a \u547d\u4ee4\u683c\u5f0f \u00a7 jindo distjob -mkchangelog -auditLogDir <auditLogDir> -changeLogDir <changeLogDir> -startTime <startTime> \u53c2\u6570\u8bf4\u660e\uff1a -auditLogDir \uff1aAuditlog\u7684\u8def\u5f84 -changeLogDir \uff1a\u53d8\u66f4\u65e5\u5fd7\u7684\u8f93\u51fa\u8def\u5f84 -startTime \uff1a\u5f00\u59cb\u5904\u7406Auditlog\u7684\u8d77\u59cb\u65f6\u95f4 \u793a\u4f8b \u00a7 \u5047\u5982\u5728\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u4e2dAuditLog\u7684\u8def\u5f84\u4e3aoss://samplebuket/sysinfo/auditlog\u3002\u60f3\u8981\u8f93\u51fa\u7684\u76ee\u5f55\u7684\u53d8\u66f4\u65e5\u5fd7\u5b58\u653e\u5728oss://samplebuket/sysinfo/changelog\u4e0b\u3002\u5e76\u4e14\u53ea\u5904\u7406\u4ece2022\u5e741\u67081\u65e5\u5f00\u59cb\u7684AuditLog\u3002\u5219\u547d\u4ee4\u5982\u4e0b\uff1a jindo distjob -mkchangelog -auditLogDir oss://samplebuket/sysinfo/auditlog -changeLogDir oss://samplebuket/sysinfo/changelog -startTime 2022-01-01-12:00:00 \u6b65\u9aa43. \u589e\u91cf\u5bfc\u5165 \u00a7 \u901a\u8fc7\u6b65\u9aa42\u751f\u6210\u4e86Change Log\u4e4b\u540e\uff0c\u5c31\u80fd\u8fd0\u884c\u589e\u91cf\u5bfc\u5165\u547d\u4ee4\u3002\u6b64\u6a21\u5f0f\u4e0b\uff0c\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u4ea7\u751f\u7684\u5143\u6570\u636e\u7684\u589e\u91cf\u66f4\u65b0\uff0c\u4f1a\u901a\u8fc7\u5bf9\u5e94\u7684AuditLog\u8f6c\u6362\u4e3aChangeLog\u518d\u7ecf\u8fc7\u5904\u7406\u8fc1\u79fb\u81f3JindoFS\u670d\u52a1\u4e2d\u3002\u5177\u4f53\u7684\u547d\u4ee4\u683c\u5f0f\u5982\u4e0b\u3002 \u547d\u4ee4\u683c\u5f0f \u00a7 jindo distjob -migrateImport -srcPath <srcPath> -destPath <desPth> -changeLogDir <auditLogDir> -backendLoc <backendLoc> -update \u53c2\u6570\u8bf4\u660e\uff1a -srcPath \uff1a\u5f85\u8fc1\u79fb\u7684JindoFS\u534a\u6258\u7ba1\u96c6\u7fa4\u7684\u6e90\u8def\u5f84\uff08\u5fc5\u586b\uff09 -destPath \uff1a\u5bfc\u5165\u7684\u670d\u52a1\u5316JindoFS\u96c6\u7fa4\u7684\u76ee\u6807\u8def\u5f84\uff08\u5fc5\u586b\uff09 -changeLogDir \uff1a Change Log\u6240\u5728\u7684\u76ee\u5f55\uff0c\u4e0e\u6b65\u9aa42\u7684\u547d\u4ee4\u53c2\u6570\u542b\u4e49\u76f8\u540c\uff08\u5fc5\u586b\uff09 -backendLoc \uff1a \u534a\u6258\u7ba1\u96c6\u7fa4\u7684\u6e90\u6570\u636e\u5757\u5bf9\u5e94\u7684OSS\u8def\u5f84\uff08\u5fc5\u586b\uff09 -update : \u5f00\u542f\u589e\u91cf\u5bfc\u5165\uff08\u9ed8\u8ba4\u6a21\u5f0f\u4e3a\u5168\u91cf\u5bfc\u5165\uff09 \u793a\u4f8b \u00a7 \u5047\u5982\u5728\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u4e2d\u6709\u4e00\u4e2a\u76ee\u5f55 jfs://my-cluster/foo\u3002\u8981\u5c06\u8fd9\u4e2a\u76ee\u5f55\u8fc1\u79fb\u5230DLS\u4e2d\u7684bar\u76ee\u5f55\u4e2d\u3002\u5047\u8bbeJindoFS\u670d\u52a1\u5316\u96c6\u7fa4\u7684bucket\u540d\u4e3adlsbucket\uff0c\u5219\u76ee\u6807\u76ee\u5f55\u4e3aoss://dlsbucket/bar\u3002\u5e76\u4e14Directory Change Log\u7684\u76ee\u5f55\u4e3aoss://logBucket/logDir/\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u589e\u91cf\u5bfc\u5165\u7684\u547d\u4ee4\u4e3a\uff1a jindo distjob -migrateImport -srcPath jfs://my-cluster/foo -destPath oss://dlsbucket/bar/ -changeLogDir oss://logBucket/logDir/ -update \u6b65\u9aa44\u3002\u591a\u6b21\u589e\u91cf\u5bfc\u5165 \u00a7 \u5982\u679c\u9700\u8981\u591a\u6b21\u589e\u91cf\u5bfc\u5165\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u5230JindoFS\u670d\u52a1\u3002\u53ef\u4ee5\u901a\u8fc7\u4fee\u6539\u6b65\u9aa42\u4e2d\u5f00\u59cb\u5904\u7406Auditlog\u7684\u8d77\u59cb\u65f6\u95f4 -startTime \u53c2\u6570\uff0c\u7136\u540e\u591a\u6b21\u8fd0\u884c\u6b65\u9aa42\u548c\u6b65\u9aa43\u7684\u547d\u4ee4\u5b8c\u6210\u3002","title":"\u534a\u6258\u7ba1 JindoFS \u8fc1\u79fb\u5230\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/jindofs_dls_migration_import/#jindofs-oss-hdfs-jindofs","text":"(\u4ece 4.5.0 \u5f00\u59cb\u652f\u6301)","title":"\u534a\u6258\u7ba1 JindoFS \u8fc1\u79fb\u5230\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/jindofs_dls_migration_import/#0","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u662f OSS \u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9HDFS\u63a5\u53e3, \u652f\u6301\u76ee\u5f55\u4ee5\u53ca\u76ee\u5f55\u5c42\u7ea7\u3002\u5982\u679c\u8981\u4eceJindoFS\u534a\u6258\u7ba1\u96c6\u7fa4\u8fc1\u79fb\u5230OSS-HDFS\u670d\u52a1\u3002\u9996\u5148\u9700\u8981\u786e\u4fdd\u5728\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u5bf9\u5e94\u7684OSS Bucket\u4e0a\u5f00\u901aOSS-HDFS\u670d\u52a1\uff0c\u540c\u65f6\u9700\u8981\u5728\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u5f00\u542fAuditLog\u3002","title":"0.\u80cc\u666f"},{"location":"backup/jindofs/jindofs_dls_migration_import/#1","text":"\u5168\u91cf\u5bfc\u5165\u6a21\u5f0f\u4f1a\u8d1f\u8d23\u5c06\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u4e2d\u7684\u4e00\u4e2a\u76ee\u5f55\u4e2d\u7684\u5143\u6570\u636e\u4e00\u6b21\u6027\u5168\u91cf\u8fc1\u79fb\u5bfc\u5165JindoFS\u670d\u52a1\u4e2d\u7684\u67d0\u4e00\u4e2a\u76ee\u5f55\u4e2d\u3002\u76ee\u524d\u53ea\u652f\u6301\u5bfc\u5165\u5230JindoFS\u670d\u52a1\u4e2d\u7684\u4e00\u7ea7\u5b50\u76ee\u5f55\u3002","title":"\u6b65\u9aa41. \u5168\u91cf\u5bfc\u5165"},{"location":"backup/jindofs/jindofs_dls_migration_import/#_1","text":"jindo distjob -migrateImport -srcPath <srcPath> -destPath <desPth> -backendLoc <backendLoc> \u53c2\u6570\u8bf4\u660e\uff1a -srcPath \uff1a\u5f85\u8fc1\u79fb\u7684\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u7684\u6e90\u8def\u5f84\uff08\u5fc5\u586b\uff09 -destPath \uff1a\u5bfc\u5165\u7684JindoFS\u670d\u52a1\u7684\u76ee\u6807\u8def\u5f84\uff08\u5fc5\u586b\uff09 -backendLoc \uff1a \u534a\u6258\u7ba1\u96c6\u7fa4\u7684\u6e90\u6570\u636e\u5757\u5bf9\u5e94\u7684OSS\u8def\u5f84\uff08\u5fc5\u586b\uff09","title":"\u547d\u4ee4\u683c\u5f0f"},{"location":"backup/jindofs/jindofs_dls_migration_import/#_2","text":"\u5047\u5982\u5728\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u4e2d\u6709\u4e00\u4e2a\u76ee\u5f55 jfs://my-cluster/foo\u3002\u8981\u5c06\u8fd9\u4e2a\u76ee\u5f55\u8fc1\u79fb\u5230JindoFS\u670d\u52a1\u5316\u96c6\u7fa4\u4e2d\u7684bar\u76ee\u5f55\u4e2d\u3002\u5047\u8bbeJindoFS\u670d\u52a1\u5316\u7684bucket\u540d\u4e3adlsbucket\uff0c\u5219\u76ee\u6807\u76ee\u5f55\u4e3aoss://dlsbucket/bar\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u5168\u91cf\u5bfc\u5165\u7684\u547d\u4ee4\u4e3a\uff1a jindo distjob -migrateImport -srcPath jfs://my-cluster/foo -destPath oss://dlsbucket/bar/","title":"\u793a\u4f8b"},{"location":"backup/jindofs/jindofs_dls_migration_import/#2-change-log","text":"\u5982\u679c\u9700\u8981\u589e\u91cf\u8fc1\u79fb\u5bfc\u5165\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u5230JindoFS\u670d\u52a1\u3002\u9700\u8981\u5148\u8fd0\u884cjindo\u5de5\u5177\u53ef\u4ee5\u5c06\u534a\u6258\u7ba1JindoFS\u7684AuditLog\u8f6c\u6362\u6210\u5bf9\u5e94\u76ee\u5f55\u7684\u53d8\u66f4\u65e5\u5fd7\uff08Change Log\uff09\u3002\u5177\u4f53\u547d\u4ee4\u683c\u5f0f\u4e3a\uff1a","title":"\u6b65\u9aa42. \u751f\u6210Change Log"},{"location":"backup/jindofs/jindofs_dls_migration_import/#_3","text":"jindo distjob -mkchangelog -auditLogDir <auditLogDir> -changeLogDir <changeLogDir> -startTime <startTime> \u53c2\u6570\u8bf4\u660e\uff1a -auditLogDir \uff1aAuditlog\u7684\u8def\u5f84 -changeLogDir \uff1a\u53d8\u66f4\u65e5\u5fd7\u7684\u8f93\u51fa\u8def\u5f84 -startTime \uff1a\u5f00\u59cb\u5904\u7406Auditlog\u7684\u8d77\u59cb\u65f6\u95f4","title":"\u547d\u4ee4\u683c\u5f0f"},{"location":"backup/jindofs/jindofs_dls_migration_import/#_4","text":"\u5047\u5982\u5728\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u4e2dAuditLog\u7684\u8def\u5f84\u4e3aoss://samplebuket/sysinfo/auditlog\u3002\u60f3\u8981\u8f93\u51fa\u7684\u76ee\u5f55\u7684\u53d8\u66f4\u65e5\u5fd7\u5b58\u653e\u5728oss://samplebuket/sysinfo/changelog\u4e0b\u3002\u5e76\u4e14\u53ea\u5904\u7406\u4ece2022\u5e741\u67081\u65e5\u5f00\u59cb\u7684AuditLog\u3002\u5219\u547d\u4ee4\u5982\u4e0b\uff1a jindo distjob -mkchangelog -auditLogDir oss://samplebuket/sysinfo/auditlog -changeLogDir oss://samplebuket/sysinfo/changelog -startTime 2022-01-01-12:00:00","title":"\u793a\u4f8b"},{"location":"backup/jindofs/jindofs_dls_migration_import/#3","text":"\u901a\u8fc7\u6b65\u9aa42\u751f\u6210\u4e86Change Log\u4e4b\u540e\uff0c\u5c31\u80fd\u8fd0\u884c\u589e\u91cf\u5bfc\u5165\u547d\u4ee4\u3002\u6b64\u6a21\u5f0f\u4e0b\uff0c\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u4ea7\u751f\u7684\u5143\u6570\u636e\u7684\u589e\u91cf\u66f4\u65b0\uff0c\u4f1a\u901a\u8fc7\u5bf9\u5e94\u7684AuditLog\u8f6c\u6362\u4e3aChangeLog\u518d\u7ecf\u8fc7\u5904\u7406\u8fc1\u79fb\u81f3JindoFS\u670d\u52a1\u4e2d\u3002\u5177\u4f53\u7684\u547d\u4ee4\u683c\u5f0f\u5982\u4e0b\u3002","title":"\u6b65\u9aa43. \u589e\u91cf\u5bfc\u5165"},{"location":"backup/jindofs/jindofs_dls_migration_import/#_5","text":"jindo distjob -migrateImport -srcPath <srcPath> -destPath <desPth> -changeLogDir <auditLogDir> -backendLoc <backendLoc> -update \u53c2\u6570\u8bf4\u660e\uff1a -srcPath \uff1a\u5f85\u8fc1\u79fb\u7684JindoFS\u534a\u6258\u7ba1\u96c6\u7fa4\u7684\u6e90\u8def\u5f84\uff08\u5fc5\u586b\uff09 -destPath \uff1a\u5bfc\u5165\u7684\u670d\u52a1\u5316JindoFS\u96c6\u7fa4\u7684\u76ee\u6807\u8def\u5f84\uff08\u5fc5\u586b\uff09 -changeLogDir \uff1a Change Log\u6240\u5728\u7684\u76ee\u5f55\uff0c\u4e0e\u6b65\u9aa42\u7684\u547d\u4ee4\u53c2\u6570\u542b\u4e49\u76f8\u540c\uff08\u5fc5\u586b\uff09 -backendLoc \uff1a \u534a\u6258\u7ba1\u96c6\u7fa4\u7684\u6e90\u6570\u636e\u5757\u5bf9\u5e94\u7684OSS\u8def\u5f84\uff08\u5fc5\u586b\uff09 -update : \u5f00\u542f\u589e\u91cf\u5bfc\u5165\uff08\u9ed8\u8ba4\u6a21\u5f0f\u4e3a\u5168\u91cf\u5bfc\u5165\uff09","title":"\u547d\u4ee4\u683c\u5f0f"},{"location":"backup/jindofs/jindofs_dls_migration_import/#_6","text":"\u5047\u5982\u5728\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u4e2d\u6709\u4e00\u4e2a\u76ee\u5f55 jfs://my-cluster/foo\u3002\u8981\u5c06\u8fd9\u4e2a\u76ee\u5f55\u8fc1\u79fb\u5230DLS\u4e2d\u7684bar\u76ee\u5f55\u4e2d\u3002\u5047\u8bbeJindoFS\u670d\u52a1\u5316\u96c6\u7fa4\u7684bucket\u540d\u4e3adlsbucket\uff0c\u5219\u76ee\u6807\u76ee\u5f55\u4e3aoss://dlsbucket/bar\u3002\u5e76\u4e14Directory Change Log\u7684\u76ee\u5f55\u4e3aoss://logBucket/logDir/\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u589e\u91cf\u5bfc\u5165\u7684\u547d\u4ee4\u4e3a\uff1a jindo distjob -migrateImport -srcPath jfs://my-cluster/foo -destPath oss://dlsbucket/bar/ -changeLogDir oss://logBucket/logDir/ -update","title":"\u793a\u4f8b"},{"location":"backup/jindofs/jindofs_dls_migration_import/#4","text":"\u5982\u679c\u9700\u8981\u591a\u6b21\u589e\u91cf\u5bfc\u5165\u534a\u6258\u7ba1JindoFS\u96c6\u7fa4\u5230JindoFS\u670d\u52a1\u3002\u53ef\u4ee5\u901a\u8fc7\u4fee\u6539\u6b65\u9aa42\u4e2d\u5f00\u59cb\u5904\u7406Auditlog\u7684\u8d77\u59cb\u65f6\u95f4 -startTime \u53c2\u6570\uff0c\u7136\u540e\u591a\u6b21\u8fd0\u884c\u6b65\u9aa42\u548c\u6b65\u9aa43\u7684\u547d\u4ee4\u5b8c\u6210\u3002","title":"\u6b65\u9aa44\u3002\u591a\u6b21\u589e\u91cf\u5bfc\u5165"},{"location":"backup/jindofs/jindofs_dls_root_policy/","text":"\u4f7f\u7528 RootPolicy \u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 (\u4ece 4.6.0 \u5f00\u59cb\u652f\u6301) \u80cc\u666f \u00a7 JindoFS \u652f\u6301 RootPolicy \u89c4\u5219\uff0c\u901a\u8fc7 RootPolicy \u53ef\u4ee5\u4e3a OSS-HDFS\uff08JindoFS \u670d\u52a1\uff09\u8bbe\u7f6e\u81ea\u5b9a\u4e49\u524d\u7f00\uff0c\u8fd9\u6837\u7528\u6237\u539f\u6709\u7684\u8bbf\u95ee hdfs:// \u524d\u7f00\u7684\u4f5c\u4e1a\uff0c\u53ef\u4ee5\u4e0d\u7ecf\u4fee\u6539\u76f4\u63a5\u8fd0\u884c\u5728 JindoFS \u670d\u52a1\u4e0a\u3002 1\u3001\u8bbe\u7f6e RootPolicy \u00a7 \u6211\u4eec\u63d0\u4f9b\u4e86\u4e00\u7ec4 Shell \u547d\u4ee4\uff0c\u5e2e\u52a9\u7528\u6237\u5411\u7279\u5b9a\u7684bucket\u6ce8\u518c\u81ea\u5b9a\u4e49\u5730\u5740\u3002\u901a\u8fc7 SetRootPolicy\u547d\u4ee4 \uff0c\u53ef\u4ee5\u4e3a\u7279\u5b9a bucket \u6ce8\u518c\u81ea\u5b9a\u4e49\u524d\u7f00\u7684\u8bbf\u95ee\u5730\u5740\u3002\u5982\uff1a jindo admin -setRootPolicy oss://<bucket_name>.<dls_endpoint>/ hdfs://<your_ns_name>/ : OSS-HDFS \u670d\u52a1 bucket \u7684\u540d\u5b57\uff0c\u5f53\u524d\u7248\u672c\u4ec5\u652f\u6301\u6839\u76ee\u5f55\u3002 : OSS-HDFS \u670d\u52a1\u7684 endpoint\uff0c\u5982\u676d\u5dderegion\u4e3a cn-hangzhou.oss-dls.aliyuncs.com \u3002\u5982\u679c Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u6709\u5982\u4e0b\u914d\u7f6e\u9879 <configuration> <property> <name>fs.oss.endpoint</name> <value><dls_endpoint></value> </property> \u6216 <property> <name>fs.oss.bucket.<bucket_name>.endpoint</name> <value><dls_endpoint></value> </property> </configuration> \u4e0a\u9762\u4f8b\u5b50\u4e2d\u7684 oss://<bucket_name>.<dls_endpoint>/ \u53ef\u4ee5\u5199\u6210 oss://<bucket_name>/ \u7684\u5f62\u5f0f\uff0c\u4e0b\u6587\u4e2d\u9700\u8981 endpoint \u7684\u5730\u65b9\u90fd\u53ef\u7701\u7565\u4e0d\u586b\u3002 * : \u81ea\u5b9a\u4e49\u8bbf\u95ee hdfs \u670d\u52a1\u7684 nsname\uff0c\u4e3a\u4efb\u610f\u975e\u7a7a\u5b57\u7b26\u4e32\uff0c\u5982 test \uff0c\u5f53\u524d\u7248\u672c\u4ec5\u652f\u6301\u6839\u76ee\u5f55\u3002 2\u3001\u914d\u7f6e access policy \u53d1\u73b0\u5730\u5740\u548c scheme \u5b9e\u73b0\u7c7b \u00a7 \u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u6dfb\u52a0\u5982\u4e0b\u914d\u7f6e\u9879 <configuration> <property> <name>fs.accessPolicies.discovery</name> <value>oss://<bucket_name>.<dls_endpoint>/</value> </property> <property> <name>fs.AbstractFileSystem.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.v28.HDFS</value> </property> <property> <name>fs.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.v28.JindoDistributedFileSystem</value> </property> </configuration> \u5982\u679c\u662f\u591a\u4e2abucket\uff0c\u7528 , \u5206\u9694\u3002 3\u3001\u9a8c\u8bc1\u548c\u4f7f\u7528 \u00a7 \u914d\u7f6e\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u5bf9\u914d\u7f6e\u8fdb\u884c\u7b80\u5355\u9a8c\u8bc1\u3002 hadoop fs -ls hdfs://<your_ns_name>/ \u9a8c\u8bc1\u901a\u8fc7\u540e\uff0c\u91cd\u542f\u76f8\u5173\u670d\u52a1\uff0c\u5982 Hive/Spark \u7b49\uff0c\u5373\u53ef\u4f7f\u7528\u81ea\u5b9a\u4e49\u524d\u7f00\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u3002 4\u3001\u5176\u4ed6\u529f\u80fd \u00a7 4.1\u3001\u5220\u9664 RootPolicy \u89c4\u5219 \u00a7 \u901a\u8fc7 UnsetRootPolicy\u547d\u4ee4 \u53ef\u4ee5\u5220\u9664\u76f8\u5e94\u5730\u5740\u3002 jindo admin -unsetRootPolicy oss://<bucket_name>.<dls_endpoint>/ hdfs://<your_ns_name>/ 4.2\u3001\u5217\u4e3e\u6240\u6709 RootPolicy \u89c4\u5219 \u00a7 \u4f7f\u7528 ListAccessPolicies\u547d\u4ee4 \uff0c\u53ef\u4ee5\u5217\u51fa\u7279\u5b9a bucket \u5f53\u524d\u6ce8\u518c\u7684\u6240\u6709\u524d\u7f00\u5730\u5740\u3002 jindo admin -listAccessPolicies oss://<bucket_name>.<dls_endpoint>/","title":"\u4f7f\u7528 RootPolicy \u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/jindofs_dls_root_policy/#rootpolicy-oss-hdfs-jindofs","text":"(\u4ece 4.6.0 \u5f00\u59cb\u652f\u6301)","title":"\u4f7f\u7528 RootPolicy \u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/jindofs_dls_root_policy/#_1","text":"JindoFS \u652f\u6301 RootPolicy \u89c4\u5219\uff0c\u901a\u8fc7 RootPolicy \u53ef\u4ee5\u4e3a OSS-HDFS\uff08JindoFS \u670d\u52a1\uff09\u8bbe\u7f6e\u81ea\u5b9a\u4e49\u524d\u7f00\uff0c\u8fd9\u6837\u7528\u6237\u539f\u6709\u7684\u8bbf\u95ee hdfs:// \u524d\u7f00\u7684\u4f5c\u4e1a\uff0c\u53ef\u4ee5\u4e0d\u7ecf\u4fee\u6539\u76f4\u63a5\u8fd0\u884c\u5728 JindoFS \u670d\u52a1\u4e0a\u3002","title":"\u80cc\u666f"},{"location":"backup/jindofs/jindofs_dls_root_policy/#1-rootpolicy","text":"\u6211\u4eec\u63d0\u4f9b\u4e86\u4e00\u7ec4 Shell \u547d\u4ee4\uff0c\u5e2e\u52a9\u7528\u6237\u5411\u7279\u5b9a\u7684bucket\u6ce8\u518c\u81ea\u5b9a\u4e49\u5730\u5740\u3002\u901a\u8fc7 SetRootPolicy\u547d\u4ee4 \uff0c\u53ef\u4ee5\u4e3a\u7279\u5b9a bucket \u6ce8\u518c\u81ea\u5b9a\u4e49\u524d\u7f00\u7684\u8bbf\u95ee\u5730\u5740\u3002\u5982\uff1a jindo admin -setRootPolicy oss://<bucket_name>.<dls_endpoint>/ hdfs://<your_ns_name>/ : OSS-HDFS \u670d\u52a1 bucket \u7684\u540d\u5b57\uff0c\u5f53\u524d\u7248\u672c\u4ec5\u652f\u6301\u6839\u76ee\u5f55\u3002 : OSS-HDFS \u670d\u52a1\u7684 endpoint\uff0c\u5982\u676d\u5dderegion\u4e3a cn-hangzhou.oss-dls.aliyuncs.com \u3002\u5982\u679c Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u6709\u5982\u4e0b\u914d\u7f6e\u9879 <configuration> <property> <name>fs.oss.endpoint</name> <value><dls_endpoint></value> </property> \u6216 <property> <name>fs.oss.bucket.<bucket_name>.endpoint</name> <value><dls_endpoint></value> </property> </configuration> \u4e0a\u9762\u4f8b\u5b50\u4e2d\u7684 oss://<bucket_name>.<dls_endpoint>/ \u53ef\u4ee5\u5199\u6210 oss://<bucket_name>/ \u7684\u5f62\u5f0f\uff0c\u4e0b\u6587\u4e2d\u9700\u8981 endpoint \u7684\u5730\u65b9\u90fd\u53ef\u7701\u7565\u4e0d\u586b\u3002 * : \u81ea\u5b9a\u4e49\u8bbf\u95ee hdfs \u670d\u52a1\u7684 nsname\uff0c\u4e3a\u4efb\u610f\u975e\u7a7a\u5b57\u7b26\u4e32\uff0c\u5982 test \uff0c\u5f53\u524d\u7248\u672c\u4ec5\u652f\u6301\u6839\u76ee\u5f55\u3002","title":"1\u3001\u8bbe\u7f6e RootPolicy"},{"location":"backup/jindofs/jindofs_dls_root_policy/#2-access-policy-scheme","text":"\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u6dfb\u52a0\u5982\u4e0b\u914d\u7f6e\u9879 <configuration> <property> <name>fs.accessPolicies.discovery</name> <value>oss://<bucket_name>.<dls_endpoint>/</value> </property> <property> <name>fs.AbstractFileSystem.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.v28.HDFS</value> </property> <property> <name>fs.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.v28.JindoDistributedFileSystem</value> </property> </configuration> \u5982\u679c\u662f\u591a\u4e2abucket\uff0c\u7528 , \u5206\u9694\u3002","title":"2\u3001\u914d\u7f6e access policy \u53d1\u73b0\u5730\u5740\u548c scheme \u5b9e\u73b0\u7c7b"},{"location":"backup/jindofs/jindofs_dls_root_policy/#3","text":"\u914d\u7f6e\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u5bf9\u914d\u7f6e\u8fdb\u884c\u7b80\u5355\u9a8c\u8bc1\u3002 hadoop fs -ls hdfs://<your_ns_name>/ \u9a8c\u8bc1\u901a\u8fc7\u540e\uff0c\u91cd\u542f\u76f8\u5173\u670d\u52a1\uff0c\u5982 Hive/Spark \u7b49\uff0c\u5373\u53ef\u4f7f\u7528\u81ea\u5b9a\u4e49\u524d\u7f00\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u3002","title":"3\u3001\u9a8c\u8bc1\u548c\u4f7f\u7528"},{"location":"backup/jindofs/jindofs_dls_root_policy/#4","text":"","title":"4\u3001\u5176\u4ed6\u529f\u80fd"},{"location":"backup/jindofs/jindofs_dls_root_policy/#41-rootpolicy","text":"\u901a\u8fc7 UnsetRootPolicy\u547d\u4ee4 \u53ef\u4ee5\u5220\u9664\u76f8\u5e94\u5730\u5740\u3002 jindo admin -unsetRootPolicy oss://<bucket_name>.<dls_endpoint>/ hdfs://<your_ns_name>/","title":"4.1\u3001\u5220\u9664 RootPolicy \u89c4\u5219"},{"location":"backup/jindofs/jindofs_dls_root_policy/#42-rootpolicy","text":"\u4f7f\u7528 ListAccessPolicies\u547d\u4ee4 \uff0c\u53ef\u4ee5\u5217\u51fa\u7279\u5b9a bucket \u5f53\u524d\u6ce8\u518c\u7684\u6240\u6709\u524d\u7f00\u5730\u5740\u3002 jindo admin -listAccessPolicies oss://<bucket_name>.<dls_endpoint>/","title":"4.2\u3001\u5217\u4e3e\u6240\u6709 RootPolicy \u89c4\u5219"},{"location":"backup/jindofs/jindofs_dls_shell_howto/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Shell \u547d\u4ee4\u4f7f\u7528\u8bf4\u660e \u00a7 (\u4ece 4.5.0 \u5f00\u59cb\u652f\u6301) \u8bbe\u7f6e Hadoop Proxy User \u547d\u4ee4 \u00a7 AddProxyUser \u547d\u4ee4 \u00a7 AddProxyUser \u547d\u4ee4\u53ef\u4ee5\u6dfb\u52a0\u4ee3\u7406\u7528\u6237\u3002 jindo admin -addProxyUser [-dlsUri <uri>] [-proxyUser <proxyUser>] [-users <user1,user2...>]|[-groups <group1,group2...>] [-hosts <host1,host2...>] \u5728\u540c\u4e00\u6761\u547d\u4ee4\u4e2d\uff0c\u53c2\u6570 users \u548c groups \u4e0d\u53ef\u540c\u65f6\u4f7f\u7528\uff0c\u4efb\u9009\u5176\u4e00\u3002 \u4f8b\uff1a jindo admin -addProxyUser -dlsUri oss://jindosdk-dls-unit-test/ -proxyUser hive -groups group1,group2 -hosts host1,host2 \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u7528\u6237 hive \u53ef\u4ee5\u4ee3\u7406\u5c5e\u4e8e group1 \u6216 group2 \u4e14\u4ece host1 \u6216 host2 \u53d1\u9001\u8bf7\u6c42\u7684\u7528\u6237\u3002 DeleteProxyUser \u547d\u4ee4 \u00a7 DeleteProxyUser \u547d\u4ee4\u53ef\u4ee5\u5220\u9664\u4ee3\u7406\u7528\u6237\u3002 jindo admin -deleteProxyUser [-dlsUri <uri>] [-proxyUser <proxyUser>] \u4f8b\uff1a jindo admin -deleteProxyUser -dlsUri oss://jindosdk-dls-unit-test/ -proxyUser hive \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u7528\u6237 hive \u4e0d\u53ef\u4ee5\u4ee3\u7406\u4efb\u4f55\u7528\u6237\u3002 ListProxyUsers \u547d\u4ee4 \u00a7 ListProxyUsers \u53ef\u4ee5\u67e5\u770b\u6240\u6709\u7684\u4ee3\u7406\u7528\u6237\u548c\u4ee3\u7406\u4fe1\u606f\u3002 jindo admin -listProxyUsers [-dlsUri <uri>] [-maxKeys <maxKeys>] [-marker <marker>] maxKeys \u6307\u5b9a\u67e5\u770b\u7684\u4ee3\u7406\u7528\u6237\u4e2a\u6570\uff0cmarker \u7528\u6765\u7b5b\u9009\u540d\u79f0\u4e2d\u542b\u6709\u7279\u6b8a\u6807\u8bb0\u7684\u4ee3\u7406\u7528\u6237\uff0c\u8fd9\u4e24\u9879\u53c2\u6570\u5747\u53ef\u7701\u7565\u3002 \u4f8b\uff1a jindo admin -listProxyUsers -dlsUri oss://jindosdk-dls-unit-test/ -maxKeys 10 -marker hive \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u5c06\u4f18\u5148\u663e\u793a oss://jindosdk-dls-unit-test/ \u4e0b\u540d\u79f0\u4e2d\u5e26 hive \u6807\u8bb0\u768410\u4e2a\u4ee3\u7406\u7528\u6237\u4fe1\u606f\u3002 \u8bbe\u7f6e User Groups Mapping \u547d\u4ee4 \u00a7 AddUserGroupsMapping \u547d\u4ee4 \u00a7 AddUserGroupsMapping \u547d\u4ee4\u6620\u5c04\u7528\u6237\u548c\u7ec4\u7684\u5173\u7cfb\u3002 jindo admin -addUserGroupsMapping [-dlsUri <uri>] [-user <user>] [-groups <group1,group2...>] \u4f8b\uff1a jindo admin -addUserGroupsMapping -dlsUri oss://jindosdk-dls-unit-test/ -user user1 -groups group1,group2 \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u7528\u6237 user1 \u5c06\u5c5e\u4e8e group1 \u6216 group2\u3002 DeleteUserGroupsMapping \u547d\u4ee4 \u00a7 DeleteUserGroupsMapping \u547d\u4ee4\u53ef\u4ee5\u5220\u9664\u7528\u6237\u548c\u7ec4\u7684\u6620\u5c04\u5173\u7cfb\u3002 jindo admin -deleteUserGroupsMapping [-dlsUri <uri>] [-user <user>] \u4f8b\uff1a jindo admin -deleteUserGroupsMapping -dlsUri oss://jindosdk-dls-unit-test/ -user user1 \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u5c06\u5220\u9664 user1 \u4e0e\u5bf9\u5e94\u7ec4\u7684\u6620\u5c04\u5173\u7cfb\u3002 ListUserGroupsMappings \u547d\u4ee4 \u00a7 ListUserGroupsMappings \u53ef\u4ee5\u67e5\u770b\u6240\u6709\u7684\u7528\u6237\u7684\u7ec4\u3002 jindo admin -listUserGroupsMappings [-dlsUri <dlsUri>] [-maxKeys <maxKeys>] [-marker <marker>] maxKeys \u6307\u5b9a\u67e5\u770b\u7684\u7528\u6237\u4e2a\u6570\uff0cmarker \u7528\u6765\u7b5b\u9009\u540d\u79f0\u4e2d\u542b\u6709\u7279\u6b8a\u6807\u8bb0\u7684\u7528\u6237\uff0c\u8fd9\u4e24\u9879\u53c2\u6570\u5747\u53ef\u7701\u7565\u3002 \u4f8b\uff1a jindo admin -listUserGroupsMappings -dlsUri oss://jindosdk-dls-unit-test/ -maxKeys 10 -marker user1 \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u5c06\u4f18\u5148\u663e\u793a oss://jindosdk-dls-unit-test/ \u4e0b\u540d\u79f0\u4e2d\u5e26 user1 \u6807\u8bb0\u768410\u4e2a\u7528\u6237\u7684\u7ec4\u4fe1\u606f\u3002 \u7ba1\u7406 Snapshot \u547d\u4ee4 \u00a7 AllowSnapshot \u547d\u4ee4 \u00a7 AllowSnapshot \u547d\u4ee4\u5141\u8bb8\u521b\u5efa\u76ee\u5f55\u5feb\u7167\u3002 jindo admin -allowSnapshot [-dlsUri <dlsSnapshotDir>] \u4f8b\uff1a jindo admin -allowSnapshot -dlsUri oss://jindosdk-dls-unit-test/testSnapshot/snapshot_allow \u6267\u884c\u4e0a\u9762\u7684\u547d\u4ee4\u540e\uff0c\u5c06\u5141\u8bb8 oss://jindosdk-dls-unit-test/testSnapshot/snapshot_allow \u76ee\u5f55\u521b\u5efa\u5feb\u7167\u3002 DisallowSnapshot \u547d\u4ee4 \u00a7 DisallowSnapshot \u547d\u4ee4\u53ef\u4ee5\u7981\u6b62\u521b\u5efa\u76ee\u5f55\u5feb\u7167\u3002 jindo admin -disallowSnapshot [-dlsUri <dlsSnapshotDir>] \u4f8b\uff1a jindo admin -disallowSnapshot -dlsUri oss://jindosdk-dls-unit-test/testSnapshot/snapshot_allow \u6267\u884c\u4e0a\u9762\u7684\u547d\u4ee4\u540e\uff0c\u5c06\u4e0d\u5141\u8bb8 oss://jindosdk-dls-unit-test/testSnapshot/snapshot_allow \u76ee\u5f55\u521b\u5efa\u5feb\u7167\u3002 SnapshotDiff \u547d\u4ee4 \u00a7 SnapshotDiff \u547d\u4ee4\u7528\u6765\u6bd4\u8f83\u540c\u4e00\u76ee\u5f55\u4e24\u4efd\u5feb\u7167\u7684\u4e0d\u540c\u3002 jindo admin -snapshotDiff [-dlsUri <dlsSnapshotDir>] [-fromSnapshot <fromSnapshot>] [-toSnapshot <toSnapshot>] \u4f8b\uff1a jindo admin -snapshotDiff -dlsUri oss://jindosdk-dls-unit-test/testSnapshotDir/ -fromSnapshot S1 -toSnapshot S2 S1 \u548c S1 \u662f\u4e8b\u5148\u901a\u8fc7 HDFS \u7684 Shell \u547d\u4ee4\u884c\u5de5\u5177\u4e3a\u6d4b\u8bd5\u76ee\u5f55\u521b\u5efa\u7684\u4e24\u4efd\u5feb\u7167\u3002 \u901a\u8fc7 AccessPolicy \u547d\u4ee4\u5b9e\u73b0\u8def\u5f84\u6539\u5199 \u00a7 SetRootPolicy \u547d\u4ee4 \u00a7 SetRootPolicy \u547d\u4ee4\u5141\u8bb8\u4e3a bucket \u8bbe\u7f6e\u4efb\u610f\u524d\u7f00\u7684\u8bbf\u95ee\u8def\u5f84\u3002 jindo admin -setRootPolicy [<dlsRootPath>] [<accessRootPath>] \u4f8b\uff1a jindo admin -setRootPolicy oss://jindosdk-dls-unit-test/ hdfs://myufs/ \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\u540e\uff0c\u652f\u6301\u4f7f\u7528 hdfs://myufs/ \u8bbf\u95ee oss://jindosdk-dls-unit-test/\u3002 UnsetRootPolicy \u547d\u4ee4 \u00a7 UnsetRootPolicy \u547d\u4ee4\u4e3a bucket \u53d6\u6d88\u8bbf\u95ee\u8def\u5f84\u3002 jindo admin -unsetRootPolicy [<dlsRootPath>] [<accessRootPath>] \u4f8b\uff1a jindo admin -unsetRootPolicy oss://jindosdk-dls-unit-test/ hdfs://myufs/ \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\u540e\uff0c\u4e0d\u518d\u652f\u6301\u7528 hdfs://myufs/ \u8bbf\u95ee oss://jindosdk-dls-unit-test/\u3002 ListAccessPolicies \u547d\u4ee4 \u00a7 ListAccessPolicies \u53ef\u67e5\u770b\u5f53\u524d bucket \u652f\u6301\u7684\u6240\u6709\u8bbf\u95ee\u8def\u5f84\u3002 jindo admin -listAccessPolicies [<dlsRootPath>] \u4f8b\uff1a jindo admin -listAccessPolicies oss://jindosdk-dls-unit-test/ \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u67e5\u770b oss://jindosdk-dls-unit-test/ \u7684\u6240\u6709\u8bbf\u95ee\u8def\u5f84\u3002 \u5176\u4ed6 \u00a7 DumpFile \u547d\u4ee4 \u00a7 DumpFile \u547d\u4ee4\u53ef\u4ee5\u67e5\u770b\u6587\u4ef6\u7684\u8be6\u7ec6\u5b58\u50a8\u4fe1\u606f\u3002 jindo admin -dumpFile [-dlsUri <dlsUri>] \u4f8b\uff1a jindo admin -dumpFile -dlsUri oss://jindosdk-dls-unit-test/testDumpFile/dumpfile_simple_file \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\u540e\uff0c\u5c06\u663e\u793a\u7c7b\u4f3c\u5982\u4e0b\u7ed3\u679c\uff1a =============Dump result============= File /testDumpFile/dumpfile_simple_file length: 11 blockSize: 33554432 modification time: 1657626266711 access time: 1657626266481 permission: 420 owner: root group: supergroup file id: 4180702376518228901 =============Block 0 block id: 8792388394945610760 offset: 0 numBytes: 11 gs: 1032 type: CLOUD loc: .dlsdata underConstruction: 0 isLastBlockComplete: 1","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Shell \u547d\u4ee4\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#oss-hdfs-jindofs-shell","text":"(\u4ece 4.5.0 \u5f00\u59cb\u652f\u6301)","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Shell \u547d\u4ee4\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#hadoop-proxy-user","text":"","title":"\u8bbe\u7f6e Hadoop Proxy User \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#addproxyuser","text":"AddProxyUser \u547d\u4ee4\u53ef\u4ee5\u6dfb\u52a0\u4ee3\u7406\u7528\u6237\u3002 jindo admin -addProxyUser [-dlsUri <uri>] [-proxyUser <proxyUser>] [-users <user1,user2...>]|[-groups <group1,group2...>] [-hosts <host1,host2...>] \u5728\u540c\u4e00\u6761\u547d\u4ee4\u4e2d\uff0c\u53c2\u6570 users \u548c groups \u4e0d\u53ef\u540c\u65f6\u4f7f\u7528\uff0c\u4efb\u9009\u5176\u4e00\u3002 \u4f8b\uff1a jindo admin -addProxyUser -dlsUri oss://jindosdk-dls-unit-test/ -proxyUser hive -groups group1,group2 -hosts host1,host2 \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u7528\u6237 hive \u53ef\u4ee5\u4ee3\u7406\u5c5e\u4e8e group1 \u6216 group2 \u4e14\u4ece host1 \u6216 host2 \u53d1\u9001\u8bf7\u6c42\u7684\u7528\u6237\u3002","title":"AddProxyUser \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#deleteproxyuser","text":"DeleteProxyUser \u547d\u4ee4\u53ef\u4ee5\u5220\u9664\u4ee3\u7406\u7528\u6237\u3002 jindo admin -deleteProxyUser [-dlsUri <uri>] [-proxyUser <proxyUser>] \u4f8b\uff1a jindo admin -deleteProxyUser -dlsUri oss://jindosdk-dls-unit-test/ -proxyUser hive \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u7528\u6237 hive \u4e0d\u53ef\u4ee5\u4ee3\u7406\u4efb\u4f55\u7528\u6237\u3002","title":"DeleteProxyUser \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#listproxyusers","text":"ListProxyUsers \u53ef\u4ee5\u67e5\u770b\u6240\u6709\u7684\u4ee3\u7406\u7528\u6237\u548c\u4ee3\u7406\u4fe1\u606f\u3002 jindo admin -listProxyUsers [-dlsUri <uri>] [-maxKeys <maxKeys>] [-marker <marker>] maxKeys \u6307\u5b9a\u67e5\u770b\u7684\u4ee3\u7406\u7528\u6237\u4e2a\u6570\uff0cmarker \u7528\u6765\u7b5b\u9009\u540d\u79f0\u4e2d\u542b\u6709\u7279\u6b8a\u6807\u8bb0\u7684\u4ee3\u7406\u7528\u6237\uff0c\u8fd9\u4e24\u9879\u53c2\u6570\u5747\u53ef\u7701\u7565\u3002 \u4f8b\uff1a jindo admin -listProxyUsers -dlsUri oss://jindosdk-dls-unit-test/ -maxKeys 10 -marker hive \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u5c06\u4f18\u5148\u663e\u793a oss://jindosdk-dls-unit-test/ \u4e0b\u540d\u79f0\u4e2d\u5e26 hive \u6807\u8bb0\u768410\u4e2a\u4ee3\u7406\u7528\u6237\u4fe1\u606f\u3002","title":"ListProxyUsers \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#user-groups-mapping","text":"","title":"\u8bbe\u7f6e User Groups Mapping \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#addusergroupsmapping","text":"AddUserGroupsMapping \u547d\u4ee4\u6620\u5c04\u7528\u6237\u548c\u7ec4\u7684\u5173\u7cfb\u3002 jindo admin -addUserGroupsMapping [-dlsUri <uri>] [-user <user>] [-groups <group1,group2...>] \u4f8b\uff1a jindo admin -addUserGroupsMapping -dlsUri oss://jindosdk-dls-unit-test/ -user user1 -groups group1,group2 \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u7528\u6237 user1 \u5c06\u5c5e\u4e8e group1 \u6216 group2\u3002","title":"AddUserGroupsMapping \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#deleteusergroupsmapping","text":"DeleteUserGroupsMapping \u547d\u4ee4\u53ef\u4ee5\u5220\u9664\u7528\u6237\u548c\u7ec4\u7684\u6620\u5c04\u5173\u7cfb\u3002 jindo admin -deleteUserGroupsMapping [-dlsUri <uri>] [-user <user>] \u4f8b\uff1a jindo admin -deleteUserGroupsMapping -dlsUri oss://jindosdk-dls-unit-test/ -user user1 \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u5c06\u5220\u9664 user1 \u4e0e\u5bf9\u5e94\u7ec4\u7684\u6620\u5c04\u5173\u7cfb\u3002","title":"DeleteUserGroupsMapping \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#listusergroupsmappings","text":"ListUserGroupsMappings \u53ef\u4ee5\u67e5\u770b\u6240\u6709\u7684\u7528\u6237\u7684\u7ec4\u3002 jindo admin -listUserGroupsMappings [-dlsUri <dlsUri>] [-maxKeys <maxKeys>] [-marker <marker>] maxKeys \u6307\u5b9a\u67e5\u770b\u7684\u7528\u6237\u4e2a\u6570\uff0cmarker \u7528\u6765\u7b5b\u9009\u540d\u79f0\u4e2d\u542b\u6709\u7279\u6b8a\u6807\u8bb0\u7684\u7528\u6237\uff0c\u8fd9\u4e24\u9879\u53c2\u6570\u5747\u53ef\u7701\u7565\u3002 \u4f8b\uff1a jindo admin -listUserGroupsMappings -dlsUri oss://jindosdk-dls-unit-test/ -maxKeys 10 -marker user1 \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u5c06\u4f18\u5148\u663e\u793a oss://jindosdk-dls-unit-test/ \u4e0b\u540d\u79f0\u4e2d\u5e26 user1 \u6807\u8bb0\u768410\u4e2a\u7528\u6237\u7684\u7ec4\u4fe1\u606f\u3002","title":"ListUserGroupsMappings \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#snapshot","text":"","title":"\u7ba1\u7406 Snapshot \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#allowsnapshot","text":"AllowSnapshot \u547d\u4ee4\u5141\u8bb8\u521b\u5efa\u76ee\u5f55\u5feb\u7167\u3002 jindo admin -allowSnapshot [-dlsUri <dlsSnapshotDir>] \u4f8b\uff1a jindo admin -allowSnapshot -dlsUri oss://jindosdk-dls-unit-test/testSnapshot/snapshot_allow \u6267\u884c\u4e0a\u9762\u7684\u547d\u4ee4\u540e\uff0c\u5c06\u5141\u8bb8 oss://jindosdk-dls-unit-test/testSnapshot/snapshot_allow \u76ee\u5f55\u521b\u5efa\u5feb\u7167\u3002","title":"AllowSnapshot \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#disallowsnapshot","text":"DisallowSnapshot \u547d\u4ee4\u53ef\u4ee5\u7981\u6b62\u521b\u5efa\u76ee\u5f55\u5feb\u7167\u3002 jindo admin -disallowSnapshot [-dlsUri <dlsSnapshotDir>] \u4f8b\uff1a jindo admin -disallowSnapshot -dlsUri oss://jindosdk-dls-unit-test/testSnapshot/snapshot_allow \u6267\u884c\u4e0a\u9762\u7684\u547d\u4ee4\u540e\uff0c\u5c06\u4e0d\u5141\u8bb8 oss://jindosdk-dls-unit-test/testSnapshot/snapshot_allow \u76ee\u5f55\u521b\u5efa\u5feb\u7167\u3002","title":"DisallowSnapshot \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#snapshotdiff","text":"SnapshotDiff \u547d\u4ee4\u7528\u6765\u6bd4\u8f83\u540c\u4e00\u76ee\u5f55\u4e24\u4efd\u5feb\u7167\u7684\u4e0d\u540c\u3002 jindo admin -snapshotDiff [-dlsUri <dlsSnapshotDir>] [-fromSnapshot <fromSnapshot>] [-toSnapshot <toSnapshot>] \u4f8b\uff1a jindo admin -snapshotDiff -dlsUri oss://jindosdk-dls-unit-test/testSnapshotDir/ -fromSnapshot S1 -toSnapshot S2 S1 \u548c S1 \u662f\u4e8b\u5148\u901a\u8fc7 HDFS \u7684 Shell \u547d\u4ee4\u884c\u5de5\u5177\u4e3a\u6d4b\u8bd5\u76ee\u5f55\u521b\u5efa\u7684\u4e24\u4efd\u5feb\u7167\u3002","title":"SnapshotDiff \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#accesspolicy","text":"","title":"\u901a\u8fc7 AccessPolicy \u547d\u4ee4\u5b9e\u73b0\u8def\u5f84\u6539\u5199"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#setrootpolicy","text":"SetRootPolicy \u547d\u4ee4\u5141\u8bb8\u4e3a bucket \u8bbe\u7f6e\u4efb\u610f\u524d\u7f00\u7684\u8bbf\u95ee\u8def\u5f84\u3002 jindo admin -setRootPolicy [<dlsRootPath>] [<accessRootPath>] \u4f8b\uff1a jindo admin -setRootPolicy oss://jindosdk-dls-unit-test/ hdfs://myufs/ \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\u540e\uff0c\u652f\u6301\u4f7f\u7528 hdfs://myufs/ \u8bbf\u95ee oss://jindosdk-dls-unit-test/\u3002","title":"SetRootPolicy \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#unsetrootpolicy","text":"UnsetRootPolicy \u547d\u4ee4\u4e3a bucket \u53d6\u6d88\u8bbf\u95ee\u8def\u5f84\u3002 jindo admin -unsetRootPolicy [<dlsRootPath>] [<accessRootPath>] \u4f8b\uff1a jindo admin -unsetRootPolicy oss://jindosdk-dls-unit-test/ hdfs://myufs/ \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\u540e\uff0c\u4e0d\u518d\u652f\u6301\u7528 hdfs://myufs/ \u8bbf\u95ee oss://jindosdk-dls-unit-test/\u3002","title":"UnsetRootPolicy \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#listaccesspolicies","text":"ListAccessPolicies \u53ef\u67e5\u770b\u5f53\u524d bucket \u652f\u6301\u7684\u6240\u6709\u8bbf\u95ee\u8def\u5f84\u3002 jindo admin -listAccessPolicies [<dlsRootPath>] \u4f8b\uff1a jindo admin -listAccessPolicies oss://jindosdk-dls-unit-test/ \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\uff0c\u67e5\u770b oss://jindosdk-dls-unit-test/ \u7684\u6240\u6709\u8bbf\u95ee\u8def\u5f84\u3002","title":"ListAccessPolicies \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#_1","text":"","title":"\u5176\u4ed6"},{"location":"backup/jindofs/jindofs_dls_shell_howto/#dumpfile","text":"DumpFile \u547d\u4ee4\u53ef\u4ee5\u67e5\u770b\u6587\u4ef6\u7684\u8be6\u7ec6\u5b58\u50a8\u4fe1\u606f\u3002 jindo admin -dumpFile [-dlsUri <dlsUri>] \u4f8b\uff1a jindo admin -dumpFile -dlsUri oss://jindosdk-dls-unit-test/testDumpFile/dumpfile_simple_file \u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\u540e\uff0c\u5c06\u663e\u793a\u7c7b\u4f3c\u5982\u4e0b\u7ed3\u679c\uff1a =============Dump result============= File /testDumpFile/dumpfile_simple_file length: 11 blockSize: 33554432 modification time: 1657626266711 access time: 1657626266481 permission: 420 owner: root group: supergroup file id: 4180702376518228901 =============Block 0 block id: 8792388394945610760 offset: 0 numBytes: 11 gs: 1032 type: CLOUD loc: .dlsdata underConstruction: 0 isLastBlockComplete: 1","title":"DumpFile \u547d\u4ee4"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u7167\u4f7f\u7528\u8bf4\u660e \u00a7 (\u4ece 4.0.0 \u5f00\u59cb\u652f\u6301) \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u662fOSS\u65b0\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9HDFS\u63a5\u53e3, \u652f\u6301\u76ee\u5f55\u4ee5\u53ca\u76ee\u5f55\u5c42\u7ea7\uff0c\u901a\u8fc7Jindo SDK 4.x \u53ef\u4ee5\u517c\u5bb9\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u3002\u5173\u4e8e\u7528\u6237\u5982\u4f55\u521b\u5efa\u548c\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u7684\u57fa\u672c\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 \u94fe\u63a5 \u3002 OSS-HDFS \u670d\u52a1\u7684\u5feb\u7167\u529f\u80fd\u5728\u4f7f\u7528\u4e0a\u4e0eHDFS\u7684\u5feb\u7167\u529f\u80fd\u5b8c\u5168\u517c\u5bb9\u3002\u672c\u6587\u4e3b\u8981\u4ecb\u7ecd\u5728 OSS-HDFS \u670d\u52a1\u4e2d\u4f7f\u7528\u5feb\u7167\u529f\u80fd\u7684\u4e00\u4e9b\u5e38\u89c1\u64cd\u4f5c\u3002 \u4e3a\u4e86\u65b9\u4fbf\u4e0b\u9762\u547d\u4ee4\u7684\u4ecb\u7ecd\uff0c\u6211\u4eec\u5047\u8bbe oss://oss-dfs-test \u8fd9\u4e2aBucket\u5df2\u7ecf\u5f00\u542f OSS-HDFS \u670d\u52a1\u3002\u4e0b\u9762\u6240\u6709\u7684\u4f8b\u5b50\u90fd\u4f1a\u9488\u5bf9\u8fd9\u4e2aBucket\u8fdb\u884c\u64cd\u4f5c\u3002 \u5f00\u542f\u5feb\u7167\u529f\u80fd \u00a7 \u5047\u8bbe\u6211\u4eec\u5148\u5728 oss://oss-dfs-test \u4e0b\u521b\u5efa\u4e00\u4e2a\u76ee\u5f55TestSnapshot\u3002 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u662f\u5173\u95ed\u7684\u3002\u9700\u8981\u5f00\u542f\u548c\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\uff0c\u9700\u8981\u4f7f\u7528Jindo SDK\u7684shell\u547d\u4ee4\u884c\u3002\u5177\u4f53\u7684\u5f00\u542f\u5feb\u7167\u7684\u683c\u5f0f\u4e3a jindo admin -allowSnapshot -dlsUri <path> \u5982\u679c\u6211\u4eec\u8981\u5f00\u542f\u524d\u9762\u521b\u5efa\u7684TestSnapshot\u7684\u5feb\u7167\u529f\u80fd\u3002\u53ef\u4ee5\u8f93\u5165\u5982\u4e0b\u7684\u547d\u4ee4\uff1a jindo admin -allowSnapshot -dlsUri oss://oss-dfs-test.<Endpoint>/TestSnapshot \u521b\u5efa\u5feb\u7167 \u00a7 \u5728\u4e00\u4e2a\u76ee\u5f55\u5f00\u542f\u5feb\u7167\u529f\u80fd\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7HDFS\u7684shell\u547d\u4ee4\u3002\u5177\u4f53\u683c\u5f0f\u4e3a\uff1a hdfs dfs -createSnapshot <path> [<snapshotName>] \u4f8b\u5982\u4e0a\u9762\u4f8b\u5b50\u7684\u4e2d\u7684TestSnapshot\uff0c\u6211\u4eec\u8fdb\u884c\u4e86\u4e00\u4e9b\u76ee\u5f55\u548c\u6587\u4ef6\u7684\u64cd\u4f5c\u3002\u7136\u540e\u53ef\u4ee5\u521b\u5efa\u4e00\u4e2a\u5feb\u7167\u5bf9\u6b64\u523b\u7684\u5feb\u7167\u7684\u72b6\u6001\u8fdb\u884c\u4fdd\u5b58\u3002\u5047\u8bbe\u6211\u4eec\u547d\u540d\u5feb\u7167\u540d\u4e3aS1\u3002 #\u4e0b\u9762\u8fd9\u4e9b\u4e3a\u521b\u5efa\u6d4b\u8bd5\u7684\u6587\u4ef6\u548c\u76ee\u5f55 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir2 hdfs dfs -touchz oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1/file1 hdfs dfs -touchz oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir3/file2 #\u901a\u8fc7HDFS\u7684shell\u547d\u4ee4\u884c\u5de5\u5177\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u521b\u5efa\u5feb\u7167\uff0c\u5e76\u5c06\u5feb\u7167\u547d\u540d\u4e3aS1\uff1a hdfs dfs -createSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S1 \u8bbf\u95ee\u5feb\u7167\u4e2d\u7684\u76ee\u5f55\u548c\u6587\u4ef6 \u00a7 \u5feb\u7167\u76ee\u5f55\u7684\u683c\u5f0f \u00a7 \u4e3a\u4e86\u533a\u522b\u4e8e\u6587\u4ef6\u7cfb\u7edf\u73b0\u6709\u7684\u76ee\u5f55\u548c\u6587\u4ef6\uff0c\u8bbf\u95ee\u5feb\u7167\u4e2d\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u9700\u8981\u6dfb\u52a0\u5feb\u7167\u7684\u4fe1\u606f\u6765\u6307\u51fa\u81ea\u5df1\u8981\u8bbf\u95ee\u7684\u5feb\u7167\u540d\u3002\u5bf9\u4e8e\u4e00\u4e2a\u5f00\u542f\u4e86\u5feb\u7167\u7684\u76ee\u5f55\uff0c\u6211\u4eec\u8981\u8bbf\u95ee\u5b83\u4e0b\u9762\u67d0\u4e2a\u5feb\u7167\u7684\u76ee\u5f55\u548c\u6587\u4ef6\uff0c\u6211\u4eec\u53ef\u4ee5\u4e0b\u9762\u7684\u683c\u5f0f\u6765\u8bbf\u95ee\uff1a <snapshotRoot>/.snapshot/<snapshotName>/<actual subPath> \u5176\u4e2dsnapshotRoot\u4e3a\u5feb\u7167\u7684\u6839\u76ee\u5f55\uff0c\u4e5f\u5c31\u662f\u5f00\u542f\u5feb\u7167\u547d\u4ee4\u4e2ddlsUri\u53c2\u6570\u5236\u5b9a\u7684\u8def\u5f84\u3002snapshotName\u662f\u4e4b\u524d\u521b\u5efa\u7684\u5feb\u7167\u540d\u3002\u5236\u5b9a\u4e86\u5feb\u7167\u540d\u4e4b\u540e\uff0c\u4e4b\u540e\u7684\u8def\u5f84\u5c31\u662f\u539f\u6765\u5feb\u7167\u6839\u76ee\u5f55\uff08snapshotRoot\uff09\u4e0b\u8981\u8bbf\u95ee\u7684\u8def\u5f84\u3002 \u6bd4\u5982\u4e0a\u9762\u7684\u4f8b\u5b50TestSnapshot\u76ee\u5f55\uff0c\u6211\u4eec\u5982\u679c\u8981\u5217\u51fa oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u4e0b\u7684\u6587\u4ef6\u3002\u53ef\u4ee5\u7528\u5e38\u89c4\u7684ls\u547d\u4ee4\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u7531\u4e8e\u6211\u4eec\u5728TestSnapshot\u4e0b\u5f00\u542f\u7684\u5feb\u7167\uff0c\u5e76\u521b\u5efa\u4e86\u5feb\u7167S1\u3002\u6211\u4eec\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u8def\u5f84\u6765\u5217\u51fa\u5feb\u7167S1\u4e0b\u7684\u76ee\u5f55\u548c\u6587\u4ef6\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/.snapshot/S1/dir1 \u5176\u4e2d.snapshot/S1 \u5c31\u662f\u7528\u6765\u8868\u660e\u76ee\u5f55\u7684\u5feb\u7167\u7684\u4fe1\u606f\u3002 \u5229\u7528\u5feb\u7167\u6062\u590d\u6570\u636e \u00a7 \u5feb\u7167\u7684\u4e00\u4e2a\u5e38\u89c1\u4f5c\u7528\u662f\u8fdb\u884c\u6570\u636e\u5907\u4efd\u548c\u6062\u590d\u3002\u5229\u7528\u5feb\u7167\u53ef\u4ee5\u5bf9\u91cd\u8981\u6570\u636e\u8fdb\u884c\u6062\u590d\uff0c\u9632\u6b62\u7528\u6237\u9519\u8bef\u6027\u7684\u64cd\u4f5c\u3002\u53ea\u8981\u6839\u636e\u4e0a\u9762\u8bb2\u7684 \u5feb\u7167\u76ee\u5f55\u7684\u683c\u5f0f \u5c31\u53ef\u4ee5\u8bbf\u95ee\u5230\u53ea\u8bfb\u526f\u672c\u4e2d\u7684\u6570\u636e\uff0c\u7136\u540e\u53ef\u4ee5\u6839\u636e\u9700\u6c42\u8fdb\u884c\u6062\u590d\u64cd\u4f5c\u3002 \u5728\u4e0a\u9762\u7684\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u5047\u8bbe\u5728\u540e\u9762\u7684\u67d0\u6b21\u4efb\u52a1\u4e2d\u8bef\u5220\u4e86 oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 hdfs dfs -rm -r oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u7531\u4e8e\u6211\u4eec\u5df2\u7ecf\u5bf9/TestSnapshot\u6dfb\u52a0\u4e86\u5feb\u7167S1\u3002\u6211\u4eec\u53ef\u4ee5\u7528\u5982\u4e0b\u7684\u547d\u4ee4\u6062\u590d\u6570\u636e\u3002 hdfs dfs -cp oss://oss-dfs-test.<Endpoint>/TestSnapshot/.snapshot/S1/dir1 oss://oss-dfs-test.<Endpoint>/TestSnapshot \u8fd9\u4e2a\u65f6\u5019\u6211\u4eec\u518d\u8bbf\u95ee/TestSnapshot/dir1\uff0c\u5c31\u4f1a\u53d1\u73b0\u539f\u6765\u7684\u8bef\u5220\u7684\u6587\u4ef6\u5939\u548c\u6587\u4ef6\u90fd\u6062\u590d\u4e86\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u91cd\u547d\u540d\u5feb\u7167 \u00a7 \u5bf9\u4e8e\u4e00\u4e2a\u5df2\u7ecf\u521b\u5efa\u7684\u5feb\u7167\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u91cd\u547d\u540d hdfs dfs -renameSnapshot <path> <oldName> <newName> \u4f8b\u5982\u4e0a\u9762\u4f8b\u5b50\u7684\u4e2d\u7684TestSnapshot\u7684\u5feb\u7167S1\uff0c\u6211\u4eec\u53ef\u4ee5\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u91cd\u547d\u540d\u4e3aS100 hdfs dfs -renameSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S1 S100 \u5220\u9664\u5feb\u7167 \u00a7 \u5f53\u4e0d\u9700\u8981\u5feb\u7167\u7684\u65f6\u5019\uff0c\u53ef\u4ee5\u8fd0\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5220\u9664\u5236\u5b9a\u7684\u5feb\u7167\u3002 hdfs dfs -deleteSnapshot <path> <snapshotName> \u5728\u4e0a\u9762\u7684\u4f8b\u5b50\u4e2d\uff0c\u6211\u4eec\u53ef\u4ee5\u8fd0\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5220\u9664\u4e4b\u524d\u521b\u5efa\u5e76\u91cd\u547d\u540d\u7684\u5feb\u7167S100 hdfs dfs -deleteSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S100 \u5173\u95ed\u5feb\u7167\u529f\u80fd \u00a7 \u5f53\u6211\u4eec\u9700\u8981\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u7684\u65f6\u5019\uff0c\u6211\u4eec\u540c\u6837\u9700\u8981\u4f7f\u7528Jindo SDK\u7684shell\u547d\u4ee4\u884c\u3002\u5177\u4f53\u7684\u5173\u95ed\u5feb\u7167\u7684\u683c\u5f0f\u4e3a jindo admin -disallowSnapshot -dlsUri <path> \u67e5\u770b\u4e24\u4e2a\u5feb\u7167\u4e4b\u95f4\u7684\u5dee\u5f02 \u00a7 \u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u5f53\u547d\u4ee4\u67e5\u770b\u4e24\u4e2a\u5feb\u7167\u4e4b\u95f4\u7684\u5dee\u5f02 jindo dls -snapshotDiff -dlsUri <uri> -fromSnapshot <fromSnapshot> -toSnapshot <toSnapshot> \u6ce8\u610f\u4e8b\u9879 \u00a7 \u5982\u679c\u9700\u8981\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\uff0c\u9996\u5148\u8981\u786e\u4fdd\u8be5\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u5feb\u7167\u5df2\u7ecf\u88ab\u5220\u9664\u3002\u5220\u9664\u5feb\u7167\u7684\u547d\u4ee4\u53ef\u4ee5\u53c2\u8003\u524d\u9762\u7684\u5220\u9664\u5feb\u7167\u547d\u4ee4\u3002\u5982\u679c\u8be5\u76ee\u5f55\u4e0b\u8fd8\u5b58\u5728\u5feb\u7167\uff0c\u5173\u95ed\u5feb\u7167\u529f\u80fd\u4f1a\u62a5\u9519\u3002 \u5bf9\u4e8e\u6211\u4eec\u524d\u9762\u7684\u4f8b\u5b50TestSnapshot\uff0c\u5047\u8bbe\u6211\u4eec\u5df2\u7ecf\u5220\u9664\u4e86\u6240\u6709\u5feb\u7167\u3002\u6211\u4eec\u53ef\u4ee5\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u3002 jindo admin -disallowSnapshot -dlsUri oss://oss-dfs-test.<Endpoint>/TestSnapshot","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u7167\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#oss-hdfs-jindofs","text":"(\u4ece 4.0.0 \u5f00\u59cb\u652f\u6301) \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u662fOSS\u65b0\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9HDFS\u63a5\u53e3, \u652f\u6301\u76ee\u5f55\u4ee5\u53ca\u76ee\u5f55\u5c42\u7ea7\uff0c\u901a\u8fc7Jindo SDK 4.x \u53ef\u4ee5\u517c\u5bb9\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u3002\u5173\u4e8e\u7528\u6237\u5982\u4f55\u521b\u5efa\u548c\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u7684\u57fa\u672c\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 \u94fe\u63a5 \u3002 OSS-HDFS \u670d\u52a1\u7684\u5feb\u7167\u529f\u80fd\u5728\u4f7f\u7528\u4e0a\u4e0eHDFS\u7684\u5feb\u7167\u529f\u80fd\u5b8c\u5168\u517c\u5bb9\u3002\u672c\u6587\u4e3b\u8981\u4ecb\u7ecd\u5728 OSS-HDFS \u670d\u52a1\u4e2d\u4f7f\u7528\u5feb\u7167\u529f\u80fd\u7684\u4e00\u4e9b\u5e38\u89c1\u64cd\u4f5c\u3002 \u4e3a\u4e86\u65b9\u4fbf\u4e0b\u9762\u547d\u4ee4\u7684\u4ecb\u7ecd\uff0c\u6211\u4eec\u5047\u8bbe oss://oss-dfs-test \u8fd9\u4e2aBucket\u5df2\u7ecf\u5f00\u542f OSS-HDFS \u670d\u52a1\u3002\u4e0b\u9762\u6240\u6709\u7684\u4f8b\u5b50\u90fd\u4f1a\u9488\u5bf9\u8fd9\u4e2aBucket\u8fdb\u884c\u64cd\u4f5c\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u7167\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#_1","text":"\u5047\u8bbe\u6211\u4eec\u5148\u5728 oss://oss-dfs-test \u4e0b\u521b\u5efa\u4e00\u4e2a\u76ee\u5f55TestSnapshot\u3002 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u662f\u5173\u95ed\u7684\u3002\u9700\u8981\u5f00\u542f\u548c\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\uff0c\u9700\u8981\u4f7f\u7528Jindo SDK\u7684shell\u547d\u4ee4\u884c\u3002\u5177\u4f53\u7684\u5f00\u542f\u5feb\u7167\u7684\u683c\u5f0f\u4e3a jindo admin -allowSnapshot -dlsUri <path> \u5982\u679c\u6211\u4eec\u8981\u5f00\u542f\u524d\u9762\u521b\u5efa\u7684TestSnapshot\u7684\u5feb\u7167\u529f\u80fd\u3002\u53ef\u4ee5\u8f93\u5165\u5982\u4e0b\u7684\u547d\u4ee4\uff1a jindo admin -allowSnapshot -dlsUri oss://oss-dfs-test.<Endpoint>/TestSnapshot","title":"\u5f00\u542f\u5feb\u7167\u529f\u80fd"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#_2","text":"\u5728\u4e00\u4e2a\u76ee\u5f55\u5f00\u542f\u5feb\u7167\u529f\u80fd\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7HDFS\u7684shell\u547d\u4ee4\u3002\u5177\u4f53\u683c\u5f0f\u4e3a\uff1a hdfs dfs -createSnapshot <path> [<snapshotName>] \u4f8b\u5982\u4e0a\u9762\u4f8b\u5b50\u7684\u4e2d\u7684TestSnapshot\uff0c\u6211\u4eec\u8fdb\u884c\u4e86\u4e00\u4e9b\u76ee\u5f55\u548c\u6587\u4ef6\u7684\u64cd\u4f5c\u3002\u7136\u540e\u53ef\u4ee5\u521b\u5efa\u4e00\u4e2a\u5feb\u7167\u5bf9\u6b64\u523b\u7684\u5feb\u7167\u7684\u72b6\u6001\u8fdb\u884c\u4fdd\u5b58\u3002\u5047\u8bbe\u6211\u4eec\u547d\u540d\u5feb\u7167\u540d\u4e3aS1\u3002 #\u4e0b\u9762\u8fd9\u4e9b\u4e3a\u521b\u5efa\u6d4b\u8bd5\u7684\u6587\u4ef6\u548c\u76ee\u5f55 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir2 hdfs dfs -touchz oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1/file1 hdfs dfs -touchz oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir3/file2 #\u901a\u8fc7HDFS\u7684shell\u547d\u4ee4\u884c\u5de5\u5177\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u521b\u5efa\u5feb\u7167\uff0c\u5e76\u5c06\u5feb\u7167\u547d\u540d\u4e3aS1\uff1a hdfs dfs -createSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S1","title":"\u521b\u5efa\u5feb\u7167"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#_3","text":"","title":"\u8bbf\u95ee\u5feb\u7167\u4e2d\u7684\u76ee\u5f55\u548c\u6587\u4ef6"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#_4","text":"\u4e3a\u4e86\u533a\u522b\u4e8e\u6587\u4ef6\u7cfb\u7edf\u73b0\u6709\u7684\u76ee\u5f55\u548c\u6587\u4ef6\uff0c\u8bbf\u95ee\u5feb\u7167\u4e2d\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u9700\u8981\u6dfb\u52a0\u5feb\u7167\u7684\u4fe1\u606f\u6765\u6307\u51fa\u81ea\u5df1\u8981\u8bbf\u95ee\u7684\u5feb\u7167\u540d\u3002\u5bf9\u4e8e\u4e00\u4e2a\u5f00\u542f\u4e86\u5feb\u7167\u7684\u76ee\u5f55\uff0c\u6211\u4eec\u8981\u8bbf\u95ee\u5b83\u4e0b\u9762\u67d0\u4e2a\u5feb\u7167\u7684\u76ee\u5f55\u548c\u6587\u4ef6\uff0c\u6211\u4eec\u53ef\u4ee5\u4e0b\u9762\u7684\u683c\u5f0f\u6765\u8bbf\u95ee\uff1a <snapshotRoot>/.snapshot/<snapshotName>/<actual subPath> \u5176\u4e2dsnapshotRoot\u4e3a\u5feb\u7167\u7684\u6839\u76ee\u5f55\uff0c\u4e5f\u5c31\u662f\u5f00\u542f\u5feb\u7167\u547d\u4ee4\u4e2ddlsUri\u53c2\u6570\u5236\u5b9a\u7684\u8def\u5f84\u3002snapshotName\u662f\u4e4b\u524d\u521b\u5efa\u7684\u5feb\u7167\u540d\u3002\u5236\u5b9a\u4e86\u5feb\u7167\u540d\u4e4b\u540e\uff0c\u4e4b\u540e\u7684\u8def\u5f84\u5c31\u662f\u539f\u6765\u5feb\u7167\u6839\u76ee\u5f55\uff08snapshotRoot\uff09\u4e0b\u8981\u8bbf\u95ee\u7684\u8def\u5f84\u3002 \u6bd4\u5982\u4e0a\u9762\u7684\u4f8b\u5b50TestSnapshot\u76ee\u5f55\uff0c\u6211\u4eec\u5982\u679c\u8981\u5217\u51fa oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u4e0b\u7684\u6587\u4ef6\u3002\u53ef\u4ee5\u7528\u5e38\u89c4\u7684ls\u547d\u4ee4\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u7531\u4e8e\u6211\u4eec\u5728TestSnapshot\u4e0b\u5f00\u542f\u7684\u5feb\u7167\uff0c\u5e76\u521b\u5efa\u4e86\u5feb\u7167S1\u3002\u6211\u4eec\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u8def\u5f84\u6765\u5217\u51fa\u5feb\u7167S1\u4e0b\u7684\u76ee\u5f55\u548c\u6587\u4ef6\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/.snapshot/S1/dir1 \u5176\u4e2d.snapshot/S1 \u5c31\u662f\u7528\u6765\u8868\u660e\u76ee\u5f55\u7684\u5feb\u7167\u7684\u4fe1\u606f\u3002","title":"\u5feb\u7167\u76ee\u5f55\u7684\u683c\u5f0f"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#_5","text":"\u5feb\u7167\u7684\u4e00\u4e2a\u5e38\u89c1\u4f5c\u7528\u662f\u8fdb\u884c\u6570\u636e\u5907\u4efd\u548c\u6062\u590d\u3002\u5229\u7528\u5feb\u7167\u53ef\u4ee5\u5bf9\u91cd\u8981\u6570\u636e\u8fdb\u884c\u6062\u590d\uff0c\u9632\u6b62\u7528\u6237\u9519\u8bef\u6027\u7684\u64cd\u4f5c\u3002\u53ea\u8981\u6839\u636e\u4e0a\u9762\u8bb2\u7684 \u5feb\u7167\u76ee\u5f55\u7684\u683c\u5f0f \u5c31\u53ef\u4ee5\u8bbf\u95ee\u5230\u53ea\u8bfb\u526f\u672c\u4e2d\u7684\u6570\u636e\uff0c\u7136\u540e\u53ef\u4ee5\u6839\u636e\u9700\u6c42\u8fdb\u884c\u6062\u590d\u64cd\u4f5c\u3002 \u5728\u4e0a\u9762\u7684\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u5047\u8bbe\u5728\u540e\u9762\u7684\u67d0\u6b21\u4efb\u52a1\u4e2d\u8bef\u5220\u4e86 oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 hdfs dfs -rm -r oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u7531\u4e8e\u6211\u4eec\u5df2\u7ecf\u5bf9/TestSnapshot\u6dfb\u52a0\u4e86\u5feb\u7167S1\u3002\u6211\u4eec\u53ef\u4ee5\u7528\u5982\u4e0b\u7684\u547d\u4ee4\u6062\u590d\u6570\u636e\u3002 hdfs dfs -cp oss://oss-dfs-test.<Endpoint>/TestSnapshot/.snapshot/S1/dir1 oss://oss-dfs-test.<Endpoint>/TestSnapshot \u8fd9\u4e2a\u65f6\u5019\u6211\u4eec\u518d\u8bbf\u95ee/TestSnapshot/dir1\uff0c\u5c31\u4f1a\u53d1\u73b0\u539f\u6765\u7684\u8bef\u5220\u7684\u6587\u4ef6\u5939\u548c\u6587\u4ef6\u90fd\u6062\u590d\u4e86\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1","title":"\u5229\u7528\u5feb\u7167\u6062\u590d\u6570\u636e"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#_6","text":"\u5bf9\u4e8e\u4e00\u4e2a\u5df2\u7ecf\u521b\u5efa\u7684\u5feb\u7167\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u91cd\u547d\u540d hdfs dfs -renameSnapshot <path> <oldName> <newName> \u4f8b\u5982\u4e0a\u9762\u4f8b\u5b50\u7684\u4e2d\u7684TestSnapshot\u7684\u5feb\u7167S1\uff0c\u6211\u4eec\u53ef\u4ee5\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u91cd\u547d\u540d\u4e3aS100 hdfs dfs -renameSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S1 S100","title":"\u91cd\u547d\u540d\u5feb\u7167"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#_7","text":"\u5f53\u4e0d\u9700\u8981\u5feb\u7167\u7684\u65f6\u5019\uff0c\u53ef\u4ee5\u8fd0\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5220\u9664\u5236\u5b9a\u7684\u5feb\u7167\u3002 hdfs dfs -deleteSnapshot <path> <snapshotName> \u5728\u4e0a\u9762\u7684\u4f8b\u5b50\u4e2d\uff0c\u6211\u4eec\u53ef\u4ee5\u8fd0\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5220\u9664\u4e4b\u524d\u521b\u5efa\u5e76\u91cd\u547d\u540d\u7684\u5feb\u7167S100 hdfs dfs -deleteSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S100","title":"\u5220\u9664\u5feb\u7167"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#_8","text":"\u5f53\u6211\u4eec\u9700\u8981\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u7684\u65f6\u5019\uff0c\u6211\u4eec\u540c\u6837\u9700\u8981\u4f7f\u7528Jindo SDK\u7684shell\u547d\u4ee4\u884c\u3002\u5177\u4f53\u7684\u5173\u95ed\u5feb\u7167\u7684\u683c\u5f0f\u4e3a jindo admin -disallowSnapshot -dlsUri <path>","title":"\u5173\u95ed\u5feb\u7167\u529f\u80fd"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#_9","text":"\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u5f53\u547d\u4ee4\u67e5\u770b\u4e24\u4e2a\u5feb\u7167\u4e4b\u95f4\u7684\u5dee\u5f02 jindo dls -snapshotDiff -dlsUri <uri> -fromSnapshot <fromSnapshot> -toSnapshot <toSnapshot>","title":"\u67e5\u770b\u4e24\u4e2a\u5feb\u7167\u4e4b\u95f4\u7684\u5dee\u5f02"},{"location":"backup/jindofs/jindofs_dls_snapshot_howto/#_10","text":"\u5982\u679c\u9700\u8981\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\uff0c\u9996\u5148\u8981\u786e\u4fdd\u8be5\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u5feb\u7167\u5df2\u7ecf\u88ab\u5220\u9664\u3002\u5220\u9664\u5feb\u7167\u7684\u547d\u4ee4\u53ef\u4ee5\u53c2\u8003\u524d\u9762\u7684\u5220\u9664\u5feb\u7167\u547d\u4ee4\u3002\u5982\u679c\u8be5\u76ee\u5f55\u4e0b\u8fd8\u5b58\u5728\u5feb\u7167\uff0c\u5173\u95ed\u5feb\u7167\u529f\u80fd\u4f1a\u62a5\u9519\u3002 \u5bf9\u4e8e\u6211\u4eec\u524d\u9762\u7684\u4f8b\u5b50TestSnapshot\uff0c\u5047\u8bbe\u6211\u4eec\u5df2\u7ecf\u5220\u9664\u4e86\u6240\u6709\u5feb\u7167\u3002\u6211\u4eec\u53ef\u4ee5\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u3002 jindo admin -disallowSnapshot -dlsUri oss://oss-dfs-test.<Endpoint>/TestSnapshot","title":"\u6ce8\u610f\u4e8b\u9879"},{"location":"backup/jindofs/jindofs_dls_trash_howto/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u56de\u6536\u7ad9\u4f7f\u7528\u8bf4\u660e \u00a7 (\u4ece 4.5.1 \u5f00\u59cb\u652f\u6301) \u4ecb\u7ecd \u00a7 \u5f53\u4eceJindoFS\u5220\u9664\u6587\u4ef6\u65f6\uff0c\u6587\u4ef6\u4e0d\u662f\u7acb\u5373\u88ab\u7acb\u5373\u5220\u9664\u3002\u88ab\u5220\u9664\u7684\u6587\u4ef6\u88ab\u632a\u5230\u4e86 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\u3002\u5f53\u7ecf\u8fc730\u5206\u949f\u540e\uff0cCurrent\u76ee\u5f55\u4f1a\u88ab\u632a\u5230 ```bash /user/ /.Trash/ \u4e5f\u5c31\u662f\u8bf4\u67d0\u4e00\u65f6\u95f4\u6bb5\u5185\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u4f1a\u88ab\u5f52\u7c7b\u5230\u4e00\u4e2a\u5e26\u65f6\u95f4\u6233\u7684\u76ee\u5f55\u4e0b\uff0c\u8868\u793a\u662f\u5728\u8fd9\u4e2a\u65f6\u95f4\u5185\u88ab\u5220\u9664\u7684\uff0c\u76f8\u5f53\u4e8e\u4e00\u6b21checkpoint\u3002\u5f53\u7ecf\u8fc7**7\u5929**\u540e\uff0c\u8fd9\u4e2a<timestamp>\u76ee\u5f55\u5c06\u4f1a\u88ab\u6c38\u8fdc\u5220\u9664\u3002\u56e0\u6b64\u57287\u5929\u5185\uff0c\u60a8\u6709\u673a\u4f1a\u4ece.Trash\u76ee\u5f55\u4e0b\u627e\u5230\u5bf9\u5e94\u65f6\u523b\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u5c06\u5176\u4ece.Trash\u632a\u51fa\uff0c\u4ece\u800c\u6062\u590d\u5b83\u3002 \u6ce8\u610f\uff1a\u8fd9\u9879\u529f\u80fd\uff0c\u662f\u7531\u5ba2\u6237\u7aef\u548c\u670d\u52a1\u7aef\u914d\u5408\uff0c\u65b9\u80fd\u5f62\u6210\u56de\u6536\u7ad9\u7684\u529f\u80fd\u3002\u670d\u52a1\u7aef\u53ea\u8d1f\u8d23\u7ef4\u62a4 ```/user/<username>/.Trash``` \u7684\u5b9a\u65f6\u6e05\u7406\uff0c\u670d\u52a1\u7aef\u7684\u5b9a\u65f6\u6e05\u7406\u662f\u9ed8\u8ba4\u5f00\u542f\u7684\u3002\u5ba2\u6237\u7aef\u8d1f\u8d23\u5c06\u5f85\u5220\u9664\u6587\u4ef6\u632a\u5230.Trash\u76ee\u5f55\u4e0b\u3002 ## Hadoop FileSystem Shell \u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd ```bash hadoop fs -rm oss://bucket/a/b/c \u5ba2\u6237\u7aef\u7684Hadoop Shell\u547d\u4ee4\u9ed8\u8ba4\u4e0d\u5f00\u542fTrash\u529f\u80fd\uff0c\u56e0\u6b64\u9700\u8981\u5728core-site.xml\u91cc\u6dfb\u52a0\u4e00\u6761\u914d\u7f6e <property> <name>fs.trash.interval</name> <value>1440</value> </property> (\u8be5\u503c\u53ea\u9700\u5927\u4e8e0\u5373\u53ef) \u6b64\u65f6\u5728\u5ba2\u6237\u7aef\u4f1a\u81ea\u52a8\u5c06rm\u547d\u4ee4\u8f6c\u6362\u4e3a\u4e00\u6761 hadoop fs -mv oss://bucket/a/b/c /user/<username>/.Trash/Current/a/b/c \u547d\u4ee4\uff0c\u56e0\u6b64\uff0c\u60a8\u4e0d\u9700\u8981\u611f\u77e5\u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u670d\u52a1\u7aef\u4f1a\u8d1f\u8d23\u6e05\u7406\u3002 \u5982\u679c\u60a8\u60f3\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\uff0c\u91ca\u653e\u7a7a\u95f4\uff0c\u53ef\u4ee5\u6dfb\u52a0 -skipTrash \u53c2\u6570\uff0c\u6b64\u65f6\u5c06\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\u3002 Hadoop \u751f\u6001\u7ec4\u4ef6\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd \u00a7 Hive/Spark/Flink \u7b49\u7ec4\u4ef6\u5e76\u4e0d\u611f\u77e5 JindoFS \u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u4f7f\u7528 FileSystem(HCFS) \u7684 delete \u63a5\u53e3\u610f\u5473\u7740\u7acb\u5373\u5220\u9664\u3002\u56e0\u6b64\u5982\u679c\u60f3\u8981\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd\uff0c\u9700\u8981\u663e\u5f0f\u5730\u8c03\u7528 FileSystem \u7684 rename \u63a5\u53e3\uff0c\u5c06\u76ee\u6807\u6587\u4ef6\u624b\u52a8\u632a\u5230 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\uff0c\u7531 JindoFS \u670d\u52a1\u7aef\u8d1f\u8d23\u5b9a\u671f\u5220\u9664\u3002\u8fd9\u4e00\u70b9\u4e0a JindoFS \u91c7\u53d6\u4e86\u8ddf\u5f00\u6e90 Hadoop \u76f8\u4f3c\u7684\u7b56\u7565\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u56de\u6536\u7ad9\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindofs_dls_trash_howto/#oss-hdfs-jindofs","text":"(\u4ece 4.5.1 \u5f00\u59cb\u652f\u6301)","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u56de\u6536\u7ad9\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindofs_dls_trash_howto/#_1","text":"\u5f53\u4eceJindoFS\u5220\u9664\u6587\u4ef6\u65f6\uff0c\u6587\u4ef6\u4e0d\u662f\u7acb\u5373\u88ab\u7acb\u5373\u5220\u9664\u3002\u88ab\u5220\u9664\u7684\u6587\u4ef6\u88ab\u632a\u5230\u4e86 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\u3002\u5f53\u7ecf\u8fc730\u5206\u949f\u540e\uff0cCurrent\u76ee\u5f55\u4f1a\u88ab\u632a\u5230 ```bash /user/ /.Trash/ \u4e5f\u5c31\u662f\u8bf4\u67d0\u4e00\u65f6\u95f4\u6bb5\u5185\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u4f1a\u88ab\u5f52\u7c7b\u5230\u4e00\u4e2a\u5e26\u65f6\u95f4\u6233\u7684\u76ee\u5f55\u4e0b\uff0c\u8868\u793a\u662f\u5728\u8fd9\u4e2a\u65f6\u95f4\u5185\u88ab\u5220\u9664\u7684\uff0c\u76f8\u5f53\u4e8e\u4e00\u6b21checkpoint\u3002\u5f53\u7ecf\u8fc7**7\u5929**\u540e\uff0c\u8fd9\u4e2a<timestamp>\u76ee\u5f55\u5c06\u4f1a\u88ab\u6c38\u8fdc\u5220\u9664\u3002\u56e0\u6b64\u57287\u5929\u5185\uff0c\u60a8\u6709\u673a\u4f1a\u4ece.Trash\u76ee\u5f55\u4e0b\u627e\u5230\u5bf9\u5e94\u65f6\u523b\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u5c06\u5176\u4ece.Trash\u632a\u51fa\uff0c\u4ece\u800c\u6062\u590d\u5b83\u3002 \u6ce8\u610f\uff1a\u8fd9\u9879\u529f\u80fd\uff0c\u662f\u7531\u5ba2\u6237\u7aef\u548c\u670d\u52a1\u7aef\u914d\u5408\uff0c\u65b9\u80fd\u5f62\u6210\u56de\u6536\u7ad9\u7684\u529f\u80fd\u3002\u670d\u52a1\u7aef\u53ea\u8d1f\u8d23\u7ef4\u62a4 ```/user/<username>/.Trash``` \u7684\u5b9a\u65f6\u6e05\u7406\uff0c\u670d\u52a1\u7aef\u7684\u5b9a\u65f6\u6e05\u7406\u662f\u9ed8\u8ba4\u5f00\u542f\u7684\u3002\u5ba2\u6237\u7aef\u8d1f\u8d23\u5c06\u5f85\u5220\u9664\u6587\u4ef6\u632a\u5230.Trash\u76ee\u5f55\u4e0b\u3002 ## Hadoop FileSystem Shell \u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd ```bash hadoop fs -rm oss://bucket/a/b/c \u5ba2\u6237\u7aef\u7684Hadoop Shell\u547d\u4ee4\u9ed8\u8ba4\u4e0d\u5f00\u542fTrash\u529f\u80fd\uff0c\u56e0\u6b64\u9700\u8981\u5728core-site.xml\u91cc\u6dfb\u52a0\u4e00\u6761\u914d\u7f6e <property> <name>fs.trash.interval</name> <value>1440</value> </property> (\u8be5\u503c\u53ea\u9700\u5927\u4e8e0\u5373\u53ef) \u6b64\u65f6\u5728\u5ba2\u6237\u7aef\u4f1a\u81ea\u52a8\u5c06rm\u547d\u4ee4\u8f6c\u6362\u4e3a\u4e00\u6761 hadoop fs -mv oss://bucket/a/b/c /user/<username>/.Trash/Current/a/b/c \u547d\u4ee4\uff0c\u56e0\u6b64\uff0c\u60a8\u4e0d\u9700\u8981\u611f\u77e5\u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u670d\u52a1\u7aef\u4f1a\u8d1f\u8d23\u6e05\u7406\u3002 \u5982\u679c\u60a8\u60f3\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\uff0c\u91ca\u653e\u7a7a\u95f4\uff0c\u53ef\u4ee5\u6dfb\u52a0 -skipTrash \u53c2\u6570\uff0c\u6b64\u65f6\u5c06\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\u3002","title":"\u4ecb\u7ecd"},{"location":"backup/jindofs/jindofs_dls_trash_howto/#hadoop","text":"Hive/Spark/Flink \u7b49\u7ec4\u4ef6\u5e76\u4e0d\u611f\u77e5 JindoFS \u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u4f7f\u7528 FileSystem(HCFS) \u7684 delete \u63a5\u53e3\u610f\u5473\u7740\u7acb\u5373\u5220\u9664\u3002\u56e0\u6b64\u5982\u679c\u60f3\u8981\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd\uff0c\u9700\u8981\u663e\u5f0f\u5730\u8c03\u7528 FileSystem \u7684 rename \u63a5\u53e3\uff0c\u5c06\u76ee\u6807\u6587\u4ef6\u624b\u52a8\u632a\u5230 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\uff0c\u7531 JindoFS \u670d\u52a1\u7aef\u8d1f\u8d23\u5b9a\u671f\u5220\u9664\u3002\u8fd9\u4e00\u70b9\u4e0a JindoFS \u91c7\u53d6\u4e86\u8ddf\u5f00\u6e90 Hadoop \u76f8\u4f3c\u7684\u7b56\u7565\u3002","title":"Hadoop \u751f\u6001\u7ec4\u4ef6\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd"},{"location":"backup/jindofs/outline/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7528\u6237\u6587\u6863 \u00a7 \u5feb\u901f\u5165\u95e8 \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8 Hadoop \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 \u57fa\u672c\u529f\u80fd \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u7167\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5206\u5c42\u5b58\u50a8\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Shell \u547d\u4ee4\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u56de\u6536\u7ad9\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5143\u6570\u636e\u5bfc\u51fa\u4f7f\u7528\u8bf4\u660e \u5927\u6570\u636e\u751f\u6001 \u00a7 Hadoop \u7ec4\u4ef6 \u00a7 Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 Spark \u7ec4\u4ef6 \u00a7 Spark \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e Hive \u7ec4\u4ef6 \u00a7 Hive \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e Presto \u7ec4\u4ef6 \u00a7 Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e Impala \u7ec4\u4ef6 \u00a7 Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e HBase \u7ec4\u4ef6 \u00a7 HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 Flink \u7ec4\u4ef6 \u00a7 Flink \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e Flume \u7ec4\u4ef6 \u00a7 Flume \u4f7f\u7528 JindoSDK \u5199\u5165\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 AI \u751f\u6001 \u00a7 JindoFuse \u4f7f\u7528 \u00a7 \u4f7f\u7528 JindoFuse \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u6700\u4f73\u5b9e\u8df5 \u00a7 JindoDistCp \u6570\u636e\u8fc1\u79fb \u00a7 JindoDistCp \u6570\u636e\u8fc1\u79fb JindoTable \u6570\u4ed3\u8fc1\u79fb \u00a7 \u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1) \u534a\u6258\u7ba1 JindoFS \u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 \u534a\u6258\u7ba1 JindoFS \u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u4f7f\u7528 RootPolicy \u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 \u4f7f\u7528 RootPolicy \u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u5e38\u89c1\u95ee\u9898 \u00a7 \u5e38\u89c1\u95ee\u9898 \u5386\u53f2\u7248\u672c \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.6.0 \u7528\u6237\u6587\u6863 \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.5.1 \u7528\u6237\u6587\u6863 \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.5.0 \u7528\u6237\u6587\u6863 \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.4.0 \u7528\u6237\u6587\u6863 \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.3.0 \u7528\u6237\u6587\u6863 \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.2.0 \u7528\u6237\u6587\u6863 \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.1.0 \u7528\u6237\u6587\u6863 \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.0.0 \u7528\u6237\u6587\u6863 \u00a7","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7528\u6237\u6587\u6863"},{"location":"backup/jindofs/outline/#oss-hdfs-jindofs","text":"","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7528\u6237\u6587\u6863"},{"location":"backup/jindofs/outline/#_1","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8 Hadoop \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5","title":"\u5feb\u901f\u5165\u95e8"},{"location":"backup/jindofs/outline/#_2","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u7167\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5206\u5c42\u5b58\u50a8\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Shell \u547d\u4ee4\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u56de\u6536\u7ad9\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5143\u6570\u636e\u5bfc\u51fa\u4f7f\u7528\u8bf4\u660e","title":"\u57fa\u672c\u529f\u80fd"},{"location":"backup/jindofs/outline/#_3","text":"","title":"\u5927\u6570\u636e\u751f\u6001"},{"location":"backup/jindofs/outline/#hadoop","text":"Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09","title":"Hadoop \u7ec4\u4ef6"},{"location":"backup/jindofs/outline/#spark","text":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e","title":"Spark \u7ec4\u4ef6"},{"location":"backup/jindofs/outline/#hive","text":"Hive \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e","title":"Hive \u7ec4\u4ef6"},{"location":"backup/jindofs/outline/#presto","text":"Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e","title":"Presto \u7ec4\u4ef6"},{"location":"backup/jindofs/outline/#impala","text":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e","title":"Impala \u7ec4\u4ef6"},{"location":"backup/jindofs/outline/#hbase","text":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8","title":"HBase \u7ec4\u4ef6"},{"location":"backup/jindofs/outline/#flink","text":"Flink \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e","title":"Flink \u7ec4\u4ef6"},{"location":"backup/jindofs/outline/#flume","text":"Flume \u4f7f\u7528 JindoSDK \u5199\u5165\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09","title":"Flume \u7ec4\u4ef6"},{"location":"backup/jindofs/outline/#ai","text":"","title":"AI \u751f\u6001"},{"location":"backup/jindofs/outline/#jindofuse","text":"\u4f7f\u7528 JindoFuse \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09","title":"JindoFuse \u4f7f\u7528"},{"location":"backup/jindofs/outline/#_4","text":"","title":"\u6700\u4f73\u5b9e\u8df5"},{"location":"backup/jindofs/outline/#jindodistcp","text":"JindoDistCp \u6570\u636e\u8fc1\u79fb","title":"JindoDistCp \u6570\u636e\u8fc1\u79fb"},{"location":"backup/jindofs/outline/#jindotable","text":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1)","title":"JindoTable \u6570\u4ed3\u8fc1\u79fb"},{"location":"backup/jindofs/outline/#jindofs-oss-hdfs-jindofs","text":"\u534a\u6258\u7ba1 JindoFS \u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09","title":"\u534a\u6258\u7ba1 JindoFS \u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/outline/#rootpolicy-oss-hdfs-jindofs","text":"\u4f7f\u7528 RootPolicy \u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09","title":"\u4f7f\u7528 RootPolicy \u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/outline/#_5","text":"\u5e38\u89c1\u95ee\u9898","title":"\u5e38\u89c1\u95ee\u9898"},{"location":"backup/jindofs/outline/#_6","text":"","title":"\u5386\u53f2\u7248\u672c"},{"location":"backup/jindofs/outline/#oss-hdfs-jindofs-460","text":"","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.6.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofs/outline/#oss-hdfs-jindofs-451","text":"","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.5.1 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofs/outline/#oss-hdfs-jindofs-450","text":"","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.5.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofs/outline/#oss-hdfs-jindofs-440","text":"","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.4.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofs/outline/#oss-hdfs-jindofs-430","text":"","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.3.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofs/outline/#oss-hdfs-jindofs-420","text":"","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.2.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofs/outline/#oss-hdfs-jindofs-410","text":"","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.1.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofs/outline/#oss-hdfs-jindofs-400","text":"","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff094.0.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofs/configuration/jindosdk_configuration_list/","text":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e \u00a7 SDK\u914d\u7f6e\u9879\u5217\u8868 \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u5ba2\u6237\u7aef\u5199\u5165\u65f6\u7684\u4e34\u65f6\u6587\u4ef6\u76ee\u5f55\uff0c\u53ef\u914d\u7f6e\u591a\u4e2a\uff08\u9017\u53f7\u9694\u5f00\uff09\uff0c\u4f1a\u8f6e\u6d41\u5199\u5165\uff0c\u591a\u7528\u6237\u73af\u5883\u9700\u914d\u7f6e\u53ef\u8bfb\u5199\u6743\u9650 --> <name>fs.oss.tmp.data.dirs</name> <value>/tmp/</value> </property> <property> <!-- \u4e34\u65f6\u6587\u4ef6\u81ea\u6e05\u7406\u670d\u52a1 --> <name>fs.oss.tmp.data.cleaner.enable</name> <value>true</value> </property> <property> <!-- \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u6b21\u6570 --> <name>fs.oss.retry.count</name> <value>5</value> </property> <property> <!-- \u8bf7\u6c42 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 --> <name>fs.oss.timeout.millisecond</name> <value>30000</value> </property> <property> <!-- \u8fde\u63a5 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 --> <name>fs.oss.connection.timeout.millisecond</name> <value>3000</value> </property> <property> <!-- \u5355\u4e2a\u6587\u4ef6 oss \u5e76\u53d1\u4e0a\u4f20\u7ebf\u7a0b\u6570 --> <name>fs.oss.upload.thread.concurrency</name> <value>5</value> </property> <property> <!-- oss \u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u961f\u5217\u5927\u5c0f --> <name>fs.oss.upload.queue.size</name> <value>5</value> </property> <property> <!-- \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u6570 --> <name>fs.oss.upload.max.pending.tasks.per.stream</name> <value>16</value> </property> <property> <!-- oss \u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u961f\u5217\u5927\u5c0f --> <name>fs.oss.download.queue.size</name> <value>5</value> </property> <property> <!-- \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u6570 --> <name>fs.oss.download.thread.concurrency</name> <value>16</value> </property> <property> <!-- \u9884\u8bfb oss \u7684 buffer \u5927\u5c0f --> <name>fs.oss.read.readahead.buffer.size</name> <value>1048576</value> </property> <property> <!-- \u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 --> <name>fs.oss.read.readahead.buffer.count</name> <value>4</value> </property> <property> <!--oss \u5237\u65b0\u7f13\u51b2\u533a\u95f4\u9694\uff08\u6beb\u79d2\uff09\uff0c\u5c0f\u4e8e 0 \u65f6\u4e0d\u751f\u6548 --> <name>fs.oss.flush.interval.millisecond</name> <value>-1</value> </property> <property> <!-- \u6587\u4ef6\u7ea7\u522b crc64 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u6253\u5f00 --> <name>fs.oss.checksum.crc64.enable</name> <value>true</value> </property> <property> <!--\u8bf7\u6c42\u7ea7\u522b md5 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u5173\u95ed --> <name>fs.oss.checksum.md5.enable</name> <value>false</value> </property> </configuration> \u3010\u6ce8\uff1a4.5\u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u65b9\u5f0f\u3011","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/jindofs/configuration/jindosdk_configuration_list/#jindosdk","text":"","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/jindofs/configuration/jindosdk_configuration_list/#sdk","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u5ba2\u6237\u7aef\u5199\u5165\u65f6\u7684\u4e34\u65f6\u6587\u4ef6\u76ee\u5f55\uff0c\u53ef\u914d\u7f6e\u591a\u4e2a\uff08\u9017\u53f7\u9694\u5f00\uff09\uff0c\u4f1a\u8f6e\u6d41\u5199\u5165\uff0c\u591a\u7528\u6237\u73af\u5883\u9700\u914d\u7f6e\u53ef\u8bfb\u5199\u6743\u9650 --> <name>fs.oss.tmp.data.dirs</name> <value>/tmp/</value> </property> <property> <!-- \u4e34\u65f6\u6587\u4ef6\u81ea\u6e05\u7406\u670d\u52a1 --> <name>fs.oss.tmp.data.cleaner.enable</name> <value>true</value> </property> <property> <!-- \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u6b21\u6570 --> <name>fs.oss.retry.count</name> <value>5</value> </property> <property> <!-- \u8bf7\u6c42 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 --> <name>fs.oss.timeout.millisecond</name> <value>30000</value> </property> <property> <!-- \u8fde\u63a5 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 --> <name>fs.oss.connection.timeout.millisecond</name> <value>3000</value> </property> <property> <!-- \u5355\u4e2a\u6587\u4ef6 oss \u5e76\u53d1\u4e0a\u4f20\u7ebf\u7a0b\u6570 --> <name>fs.oss.upload.thread.concurrency</name> <value>5</value> </property> <property> <!-- oss \u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u961f\u5217\u5927\u5c0f --> <name>fs.oss.upload.queue.size</name> <value>5</value> </property> <property> <!-- \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u6570 --> <name>fs.oss.upload.max.pending.tasks.per.stream</name> <value>16</value> </property> <property> <!-- oss \u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u961f\u5217\u5927\u5c0f --> <name>fs.oss.download.queue.size</name> <value>5</value> </property> <property> <!-- \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u6570 --> <name>fs.oss.download.thread.concurrency</name> <value>16</value> </property> <property> <!-- \u9884\u8bfb oss \u7684 buffer \u5927\u5c0f --> <name>fs.oss.read.readahead.buffer.size</name> <value>1048576</value> </property> <property> <!-- \u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 --> <name>fs.oss.read.readahead.buffer.count</name> <value>4</value> </property> <property> <!--oss \u5237\u65b0\u7f13\u51b2\u533a\u95f4\u9694\uff08\u6beb\u79d2\uff09\uff0c\u5c0f\u4e8e 0 \u65f6\u4e0d\u751f\u6548 --> <name>fs.oss.flush.interval.millisecond</name> <value>-1</value> </property> <property> <!-- \u6587\u4ef6\u7ea7\u522b crc64 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u6253\u5f00 --> <name>fs.oss.checksum.crc64.enable</name> <value>true</value> </property> <property> <!--\u8bf7\u6c42\u7ea7\u522b md5 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u5173\u95ed --> <name>fs.oss.checksum.md5.enable</name> <value>false</value> </property> </configuration> \u3010\u6ce8\uff1a4.5\u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u65b9\u5f0f\u3011","title":"SDK\u914d\u7f6e\u9879\u5217\u8868"},{"location":"backup/jindofs/configuration/jindosdk_configuration_list_ini/","text":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e \u00a7 SDK\u914d\u7f6e\u9879\u5217\u8868 \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 jindosdk.cfg \u4e2d\u3002 [jindosdk] # \u5ba2\u6237\u7aef\u5199\u5165\u65f6\u7684\u4e34\u65f6\u6587\u4ef6\u76ee\u5f55\uff0c\u53ef\u914d\u7f6e\u591a\u4e2a\uff08\u9017\u53f7\u9694\u5f00\uff09\uff0c\u4f1a\u8f6e\u6d41\u5199\u5165\uff0c\u591a\u7528\u6237\u73af\u5883\u9700\u914d\u7f6e\u53ef\u8bfb\u5199\u6743\u9650 fs.oss.tmp.data.dirs=/tmp/ # \u4e34\u65f6\u6587\u4ef6\u81ea\u6e05\u7406\u670d\u52a1 fs.oss.tmp.data.cleaner.enable=true # \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u6b21\u6570 fs.oss.retry.count=5 # \u8bf7\u6c42 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 fs.oss.timeout.millisecond=30000 # \u8fde\u63a5 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 fs.oss.connection.timeout.millisecond=3000 # \u5355\u4e2a\u6587\u4ef6 oss \u5e76\u53d1\u4e0a\u4f20\u7ebf\u7a0b\u6570 fs.oss.upload.thread.concurrency=5 # oss \u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u961f\u5217\u5927\u5c0f fs.oss.upload.queue.size=5 # \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u6570 fs.oss.upload.max.pending.tasks.per.stream=16 # oss \u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u961f\u5217\u5927\u5c0f fs.oss.download.queue.size=5 # \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u6570 fs.oss.download.thread.concurrency=16 # \u9884\u8bfb oss \u7684 buffer \u5927\u5c0f fs.oss.read.readahead.buffer.size=1048576 # \u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 fs.oss.read.readahead.buffer.count=4 # oss \u5237\u65b0\u7f13\u51b2\u533a\u95f4\u9694\uff08\u6beb\u79d2\uff09\uff0c\u5c0f\u4e8e 0 \u65f6\u4e0d\u751f\u6548 fs.oss.flush.interval.millisecond=-1 # \u6587\u4ef6\u7ea7\u522b crc64 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u6253\u5f00 fs.oss.checksum.crc64.enable=true # \u8bf7\u6c42\u7ea7\u522b md5 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u5173\u95ed fs.oss.checksum.md5.enable=false \u3010\u6ce8\uff1a4.5\u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u65b9\u5f0f\u3011","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/jindofs/configuration/jindosdk_configuration_list_ini/#jindosdk","text":"","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/jindofs/configuration/jindosdk_configuration_list_ini/#sdk","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 jindosdk.cfg \u4e2d\u3002 [jindosdk] # \u5ba2\u6237\u7aef\u5199\u5165\u65f6\u7684\u4e34\u65f6\u6587\u4ef6\u76ee\u5f55\uff0c\u53ef\u914d\u7f6e\u591a\u4e2a\uff08\u9017\u53f7\u9694\u5f00\uff09\uff0c\u4f1a\u8f6e\u6d41\u5199\u5165\uff0c\u591a\u7528\u6237\u73af\u5883\u9700\u914d\u7f6e\u53ef\u8bfb\u5199\u6743\u9650 fs.oss.tmp.data.dirs=/tmp/ # \u4e34\u65f6\u6587\u4ef6\u81ea\u6e05\u7406\u670d\u52a1 fs.oss.tmp.data.cleaner.enable=true # \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u6b21\u6570 fs.oss.retry.count=5 # \u8bf7\u6c42 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 fs.oss.timeout.millisecond=30000 # \u8fde\u63a5 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 fs.oss.connection.timeout.millisecond=3000 # \u5355\u4e2a\u6587\u4ef6 oss \u5e76\u53d1\u4e0a\u4f20\u7ebf\u7a0b\u6570 fs.oss.upload.thread.concurrency=5 # oss \u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u961f\u5217\u5927\u5c0f fs.oss.upload.queue.size=5 # \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u6570 fs.oss.upload.max.pending.tasks.per.stream=16 # oss \u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u961f\u5217\u5927\u5c0f fs.oss.download.queue.size=5 # \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u6570 fs.oss.download.thread.concurrency=16 # \u9884\u8bfb oss \u7684 buffer \u5927\u5c0f fs.oss.read.readahead.buffer.size=1048576 # \u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 fs.oss.read.readahead.buffer.count=4 # oss \u5237\u65b0\u7f13\u51b2\u533a\u95f4\u9694\uff08\u6beb\u79d2\uff09\uff0c\u5c0f\u4e8e 0 \u65f6\u4e0d\u751f\u6548 fs.oss.flush.interval.millisecond=-1 # \u6587\u4ef6\u7ea7\u522b crc64 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u6253\u5f00 fs.oss.checksum.crc64.enable=true # \u8bf7\u6c42\u7ea7\u522b md5 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u5173\u95ed fs.oss.checksum.md5.enable=false \u3010\u6ce8\uff1a4.5\u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u65b9\u5f0f\u3011","title":"SDK\u914d\u7f6e\u9879\u5217\u8868"},{"location":"backup/jindofs/configuration/jindosdk_endpoint_configuration/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Endpoint \u914d\u7f6e \u00a7 \u6709\u4e09\u79cd\u65b9\u5f0f\u53ef\u4ee5\u6307\u5b9a Endpoint\uff0cJindoSDK \u4f1a\u4ee5\u4e0b\u4f18\u5148\u7ea7\u67e5\u627e Endpoint\uff1a \u65b9\u5f0f\u4e00\uff1a \u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u6307\u5b9a Endpoint\u3002\uff08\u63a8\u8350\uff09 \u00a7 \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u4f7f\u7528\u8fd9\u79cd\u683c\u5f0f\u7684\u8bbf\u95ee\u8def\u5f84 oss://<Bucket>.<Endpoint>/<Object> , \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u63a5\u53e3\u3002 \u65b9\u5f0f\u4e8c\uff1a \u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint\u3002 \u00a7 \u5982\u679c\u4f7f\u7528 oss://<Bucket>/<Object> \u8fd9\u79cd\u8bbf\u95ee\u8def\u5f84\uff0c\u5373\u8bbf\u95ee\u8def\u5f84\u4e2d\u6ca1\u6709 Endpoint, JindoSDK \u5219\u4f1a\u5728\u914d\u7f6e\u4e2d\u67e5\u627e Bucket \u7ea7\u522b\u7684 Endpoint\u3002 \u53ef\u5728 Hadoop \u7684 core-site.xml \u4e2d\u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint \u6307\u5411 OSS-HDFS \u670d\u52a1\u7684 Endpoint\u3002 <configuration> <property> <name>fs.oss.bucket.XXX.endpoint</name> <value>cn-xxx.oss-dls.aliyuncs.com</value> </property> </configuration> \u8bf4\u660e: XXX \u4e3a OSS-HDFS \u670d\u52a1 Bucket \u540d\u79f0\u3002 \u65b9\u5f0f\u4e09\uff1a\u914d\u7f6e\u5168\u5c40\u9ed8\u8ba4 Endpoint\u3002 \u00a7 \u5982\u679c\u4f7f\u7528 oss://<Bucket>/<Object> \u8fd9\u79cd\u8bbf\u95ee\u8def\u5f84\u800c\u4e14\u6ca1\u6709\u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint\uff0c\u5219\u4f1a\u7528\u5168\u5c40 Endpoint \u8bbf\u95ee\uff0c\u5982\u679c\u8981\u9ed8\u8ba4\u4f7f\u7528\u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff0c\u5219\u53ef\u7528\u8be5\u65b9\u6cd5\u3002 \u53ef\u5728 Hadoop \u7684 core-site.xml \u4e2d\u914d\u7f6e\u5168\u5c40\u9ed8\u8ba4 Endpoint\u3002 <configuration> <property> <name>fs.oss.endpoint</name> <value>cn-xxx.oss-dls.aliyuncs.com</value> </property> </configuration>","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Endpoint \u914d\u7f6e"},{"location":"backup/jindofs/configuration/jindosdk_endpoint_configuration/#oss-hdfs-jindofs-endpoint","text":"\u6709\u4e09\u79cd\u65b9\u5f0f\u53ef\u4ee5\u6307\u5b9a Endpoint\uff0cJindoSDK \u4f1a\u4ee5\u4e0b\u4f18\u5148\u7ea7\u67e5\u627e Endpoint\uff1a","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Endpoint \u914d\u7f6e"},{"location":"backup/jindofs/configuration/jindosdk_endpoint_configuration/#endpoint","text":"\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u4f7f\u7528\u8fd9\u79cd\u683c\u5f0f\u7684\u8bbf\u95ee\u8def\u5f84 oss://<Bucket>.<Endpoint>/<Object> , \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u63a5\u53e3\u3002","title":"\u65b9\u5f0f\u4e00\uff1a \u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u6307\u5b9a Endpoint\u3002\uff08\u63a8\u8350\uff09"},{"location":"backup/jindofs/configuration/jindosdk_endpoint_configuration/#bucket-endpoint","text":"\u5982\u679c\u4f7f\u7528 oss://<Bucket>/<Object> \u8fd9\u79cd\u8bbf\u95ee\u8def\u5f84\uff0c\u5373\u8bbf\u95ee\u8def\u5f84\u4e2d\u6ca1\u6709 Endpoint, JindoSDK \u5219\u4f1a\u5728\u914d\u7f6e\u4e2d\u67e5\u627e Bucket \u7ea7\u522b\u7684 Endpoint\u3002 \u53ef\u5728 Hadoop \u7684 core-site.xml \u4e2d\u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint \u6307\u5411 OSS-HDFS \u670d\u52a1\u7684 Endpoint\u3002 <configuration> <property> <name>fs.oss.bucket.XXX.endpoint</name> <value>cn-xxx.oss-dls.aliyuncs.com</value> </property> </configuration> \u8bf4\u660e: XXX \u4e3a OSS-HDFS \u670d\u52a1 Bucket \u540d\u79f0\u3002","title":"\u65b9\u5f0f\u4e8c\uff1a \u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint\u3002"},{"location":"backup/jindofs/configuration/jindosdk_endpoint_configuration/#endpoint_1","text":"\u5982\u679c\u4f7f\u7528 oss://<Bucket>/<Object> \u8fd9\u79cd\u8bbf\u95ee\u8def\u5f84\u800c\u4e14\u6ca1\u6709\u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint\uff0c\u5219\u4f1a\u7528\u5168\u5c40 Endpoint \u8bbf\u95ee\uff0c\u5982\u679c\u8981\u9ed8\u8ba4\u4f7f\u7528\u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff0c\u5219\u53ef\u7528\u8be5\u65b9\u6cd5\u3002 \u53ef\u5728 Hadoop \u7684 core-site.xml \u4e2d\u914d\u7f6e\u5168\u5c40\u9ed8\u8ba4 Endpoint\u3002 <configuration> <property> <name>fs.oss.endpoint</name> <value>cn-xxx.oss-dls.aliyuncs.com</value> </property> </configuration>","title":"\u65b9\u5f0f\u4e09\uff1a\u914d\u7f6e\u5168\u5c40\u9ed8\u8ba4 Endpoint\u3002"},{"location":"backup/jindofs/flink/jindosdk_on_flink/","text":"Flink \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 \u5728\u96c6\u7fa4\u4e0a\u6709\u5f00\u6e90\u7248\u672c Flink \u8f6f\u4ef6\uff0c\u7248\u672c\u4e0d\u4f4e\u4e8e 1.10.1\u3002 \u6ce8\uff1aFlink 1.16.0 \u53ca\u66f4\u9ad8\u7248\u672c\u7684\u53ef\u7528\u6027\u5c1a\u672a\u5f97\u5230\u9a8c\u8bc1\u3002 \u4e3a\u4ec0\u4e48 Flink \u9700\u8981\u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u00a7 \u76ee\u524d\uff0c\u5f00\u6e90\u7248\u672c Flink \u5bf9\u6d41\u5f0f\u5199\u5165 OSS-HDFS \u670d\u52a1\u5e76\u4e0d\u652f\u6301\uff0c\u66f4\u4e0d\u80fd\u652f\u6301 \u201c\u4e25\u683c\u4e00\u6b21\u201d\uff08EXACTLY_ONCE\uff09\u8bed\u4e49\uff0c\u5982\u6709\u8be5\u9700\u6c42\u5219\u9700\u8981\u4f7f\u7528 JindoSDK\u3002 \u6ce8\uff1aApache Flink \u81ea 1.16 \u7248\u672c\u8d77\u539f\u751f\u63d0\u4f9b\u4e86\u6d41\u5f0f\u5199\u5165 Aliyun OSS \u5bf9\u8c61\u5b58\u50a8\u7684\u652f\u6301\uff0c\u4f46\u4e0d\u80fd\u652f\u6301\u5199\u5165 OSS-HDFS \u670d\u52a1\u3002\u82e5\u914d\u7f6e\u8fc7 Apache Flink \u7684\u81ea\u6709\u5b9e\u73b0\uff0c\u9700\u79fb\u9664\uff0c\u89c1\u4e0b\u6587\u3002 JindoSDK \u914d\u7f6e \u00a7 \u9700\u8981\u5728\u6240\u6709 Flink \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flink \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e\u4e0b\u5217 .jar \u6587\u4ef6\uff1a * jindo-flink-${version}-full.jar \u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 plugins/flink/ \u76ee\u5f55\u4e0b\u627e\u5230\u3002 \u53e6\u5916\uff0c\u5982\u679c\u96c6\u7fa4\u5185\u5b58\u5728 Apache Flink \u63d0\u4f9b\u7684\u81ea\u5e26 Flink OSS Connector\uff0c\u9700\u79fb\u9664\u3002\u79fb\u9664\u65b9\u6cd5\u4e3a\uff0c\u4ece Flink \u96c6\u7fa4\u7684 lib \u76ee\u5f55\u6216 plugins/oss-fs-hadoop \u79fb\u8d70\u5982\u4e0b jar \u5305\uff1a * flink-oss-fs-hadoop-${flink-version}.jar \u6ce8\uff1aJindoSDK \u76ee\u524d\u63d0\u4f9b\u591a\u5e73\u53f0\u652f\u6301\uff0c\u5f53\u524d\u652f\u6301\u7684\u5e73\u53f0\u4ee5\u53ca\u9700\u8981\u989d\u5916\u653e\u7f6e\u7684\u8d44\u6e90\uff0c\u53c2\u8003 JindoData 4.6.4 \u7248\u672c\u8bf4\u660e \u5982\u4f55\u4f7f\u7528 \u00a7 \u914d\u7f6e\u597d SDK \u540e\uff0c\u65e0\u9700\u989d\u5916\u914d\u7f6e\uff0c\u4ee5\u5e38\u89c4 Flink \u6d41\u5f0f\u4f5c\u4e1a\u65b9\u6cd5\u4f7f\u7528\u5373\u53ef\uff0c\u6ce8\u610f\u4f7f\u7528\u6b63\u786e\u7684\u8def\u5f84\u3002\u5199\u5165 OSS-HDFS \u670d\u52a1\u987b\u4ee5 oss:// \u4e3a\u524d\u7f00\u3002JindoFS \u670d\u52a1\u4e0e\u963f\u91cc\u4e91 OSS \u4f7f\u7528\u76f8\u540c\u7684\u524d\u7f00\uff0cJindoSDK \u5185\u4f1a\u81ea\u52a8\u8fdb\u884c\u8bc6\u522b\u3002\u4e8c\u8005\u7684\u8054\u7cfb\uff0c\u53c2\u8003 JindoFS \u670d\u52a1\u4ecb\u7ecd \u3002 \u793a\u4f8b \u00a7 \u4e0b\u9762\u662f\u4e00\u4e2a\u7b80\u5355\u7684\u793a\u4f8b\u7a0b\u5e8f\uff0c\u5982\u679c\u5df2\u7ecf\u6309\u7167\u4e0a\u4e00\u8282\u914d\u7f6e\u597d SDK\uff0c\u5219\u4e0b\u5217\u7a0b\u5e8f\u5e94\u80fd\u6b63\u786e\u8fd0\u884c\uff1a (1) \u901a\u7528\u914d\u7f6e \u6253\u5f00 Flink \u7684 \u201c\u68c0\u67e5\u70b9\u201d\uff08Checkpoint\uff09\u3002\u4f8b\u5982\uff0cenv \u662f\u4e00\u4e2a\u5df2\u7ecf\u5efa\u7acb\u7684 StreamExecutionEnvironment\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u65b9\u5f0f\u5efa\u7acb\uff1a StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); \u5219\u53ef\u4ee5\u8fd9\u6837\u542f\u7528 checkpoint\uff1a env.enableCheckpointing(<userDefinedCheckpointInterval>, CheckpointingMode.EXACTLY_ONCE); \u4f7f\u7528\u53ef\u91cd\u53d1\u7684\u6570\u636e\u6e90\uff0c\u4f8b\u5982 Kafka (2) \u793a\u4f8b\u7a0b\u5e8f \u4e0b\u6587\u4e2d\uff0coutputStream \u662f\u4e00\u4e2a\u9884\u5148\u5f62\u6210\u7684 DataStream<String> \u5bf9\u8c61\uff0c\u5e76\u671f\u671b\u5c06\u5176\u5199\u5165 OSS\uff0c\u90a3\u4e48\u53ef\u4ee5\u8fd9\u6837\u6dfb\u52a0 sink\uff1a String outputPath = \"oss://<user-defined-bucket.dls-endpoint>/<user-defined-dir>\" StreamingFileSink<String> sink = StreamingFileSink.forRowFormat( new Path(outputPath), new SimpleStringEncoder<String>(\"UTF-8\") ).build(); outputStream.addSink(sink); \u6700\u540e\u7528 env.execute() \u6267\u884c Flink \u4f5c\u4e1a\u5373\u53ef\u3002 \u9ad8\u7ea7\u4f7f\u7528\uff1a\u81ea\u5b9a\u4e49\u53c2\u6570 \u00a7 \u7528\u6237\u5728\u63d0\u4ea4 Flink \u4f5c\u4e1a\u65f6\uff0c\u53ef\u4ee5\u81ea\u5b9a\u4e49\u914d\u7f6e\u4e00\u4e9b\u53c2\u6570\uff0c\u4ee5\u5f00\u542f\u6216\u63a7\u5236\u7279\u5b9a\u529f\u80fd\u3002\u4f8b\u5982\uff0c\u4ee5 yarn-cluster \u6a21\u5f0f\u63d0\u4ea4\u65f6\uff0c\u53ef\u4ee5\u901a\u8fc7 -yD \u8fdb\u884c\u914d\u7f6e\uff0c\u65b9\u5f0f\u7c7b\u4f3c\u4e8e\uff1a <flink_home>/bin/flink run -m yarn-cluster -yD key1=value1 -yD key2=value2 ... \u76ee\u524d\u652f\u6301\u901a\u8fc7\u914d\u7f6e\u5f00\u542f \u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\u3002 \u71b5\u6ce8\u5165 \u00a7 \u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\u7684\u76ee\u7684\u662f\uff0c\u5c06\u5199\u5165\u8def\u5f84\u7684\u4e00\u6bb5\u7279\u5b9a\u5b57\u7b26\u4e32\u5339\u914d\u51fa\u6765\uff0c\u7528\u4e00\u6bb5\u968f\u673a\u7684\u5b57\u7b26\u4e32\u8fdb\u884c\u66ff\u6362\uff0c\u4ee5\u524a\u5f31\u6240\u8c13 \u201c\u7247\u533a\u201d (sharding) \u6548\u5e94\uff0c\u63d0\u9ad8\u5199\u5165\u6548\u7387\u3002\u914d\u7f6e\u53c2\u6570\uff1a oss.entropy.key=<user-defined-key> oss.entropy.length=<user-defined-length> \u5982\u6b64\uff0c\u6bcf\u6b21\u6253\u5f00\u5199\u4e00\u4e2a\u65b0\u6587\u4ef6\u65f6\uff0c\u8def\u5f84\u4e2d\u4e0e <user-defined-key> \u76f8\u540c\u7684\u5b57\u7b26\u4e32\u4f1a\u88ab\u66ff\u6362\u4e3a\u4e00\u4e2a\u968f\u673a\u5b57\u7b26\u4e32\uff0c\u968f\u673a\u4e32\u7684\u957f\u5ea6\u4e3a <user-defined-length> \u3002\u914d\u7f6e\u8be5\u53c2\u6570\u65f6\uff0c <user-defined-length> \u5fc5\u987b\u5927\u4e8e\u96f6\u3002","title":"Flink \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofs/flink/jindosdk_on_flink/#flink-jindosdk-oss-hdfs-jindofs","text":"","title":"Flink \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofs/flink/jindosdk_on_flink/#_1","text":"\u5728\u96c6\u7fa4\u4e0a\u6709\u5f00\u6e90\u7248\u672c Flink \u8f6f\u4ef6\uff0c\u7248\u672c\u4e0d\u4f4e\u4e8e 1.10.1\u3002 \u6ce8\uff1aFlink 1.16.0 \u53ca\u66f4\u9ad8\u7248\u672c\u7684\u53ef\u7528\u6027\u5c1a\u672a\u5f97\u5230\u9a8c\u8bc1\u3002","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/jindofs/flink/jindosdk_on_flink/#flink-jindosdk-oss-hdfs","text":"\u76ee\u524d\uff0c\u5f00\u6e90\u7248\u672c Flink \u5bf9\u6d41\u5f0f\u5199\u5165 OSS-HDFS \u670d\u52a1\u5e76\u4e0d\u652f\u6301\uff0c\u66f4\u4e0d\u80fd\u652f\u6301 \u201c\u4e25\u683c\u4e00\u6b21\u201d\uff08EXACTLY_ONCE\uff09\u8bed\u4e49\uff0c\u5982\u6709\u8be5\u9700\u6c42\u5219\u9700\u8981\u4f7f\u7528 JindoSDK\u3002 \u6ce8\uff1aApache Flink \u81ea 1.16 \u7248\u672c\u8d77\u539f\u751f\u63d0\u4f9b\u4e86\u6d41\u5f0f\u5199\u5165 Aliyun OSS \u5bf9\u8c61\u5b58\u50a8\u7684\u652f\u6301\uff0c\u4f46\u4e0d\u80fd\u652f\u6301\u5199\u5165 OSS-HDFS \u670d\u52a1\u3002\u82e5\u914d\u7f6e\u8fc7 Apache Flink \u7684\u81ea\u6709\u5b9e\u73b0\uff0c\u9700\u79fb\u9664\uff0c\u89c1\u4e0b\u6587\u3002","title":"\u4e3a\u4ec0\u4e48 Flink \u9700\u8981\u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS-HDFS \u670d\u52a1"},{"location":"backup/jindofs/flink/jindosdk_on_flink/#jindosdk","text":"\u9700\u8981\u5728\u6240\u6709 Flink \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flink \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e\u4e0b\u5217 .jar \u6587\u4ef6\uff1a * jindo-flink-${version}-full.jar \u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 plugins/flink/ \u76ee\u5f55\u4e0b\u627e\u5230\u3002 \u53e6\u5916\uff0c\u5982\u679c\u96c6\u7fa4\u5185\u5b58\u5728 Apache Flink \u63d0\u4f9b\u7684\u81ea\u5e26 Flink OSS Connector\uff0c\u9700\u79fb\u9664\u3002\u79fb\u9664\u65b9\u6cd5\u4e3a\uff0c\u4ece Flink \u96c6\u7fa4\u7684 lib \u76ee\u5f55\u6216 plugins/oss-fs-hadoop \u79fb\u8d70\u5982\u4e0b jar \u5305\uff1a * flink-oss-fs-hadoop-${flink-version}.jar \u6ce8\uff1aJindoSDK \u76ee\u524d\u63d0\u4f9b\u591a\u5e73\u53f0\u652f\u6301\uff0c\u5f53\u524d\u652f\u6301\u7684\u5e73\u53f0\u4ee5\u53ca\u9700\u8981\u989d\u5916\u653e\u7f6e\u7684\u8d44\u6e90\uff0c\u53c2\u8003 JindoData 4.6.4 \u7248\u672c\u8bf4\u660e","title":"JindoSDK \u914d\u7f6e"},{"location":"backup/jindofs/flink/jindosdk_on_flink/#_2","text":"\u914d\u7f6e\u597d SDK \u540e\uff0c\u65e0\u9700\u989d\u5916\u914d\u7f6e\uff0c\u4ee5\u5e38\u89c4 Flink \u6d41\u5f0f\u4f5c\u4e1a\u65b9\u6cd5\u4f7f\u7528\u5373\u53ef\uff0c\u6ce8\u610f\u4f7f\u7528\u6b63\u786e\u7684\u8def\u5f84\u3002\u5199\u5165 OSS-HDFS \u670d\u52a1\u987b\u4ee5 oss:// \u4e3a\u524d\u7f00\u3002JindoFS \u670d\u52a1\u4e0e\u963f\u91cc\u4e91 OSS \u4f7f\u7528\u76f8\u540c\u7684\u524d\u7f00\uff0cJindoSDK \u5185\u4f1a\u81ea\u52a8\u8fdb\u884c\u8bc6\u522b\u3002\u4e8c\u8005\u7684\u8054\u7cfb\uff0c\u53c2\u8003 JindoFS \u670d\u52a1\u4ecb\u7ecd \u3002","title":"\u5982\u4f55\u4f7f\u7528"},{"location":"backup/jindofs/flink/jindosdk_on_flink/#_3","text":"\u4e0b\u9762\u662f\u4e00\u4e2a\u7b80\u5355\u7684\u793a\u4f8b\u7a0b\u5e8f\uff0c\u5982\u679c\u5df2\u7ecf\u6309\u7167\u4e0a\u4e00\u8282\u914d\u7f6e\u597d SDK\uff0c\u5219\u4e0b\u5217\u7a0b\u5e8f\u5e94\u80fd\u6b63\u786e\u8fd0\u884c\uff1a (1) \u901a\u7528\u914d\u7f6e \u6253\u5f00 Flink \u7684 \u201c\u68c0\u67e5\u70b9\u201d\uff08Checkpoint\uff09\u3002\u4f8b\u5982\uff0cenv \u662f\u4e00\u4e2a\u5df2\u7ecf\u5efa\u7acb\u7684 StreamExecutionEnvironment\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u65b9\u5f0f\u5efa\u7acb\uff1a StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); \u5219\u53ef\u4ee5\u8fd9\u6837\u542f\u7528 checkpoint\uff1a env.enableCheckpointing(<userDefinedCheckpointInterval>, CheckpointingMode.EXACTLY_ONCE); \u4f7f\u7528\u53ef\u91cd\u53d1\u7684\u6570\u636e\u6e90\uff0c\u4f8b\u5982 Kafka (2) \u793a\u4f8b\u7a0b\u5e8f \u4e0b\u6587\u4e2d\uff0coutputStream \u662f\u4e00\u4e2a\u9884\u5148\u5f62\u6210\u7684 DataStream<String> \u5bf9\u8c61\uff0c\u5e76\u671f\u671b\u5c06\u5176\u5199\u5165 OSS\uff0c\u90a3\u4e48\u53ef\u4ee5\u8fd9\u6837\u6dfb\u52a0 sink\uff1a String outputPath = \"oss://<user-defined-bucket.dls-endpoint>/<user-defined-dir>\" StreamingFileSink<String> sink = StreamingFileSink.forRowFormat( new Path(outputPath), new SimpleStringEncoder<String>(\"UTF-8\") ).build(); outputStream.addSink(sink); \u6700\u540e\u7528 env.execute() \u6267\u884c Flink \u4f5c\u4e1a\u5373\u53ef\u3002","title":"\u793a\u4f8b"},{"location":"backup/jindofs/flink/jindosdk_on_flink/#_4","text":"\u7528\u6237\u5728\u63d0\u4ea4 Flink \u4f5c\u4e1a\u65f6\uff0c\u53ef\u4ee5\u81ea\u5b9a\u4e49\u914d\u7f6e\u4e00\u4e9b\u53c2\u6570\uff0c\u4ee5\u5f00\u542f\u6216\u63a7\u5236\u7279\u5b9a\u529f\u80fd\u3002\u4f8b\u5982\uff0c\u4ee5 yarn-cluster \u6a21\u5f0f\u63d0\u4ea4\u65f6\uff0c\u53ef\u4ee5\u901a\u8fc7 -yD \u8fdb\u884c\u914d\u7f6e\uff0c\u65b9\u5f0f\u7c7b\u4f3c\u4e8e\uff1a <flink_home>/bin/flink run -m yarn-cluster -yD key1=value1 -yD key2=value2 ... \u76ee\u524d\u652f\u6301\u901a\u8fc7\u914d\u7f6e\u5f00\u542f \u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\u3002","title":"\u9ad8\u7ea7\u4f7f\u7528\uff1a\u81ea\u5b9a\u4e49\u53c2\u6570"},{"location":"backup/jindofs/flink/jindosdk_on_flink/#_5","text":"\u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\u7684\u76ee\u7684\u662f\uff0c\u5c06\u5199\u5165\u8def\u5f84\u7684\u4e00\u6bb5\u7279\u5b9a\u5b57\u7b26\u4e32\u5339\u914d\u51fa\u6765\uff0c\u7528\u4e00\u6bb5\u968f\u673a\u7684\u5b57\u7b26\u4e32\u8fdb\u884c\u66ff\u6362\uff0c\u4ee5\u524a\u5f31\u6240\u8c13 \u201c\u7247\u533a\u201d (sharding) \u6548\u5e94\uff0c\u63d0\u9ad8\u5199\u5165\u6548\u7387\u3002\u914d\u7f6e\u53c2\u6570\uff1a oss.entropy.key=<user-defined-key> oss.entropy.length=<user-defined-length> \u5982\u6b64\uff0c\u6bcf\u6b21\u6253\u5f00\u5199\u4e00\u4e2a\u65b0\u6587\u4ef6\u65f6\uff0c\u8def\u5f84\u4e2d\u4e0e <user-defined-key> \u76f8\u540c\u7684\u5b57\u7b26\u4e32\u4f1a\u88ab\u66ff\u6362\u4e3a\u4e00\u4e2a\u968f\u673a\u5b57\u7b26\u4e32\uff0c\u968f\u673a\u4e32\u7684\u957f\u5ea6\u4e3a <user-defined-length> \u3002\u914d\u7f6e\u8be5\u53c2\u6570\u65f6\uff0c <user-defined-length> \u5fc5\u987b\u5927\u4e8e\u96f6\u3002","title":"\u71b5\u6ce8\u5165"},{"location":"backup/jindofs/flume/jindosdk_on_flume/","text":"Flume \u4f7f\u7528 JindoSDK \u5199\u5165\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 \u5728\u96c6\u7fa4\u4e0a\u5df2\u7ecf\u90e8\u7f72 Flume\uff0c\u5df2\u90e8\u7f72 JindoSDK 4.0 \u4ee5\u4e0a\u7248\u672c\u3002 \u4e3a\u4ec0\u4e48 Flume \u9700\u8981\u4f7f\u7528 JindoSDK \u5199\u5165 OSS-HDFS \u670d\u52a1 \u00a7 Flume \u901a\u8fc7 flush() \u8c03\u7528\u4fdd\u8bc1\u4e8b\u52a1\u6027\u5199\u5165\uff0c\u901a\u8fc7 JindoSDK \u5199\u5165 OSS-HDFS \u670d\u52a1\uff0c\u53ef\u4ee5\u8ba9 flush \u540e\u7684\u6570\u636e\u7acb\u523b\u53ef\u89c1\uff0c\u4fdd\u8bc1\u6570\u636e\u4e0d\u4e22\u5931\u3002 SDK \u914d\u7f6e \u00a7 \u9700\u8981\u5728 Flume \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flume \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e JindoSDK\u3002 \u4e0b\u8f7d\u5e76\u89e3\u538b\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 ) cp jindosdk-x.x.x/lib/*.jar $FLUME_HOME/jars/ Sink \u914d\u7f6e\u793a\u4f8b \u00a7 # \u914d\u7f6e OSS Sink xxx.sinks.oss_sink.hdfs.path = oss://${your_bucket}/flume_dir/%Y-%m-%d/%H # Sink\u53c2\u6570\uff0cbatchSize \u9700\u8981\u8bbe\u7f6e\u5927\u4e00\u4e9b\uff0c\u63a8\u8350\u6bcf\u6b21 Flush \u7684\u91cf\u5728 32MB\u4ee5\u4e0a\uff0c\u5426\u5219\u4f1a\u5f71\u54cd\u6027\u80fd xxx.sinks.oss_sink.hdfs.batchSize = 100000 ... xxx.sinks.oss_sink.hdfs.round = true xxx.sinks.oss_sink.hdfs.roundValue = 15 xxx.sinks.oss_sink.hdfs.Unit = minute xxx.sinks.oss_sink.hdfs.filePrefix = your_topic xxx.sinks.oss_sink.rollSize = 3600 xxx.sinks.oss_sink.threadsPoolSize = 30 ...","title":"Flume \u4f7f\u7528 JindoSDK \u5199\u5165\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/flume/jindosdk_on_flume/#flume-jindosdk-oss-hdfs-jindofs","text":"","title":"Flume \u4f7f\u7528 JindoSDK \u5199\u5165\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/flume/jindosdk_on_flume/#_1","text":"\u5728\u96c6\u7fa4\u4e0a\u5df2\u7ecf\u90e8\u7f72 Flume\uff0c\u5df2\u90e8\u7f72 JindoSDK 4.0 \u4ee5\u4e0a\u7248\u672c\u3002","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/jindofs/flume/jindosdk_on_flume/#flume-jindosdk-oss-hdfs","text":"Flume \u901a\u8fc7 flush() \u8c03\u7528\u4fdd\u8bc1\u4e8b\u52a1\u6027\u5199\u5165\uff0c\u901a\u8fc7 JindoSDK \u5199\u5165 OSS-HDFS \u670d\u52a1\uff0c\u53ef\u4ee5\u8ba9 flush \u540e\u7684\u6570\u636e\u7acb\u523b\u53ef\u89c1\uff0c\u4fdd\u8bc1\u6570\u636e\u4e0d\u4e22\u5931\u3002","title":"\u4e3a\u4ec0\u4e48 Flume \u9700\u8981\u4f7f\u7528 JindoSDK \u5199\u5165 OSS-HDFS \u670d\u52a1"},{"location":"backup/jindofs/flume/jindosdk_on_flume/#sdk","text":"\u9700\u8981\u5728 Flume \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flume \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e JindoSDK\u3002 \u4e0b\u8f7d\u5e76\u89e3\u538b\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 ) cp jindosdk-x.x.x/lib/*.jar $FLUME_HOME/jars/","title":"SDK \u914d\u7f6e"},{"location":"backup/jindofs/flume/jindosdk_on_flume/#sink","text":"# \u914d\u7f6e OSS Sink xxx.sinks.oss_sink.hdfs.path = oss://${your_bucket}/flume_dir/%Y-%m-%d/%H # Sink\u53c2\u6570\uff0cbatchSize \u9700\u8981\u8bbe\u7f6e\u5927\u4e00\u4e9b\uff0c\u63a8\u8350\u6bcf\u6b21 Flush \u7684\u91cf\u5728 32MB\u4ee5\u4e0a\uff0c\u5426\u5219\u4f1a\u5f71\u54cd\u6027\u80fd xxx.sinks.oss_sink.hdfs.batchSize = 100000 ... xxx.sinks.oss_sink.hdfs.round = true xxx.sinks.oss_sink.hdfs.roundValue = 15 xxx.sinks.oss_sink.hdfs.Unit = minute xxx.sinks.oss_sink.hdfs.filePrefix = your_topic xxx.sinks.oss_sink.rollSize = 3600 xxx.sinks.oss_sink.threadsPoolSize = 30 ...","title":"Sink \u914d\u7f6e\u793a\u4f8b"},{"location":"backup/jindofs/hadoop/jindosdk_ide_hadoop/","text":"Hadoop \u4f7f\u7528 JindoSDK \u96c6\u6210\u5f00\u53d1 \u00a7 \u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindosdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.net.URI; public class TestJindoSDK { public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. FileSystem fs = FileSystem.get(URI.create(\"oss://<Bucket>.<HDFS_Endpoint>/\"), conf); FSDataInputStream in = fs.open(new Path(\"/uttest/file1\")); in.read(); in.close(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency>","title":"Hadoop \u4f7f\u7528 JindoSDK \u96c6\u6210\u5f00\u53d1"},{"location":"backup/jindofs/hadoop/jindosdk_ide_hadoop/#hadoop-jindosdk","text":"\u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindosdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.net.URI; public class TestJindoSDK { public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. FileSystem fs = FileSystem.get(URI.create(\"oss://<Bucket>.<HDFS_Endpoint>/\"), conf); FSDataInputStream in = fs.open(new Path(\"/uttest/file1\")); in.read(); in.close(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency>","title":"Hadoop \u4f7f\u7528 JindoSDK \u96c6\u6210\u5f00\u53d1"},{"location":"backup/jindofs/hadoop/jindosdk_on_hadoop/","text":"Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 JindoSDK \u4e3a Apache Hadoop\u7684\u8ba1\u7b97\u5206\u6790\u5e94\u7528\uff08\u4f8b\u5982MapReduce\u3001Hive\u3001Spark\u3001Flink\u7b49\uff09\u63d0\u4f9b\u4e86\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u529f\u80fd\u3002 \u6b65\u9aa4 \u00a7 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u5c06 JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002 4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u00a7 \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 JindoFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 5. \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u00a7 \u7528 Hadoop Shell \u8bbf\u95ee JindoFS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<Bucket>.<Endpoint>/ ls \u64cd\u4f5c hadoop fs -ls oss://<Bucket>.<Endpoint>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<Bucket>.<Endpoint>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<Bucket>.<Endpoint>/<path> 6. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/hadoop/jindosdk_on_hadoop/#hadoop-jindosdk-oss-hdfs-jindofs","text":"JindoSDK \u4e3a Apache Hadoop\u7684\u8ba1\u7b97\u5206\u6790\u5e94\u7528\uff08\u4f8b\u5982MapReduce\u3001Hive\u3001Spark\u3001Flink\u7b49\uff09\u63d0\u4f9b\u4e86\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u529f\u80fd\u3002","title":"Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/hadoop/jindosdk_on_hadoop/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofs/hadoop/jindosdk_on_hadoop/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindofs/hadoop/jindosdk_on_hadoop/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofs/hadoop/jindosdk_on_hadoop/#3-oss-hdfs-access-key","text":"\u5c06 JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002","title":"3. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"backup/jindofs/hadoop/jindosdk_on_hadoop/#4-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 JindoFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002","title":"4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint"},{"location":"backup/jindofs/hadoop/jindosdk_on_hadoop/#5-jindosdk-oss-hdfs","text":"\u7528 Hadoop Shell \u8bbf\u95ee JindoFS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<Bucket>.<Endpoint>/ ls \u64cd\u4f5c hadoop fs -ls oss://<Bucket>.<Endpoint>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<Bucket>.<Endpoint>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<Bucket>.<Endpoint>/<path>","title":"5. \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS-HDFS \u670d\u52a1"},{"location":"backup/jindofs/hadoop/jindosdk_on_hadoop/#6","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"6. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofs/hbase/jindosdk_on_hbase/","text":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 \u00a7 HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002 OSS-HDFS \u670d\u52a1\u662f\u963f\u91cc\u4e91\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9 HDFS \u63a5\u53e3, JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\u53ca WAL \u6587\u4ef6\u7684\u5b58\u50a8\u5b9e\u73b0\u5b58\u7b97\u5206\u79bb\uff0c\u76f8\u5bf9\u4e8e\u672c\u5730HDFS\u5b58\u50a8\uff0c\u4f7f\u7528\u66f4\u52a0\u7075\u6d3b\uff0c\u51cf\u5c11\u8fd0\u7ef4\u6210\u672c\u3002 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06\u5df2\u5f00\u542f OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u00a7 \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 JindoFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 5. \u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84 \u00a7 \u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u548c hbase.wal.dir \u7684\u503c\u4e3a OSS-HDFS \u670d\u52a1\u5730\u5740\uff0c\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84\u548c WAL \u6587\u4ef6\u7684\u5b58\u50a8\u8def\u5f84\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 OSS-HDFS \u670d\u52a1\u3002\u53c2\u6570\u503c\u4e3a oss://bucket.endpoint/hbase-root-dir hbase.wal.dir \u6307\u5b9a HBase \u7684 WAL \u5b58\u50a8\u76ee\u5f55\u5230 OSS-HDFS \u670d\u52a1\u3002 \u53c2\u6570\u503c\u4e3a oss://bucket.endpoint/hbase","title":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8"},{"location":"backup/jindofs/hbase/jindosdk_on_hbase/#hbase-oss-hdfs-jindofs","text":"HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002 OSS-HDFS \u670d\u52a1\u662f\u963f\u91cc\u4e91\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9 HDFS \u63a5\u53e3, JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\u53ca WAL \u6587\u4ef6\u7684\u5b58\u50a8\u5b9e\u73b0\u5b58\u7b97\u5206\u79bb\uff0c\u76f8\u5bf9\u4e8e\u672c\u5730HDFS\u5b58\u50a8\uff0c\u4f7f\u7528\u66f4\u52a0\u7075\u6d3b\uff0c\u51cf\u5c11\u8fd0\u7ef4\u6210\u672c\u3002","title":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8"},{"location":"backup/jindofs/hbase/jindosdk_on_hbase/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindofs/hbase/jindosdk_on_hbase/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofs/hbase/jindosdk_on_hbase/#3-oss-hdfs-access-key","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06\u5df2\u5f00\u542f OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002","title":"3. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"backup/jindofs/hbase/jindosdk_on_hbase/#4-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 JindoFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002","title":"4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint"},{"location":"backup/jindofs/hbase/jindosdk_on_hbase/#5-hbase","text":"\u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u548c hbase.wal.dir \u7684\u503c\u4e3a OSS-HDFS \u670d\u52a1\u5730\u5740\uff0c\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84\u548c WAL \u6587\u4ef6\u7684\u5b58\u50a8\u8def\u5f84\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 OSS-HDFS \u670d\u52a1\u3002\u53c2\u6570\u503c\u4e3a oss://bucket.endpoint/hbase-root-dir hbase.wal.dir \u6307\u5b9a HBase \u7684 WAL \u5b58\u50a8\u76ee\u5f55\u5230 OSS-HDFS \u670d\u52a1\u3002 \u53c2\u6570\u503c\u4e3a oss://bucket.endpoint/hbase","title":"5. \u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84"},{"location":"backup/jindofs/hive/jindosdk_on_hive/","text":"Hive \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e \u00a7 Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002 \u5b89\u88c5\u914d\u7f6e \u00a7 1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/ 2. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u914d\u7f6e JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b \u5c06 JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key \u5c06\u5df2\u5f00\u542f OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 3. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u00a7 \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u6700\u540e\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 4. Hive on MR \u00a7 \u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002 5. Hive on Tez \u00a7 \u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002 6. Hive on Spark \u00a7 \u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002 OSS-HDFS \u670d\u52a1\u7528\u4e8e\u8868\u7684\u5b58\u50a8 \u00a7 \u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS-HDFS \u670d\u52a1\u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'oss://bucket_name.endpoint_name/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'oss://bucket_name.endpoint_name/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS-HDFS \u670d\u52a1\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>oss://bucket_name.endpoint_name/path/to/warehouse</value> </property> </configuration> \u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a \u00a7 ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'oss://bucket_name.endpoint_name/path/to/us/part210301cn'; \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Hive \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#hive-jindosdk-oss-hdfs-jindofs","text":"Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002","title":"Hive \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#_1","text":"","title":"\u5b89\u88c5\u914d\u7f6e"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#1-hive-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/","title":"1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#2-oss-hdfs-access-key","text":"\u914d\u7f6e JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b \u5c06 JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key \u5c06\u5df2\u5f00\u542f OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002","title":"2. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#3-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u6700\u540e\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002","title":"3. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#4-hive-on-mr","text":"\u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002","title":"4. Hive on MR"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#5-hive-on-tez","text":"\u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002","title":"5. Hive on Tez"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#6-hive-on-spark","text":"\u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002","title":"6. Hive on Spark"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#oss-hdfs","text":"\u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS-HDFS \u670d\u52a1\u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'oss://bucket_name.endpoint_name/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'oss://bucket_name.endpoint_name/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS-HDFS \u670d\u52a1\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>oss://bucket_name.endpoint_name/path/to/warehouse</value> </property> </configuration>","title":"OSS-HDFS \u670d\u52a1\u7528\u4e8e\u8868\u7684\u5b58\u50a8"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#oss","text":"ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'oss://bucket_name.endpoint_name/path/to/us/part210301cn';","title":"\u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a"},{"location":"backup/jindofs/hive/jindosdk_on_hive/#_2","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofs/impala/jindosdk_on_impala/","text":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e \u00a7 JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Impala \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Impala \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fddImpala\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728\u6240\u6709 Impala \u8282\u70b9\u5b89\u88c5 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06 sdk \u5305\u5b89\u88c5\u5230 Impala \u7684 classpath \u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $IMPALA_HOME/lib/ 3. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e IMPALA \u4f7f\u7528\u7684 core-site.xml \u914d\u7f6e JindoSDK \u8bbf\u95ee OSS \u00a7 \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 4. \u4f7f\u7528 Impala \u8bbf\u95ee OSS\u3002 \u00a7 \u4e0b\u8f7d\u6d4b\u8bd5\u6570\u636e\u96c6 \u94fe\u63a5 \u4e0a\u4f20\u6570\u636e bash hadoop fs -put test_data oss://bucket.endpoint/dir \u521b\u5efa\u8868 sql CREATE EXTERNAL TABLE customer_demographics ( `cd_demo_sk` INT, `cd_gender` STRING, `cd_marital_status` STRING, `cd_education_status` STRING, `cd_purchase_estimate` INT, `cd_credit_rating` STRING, `cd_dep_count` INT, `cd_dep_employed_count` INT, `cd_dep_college_count` INT) STORED AS PARQUET LOCATION 'oss://bucket.endpoint/dir'; \u67e5\u8be2 OSS \u8868 sql select * from customer_demographics; 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofs/impala/jindosdk_on_impala/#impala-jindosdk-oss-hdfs-jindofs","text":"JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Impala \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofs/impala/jindosdk_on_impala/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofs/impala/jindosdk_on_impala/#1-impala-hadoop","text":"\u786e\u4fddImpala\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Impala \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"backup/jindofs/impala/jindosdk_on_impala/#2-impala-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06 sdk \u5305\u5b89\u88c5\u5230 Impala \u7684 classpath \u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $IMPALA_HOME/lib/","title":"2. \u5728\u6240\u6709 Impala \u8282\u70b9\u5b89\u88c5 JindoSDK"},{"location":"backup/jindofs/impala/jindosdk_on_impala/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofs/impala/jindosdk_on_impala/#impala-core-sitexml-jindosdk-oss","text":"\u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002","title":"\u914d\u7f6e IMPALA \u4f7f\u7528\u7684 core-site.xml \u914d\u7f6e JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/jindofs/impala/jindosdk_on_impala/#4-impala-oss","text":"\u4e0b\u8f7d\u6d4b\u8bd5\u6570\u636e\u96c6 \u94fe\u63a5 \u4e0a\u4f20\u6570\u636e bash hadoop fs -put test_data oss://bucket.endpoint/dir \u521b\u5efa\u8868 sql CREATE EXTERNAL TABLE customer_demographics ( `cd_demo_sk` INT, `cd_gender` STRING, `cd_marital_status` STRING, `cd_education_status` STRING, `cd_purchase_estimate` INT, `cd_credit_rating` STRING, `cd_dep_count` INT, `cd_dep_employed_count` INT, `cd_dep_college_count` INT) STORED AS PARQUET LOCATION 'oss://bucket.endpoint/dir'; \u67e5\u8be2 OSS \u8868 sql select * from customer_demographics;","title":"4. \u4f7f\u7528 Impala \u8bbf\u95ee OSS\u3002"},{"location":"backup/jindofs/impala/jindosdk_on_impala/#5","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_QA/","text":"1\u3001list \u6027\u80fd\u4f4e \u00a7 \u73b0\u8c61 \u00a7 \u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u53d1\u73b0 list \u6027\u80fd\u8f83\u6162\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Successfully list objects with prefix xxx/yyy/ in bucket xxx recursive 0 result 315 dur 100036.615031MS \u5176\u4e2d dur 100036.615031MS \u4ee3\u8868\u6b64\u6b21 list \u7528\u65f6\uff0c\u5355\u4f4d\u6beb\u79d2\uff0c\u5982\u4e0a\u8ff0\u4e00\u6b21 list \u8017\u65f6 100 \u79d2\uff0cOSS \u4e0a\u6587\u4ef6\u6b63\u5e38\u7684 list \u901f\u5ea6\u662f 1000 \u6587\u4ef6\u5728 1s \u4ee5\u4e0b\uff0c\u60a8\u53ef\u6839\u636e\u5f53\u524d\u76ee\u5f55\u4e0b\u6587\u4ef6\u6570\u91cf\u6765\u5224\u65ad\u8be5 list \u8017\u65f6\u662f\u5426\u5f02\u5e38\uff0c\u5982\u4e0a\u8ff0\u4fe1\u606f\u663e\u793a list 315 \u4e2a\u6587\u4ef6\u7684\u76ee\u5f55\u9700\u8981 100s \u663e\u7136\u662f\u4e0d\u592a\u6b63\u5e38\u7684\u3002 \u89e3\u51b3\u529e\u6cd5 \u00a7 \u589e\u52a0\u5ba2\u6237\u7aef\u5185\u5b58\uff0c\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u589e\u5927\u5ba2\u6237\u7aef\u5185\u5b58 export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx4096m\" 2. checksum \u62a5\u9519 \u00a7 \u73b0\u8c61 \u00a7 \u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Failed to get checksum store. \u89e3\u51b3\u529e\u6cd5 \u00a7 OSS-HDFS \u9ed8\u8ba4\u7684 checksum \u7b97\u6cd5\u662f COMPOSITE_CRC\uff0c\u5982\u679c HDFS \u914d\u7f6e\u7684 dfs.checksum.combine.mode \u4e3a MD5MD5CRC\uff0c\u5219\u9700\u8981\u53d8\u66f4 OSS-HDFS \u7684 fs.oss.checksum.combine.mode \u914d\u7f6e\u4e3a MD5MD5CRC\u3002 hadoop jar jindo-distcp-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.checksum.combine.mode=MD5MD5CRC","title":"jindo distcp QA"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_QA/#1list","text":"","title":"1\u3001list \u6027\u80fd\u4f4e"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_QA/#_1","text":"\u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u53d1\u73b0 list \u6027\u80fd\u8f83\u6162\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Successfully list objects with prefix xxx/yyy/ in bucket xxx recursive 0 result 315 dur 100036.615031MS \u5176\u4e2d dur 100036.615031MS \u4ee3\u8868\u6b64\u6b21 list \u7528\u65f6\uff0c\u5355\u4f4d\u6beb\u79d2\uff0c\u5982\u4e0a\u8ff0\u4e00\u6b21 list \u8017\u65f6 100 \u79d2\uff0cOSS \u4e0a\u6587\u4ef6\u6b63\u5e38\u7684 list \u901f\u5ea6\u662f 1000 \u6587\u4ef6\u5728 1s \u4ee5\u4e0b\uff0c\u60a8\u53ef\u6839\u636e\u5f53\u524d\u76ee\u5f55\u4e0b\u6587\u4ef6\u6570\u91cf\u6765\u5224\u65ad\u8be5 list \u8017\u65f6\u662f\u5426\u5f02\u5e38\uff0c\u5982\u4e0a\u8ff0\u4fe1\u606f\u663e\u793a list 315 \u4e2a\u6587\u4ef6\u7684\u76ee\u5f55\u9700\u8981 100s \u663e\u7136\u662f\u4e0d\u592a\u6b63\u5e38\u7684\u3002","title":"\u73b0\u8c61"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_QA/#_2","text":"\u589e\u52a0\u5ba2\u6237\u7aef\u5185\u5b58\uff0c\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u589e\u5927\u5ba2\u6237\u7aef\u5185\u5b58 export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx4096m\"","title":"\u89e3\u51b3\u529e\u6cd5"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_QA/#2-checksum","text":"","title":"2. checksum \u62a5\u9519"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_QA/#_3","text":"\u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Failed to get checksum store.","title":"\u73b0\u8c61"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_QA/#_4","text":"OSS-HDFS \u9ed8\u8ba4\u7684 checksum \u7b97\u6cd5\u662f COMPOSITE_CRC\uff0c\u5982\u679c HDFS \u914d\u7f6e\u7684 dfs.checksum.combine.mode \u4e3a MD5MD5CRC\uff0c\u5219\u9700\u8981\u53d8\u66f4 OSS-HDFS \u7684 fs.oss.checksum.combine.mode \u914d\u7f6e\u4e3a MD5MD5CRC\u3002 hadoop jar jindo-distcp-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.checksum.combine.mode=MD5MD5CRC","title":"\u89e3\u51b3\u529e\u6cd5"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/","text":"\u4f7f\u7528 JindoDistCp \u00a7 \u4ecb\u7ecd \u00a7 JindoDistCp\uff08\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u5de5\u5177\uff09\u662f\u7528\u4e8e\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u62f7\u8d1d\u6587\u4ef6\u7684\u5de5\u5177\u3002 \u5b83\u4f7f\u7528 MapReduce \u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3a MapReduce \u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002\u5168\u91cf\u652f\u6301 HDFS->OSS-HDFS \u670d\u52a1\uff0cHDFS->HDFS\uff0cOSS-HDFS \u670d\u52a1->HDFS\uff0cOSS-HDFS \u670d\u52a1->OSS-HDFS \u670d\u52a1\uff0c\u63d0\u4f9b\u591a\u79cd\u4e2a\u6027\u5316\u62f7\u8d1d\u53c2\u6570\u548c\u591a\u79cd\u62f7\u8d1d\u7b56\u7565\u3002\u91cd\u70b9\u4f18\u5316 HDFS \u5230 OSS-HDFS \u670d\u52a1\u7684\u6570\u636e\u62f7\u8d1d\uff0c\u901a\u8fc7\u5b9a\u5236\u5316CopyCommitter\uff0c\u5b9e\u73b0No-Rename\u62f7\u8d1d\uff0c\u5e76\u4fdd\u8bc1\u6570\u636e\u62f7\u8d1d\u843d\u5730\u7684\u4e00\u81f4\u6027\u3002\u529f\u80fd\u5168\u91cf\u5bf9\u9f50S3 DistCp\u548cHDFS DistCp\uff0c\u6027\u80fd\u8f83HDFS DistCp\u6709\u8f83\u5927\u63d0\u5347\uff0c\u76ee\u6807\u63d0\u4f9b\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177\u3002 \u60a8\u53ef\u4ee5\u4f7f\u7528 JindoDistCp \u8fdb\u884c HDFS \u548c OSS-HDFS \u670d\u52a1\u4e4b\u95f4\u7684\u6570\u636e\u8fc1\u79fb\uff0c\u76f8\u5bf9\u4e8e Hadoop \u793e\u533a DistCp \u5b9e\u73b0\uff0c\u60a8\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u4f7f\u7528\u6307\u5357 \u00a7 JindoDistCp \u63d0\u4f9b jar \u5305\u5f62\u5f0f\u4f7f\u7528\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 hadoop jar \u547d\u4ee4\u914d\u5408\u4e00\u7cfb\u5217\u53c2\u6570\u6765\u5b8c\u6210 distcp \u64cd\u4f5c\u3002 1\u3001\u4f7f\u7528--src\u548c--dest (\u5fc5\u9009) \u00a7 --src \u8868\u793a\u6307\u5b9a\u6e90\u6587\u4ef6\u7684\u8def\u5f84 --dest \u8868\u793a\u76ee\u6807\u6587\u4ef6\u7684\u8def\u5f84 \u5982\u4eceHDFS\u4e0a/opt/tmp\u76ee\u5f55\u62f7\u8d1d\u5230OSS-HDFS \u670d\u52a1 bucket\uff0c\u53ef\u4ee5\u6267\u884c\u5982\u4e0b\u8bed\u53e5 hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/tmp \u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9adest\u8def\u5f84\u6765\u786e\u5b9a\u62f7\u8d1d\u540e\u7684\u6587\u4ef6\u5c42\u6b21\uff0c\u5982\u60a8\u9700\u8981\u5c06/opt/tmp\u4e0b\u7684\u6587\u4ef6\u62f7\u8d1d\u5230yang-hhht\u8fd9\u4e2abucket\u4e0b\u7684tmp\u76ee\u5f55\u4e0b\uff0c\u5219\u53ef\u4ee5\u4f7f\u7528\u4e0a\u8ff0\u8bed\u53e5\u6765\u5b8c\u6210\u3002\u6b64\u5904\u548cHadoop\u7684distcp\u884c\u4e3a\u6709\u6240\u4e0d\u540c\uff0cJindoDistCp\u4f1a\u9ed8\u8ba4\u5c06src\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6587\u4ef6\u62f7\u8d1d\u5230\u60a8\u6307\u5b9adest\u8def\u5f84\u4e0b\uff0c\u5e76\u4e0d\u5305\u62ec\u5f53\u524d\u7684\u6839\u76ee\u5f55\u540d\u79f0\uff0c\u60a8\u53ef\u4ee5\u5728dest\u4e2d\u6307\u5b9a\u62f7\u8d1d\u8def\u5f84\u7684\u6839\u76ee\u5f55\uff0c\u5982\u679c\u4e0d\u5b58\u5728\u4f1a\u81ea\u52a8\u521b\u5efa\u3002 \u6ce8\u610f\uff1a\u5982\u9700\u62f7\u8d1d\u5355\u4e2a\u6587\u4ef6, dest\u9700\u6307\u5b9a\u4e3a\u76ee\u5f55\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /test.txt --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/tmp 2\u3001\u4f7f\u7528--bandWidth \u00a7 \u5728\u60a8\u7684distcp\u8fc7\u7a0b\u4e2d\uff0c\u60a8\u53ef\u4ee5\u6307\u5b9a\u672c\u6b21distcp\u4efb\u52a1\u6240\u7528\u7684\u5355\u673a\u5e26\u5bbd(\u4ee5MB\u4e3a\u5355\u4f4d)\uff0c\u907f\u514d\u5355\u673a\u5360\u7528\u8fc7\u5927\u5e26\u5bbd \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --bandWidth 6 3\u3001\u4f7f\u7528--codec \u00a7 \u539f\u59cb\u6587\u4ef6\u901a\u5e38\u4ee5\u672a\u538b\u7f29\u7684\u6587\u672c\u683c\u5f0f\u8fdb\u5165OSS\u6216\u8005HDFS\u3002\u65e0\u8bba\u662f\u5b58\u50a8\u6210\u672c\u8fd8\u662f\u5bf9\u8be5\u6570\u636e\u8fdb\u884c\u5206\u6790\uff0c\u6b64\u683c\u5f0f\u90fd\u4e0d\u751a\u7406\u60f3\u3002JindoDistCp \u53ef\u4ee5\u4f7f\u7528 --codec \u9009\u9879\u5e2e\u52a9\u60a8\u5728\u7ebf\u9ad8\u6548\u5730\u5b58\u50a8\u6570\u636e\u548c\u538b\u7f29\u6587\u4ef6 \u547d\u4ee4\u793a\u4f8b\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --codec gz --parallelism 20 JindoDistCp \u7684\u5f53\u524d\u7248\u672c\u652f\u6301\u7f16\u89e3\u7801\u5668 gzip\u3001gz\u3001lzo\u3001lzop \u548c snappy \u4ee5\u53ca\u5173\u952e\u5b57 none \u548c keep \uff08\u9ed8\u8ba4\u503c\uff09\u3002\u8fd9\u4e9b\u5173\u952e\u5b57\u542b\u4e49\u5982\u4e0b\uff1a \"none\" \u2013 \u4fdd\u5b58\u4e3a\u672a\u538b\u7f29\u7684\u6587\u4ef6\u3002\u5982\u679c\u6587\u4ef6\u5df2\u538b\u7f29\uff0c\u5219 JindoDistCp \u4f1a\u5c06\u5176\u89e3\u538b\u7f29\u3002 \"keep\" \u2013 \u4e0d\u66f4\u6539\u6587\u4ef6\u538b\u7f29\u5f62\u6001\uff0c\u6309\u539f\u6837\u590d\u5236\u3002 \u6211\u4eec\u68c0\u67e5\u4e00\u4e0b\u76ee\u6807\u6587\u4ef6\u5939\u4e2d\u7684\u6587\u4ef6\uff0c\u8fd9\u4e9b\u6587\u4ef6\u73b0\u5728\u5df2\u7ecf\u7528 gz \u7f16\u89e3\u7801\u5668\u538b\u7f29\u4e86\uff1a [root@emr-header-1 opt]# hdfs dfs -ls oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03 Found 6 items -rw-rw-rw- 1 938 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/000151.sst.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/1.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/2.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/OPTIONS-000109.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/emp01.txt.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/emp06.txt.gz \u5982\u60a8\u5728\u5f00\u6e90Hadoop\u96c6\u7fa4\u73af\u5883\u4e2d\u4f7f\u7528lzo\u538b\u7f29\u529f\u80fd\uff0c\u5219\u60a8\u9700\u8981\u53bb\u5b89\u88c5gplcompression\u7684native\u5e93\u548chadoop-lzo\u5305\uff0c\u5982\u60a8\u7f3a\u5c11\u76f8\u5173\u73af\u5883\uff0c\u5efa\u8bae\u4f7f\u7528\u5176\u4ed6\u538b\u7f29\u65b9\u5f0f\u8fdb\u884c\u538b\u7f29\u3002 4\u3001\u4f7f\u7528--filters \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--filters\u6765\u6307\u5b9a\u6709\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --filters filter.txt \u5047\u8bbe filter.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u4e0d\u4f1a\u5c06\u8def\u5f84\u4e2d\u5e26\u6709 test \u7684\u6587\u4ef6\u62f7\u8d1d\u5230OSS\u3002 5\u3001\u4f7f\u7528--srcPrefixesFile \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--srcPrefixesFile\u6765\u6307\u5b9a\u6709\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --srcPrefixesFile prefixes.txt \u5047\u8bbe prefixes.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u53ea\u6709\u8def\u5f84\u4e2d\u5e26\u6709 test \u7684\u6587\u4ef6\u624d\u4f1a\u62f7\u8d1d\u5230OSS\u3002 6\u3001\u4f7f\u7528--parallelism \u00a7 parallelism \u53c2\u6570\u7528\u6765\u6307\u5b9aMR\u4efb\u52a1\u91cc\u7684 mapreduce.job.maps \u53c2\u6570\uff0c\u60a8\u53ef\u4ee5\u6839\u636e\u96c6\u7fa4\u7684\u8d44\u6e90\u60c5\u51b5\u81ea\u5b9a\u4e49 parallelism \u7684\u5927\u5c0f\u6765\u63a7\u5236 distcp \u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\u3002 \u5982\u4eceHDFS\u4e0a/opt/tmp\u76ee\u5f55\u62f7\u8d1d\u5230OSS bucket\uff0c\u53ef\u4ee5\u6267\u884c hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/tmp --parallelism 20 7\u3001\u4f7f\u7528--taskBatch \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--taskBatch\u6765\u6307\u5b9a\u6bcf\u4e2adistcp task\u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --taskBatch 10 8\u3001\u4f7f\u7528--tmp \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7tmp\u53c2\u6570\u6307\u5b9a\u4e00\u4e2a\u4e34\u65f6\u76ee\u5f55\u5728hdfs\u4e0a\u4f20\u5b58\u653e\u4e34\u65f6\u6570\u636e\uff0c\u9ed8\u8ba4\u503c\u4e3a/tmp\uff0c\u5373hdfs:///tmp/\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --parallelism 20 --tmp /tmp 9\u3001\u4f7f\u7528 OSS-HDFS \u670d\u52a1 AK \u00a7 \u5728 EMR \u5916\u6216\u8005\u514d\u5bc6\u670d\u52a1\u51fa\u73b0\u95ee\u9898\u7684\u60c5\u51b5\u4e0b\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9aAK\u6765\u83b7\u5f97\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u6743\u9650\u3002\u60a8\u53ef\u4ee5\u5728\u547d\u4ee4\u4e2d\u4f7f\u7528 --hadoopConf \u9009\u9879\u6765\u6307\u5b9aAK\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism 20 \u60a8\u4e5f\u53ef\u4ee5\u5c06 OSS-HDFS \u670d\u52a1\u7684 ak\u3001secret \u9884\u5148\u914d\u7f6e\u5728 hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u907f\u514d\u6bcf\u6b21\u4f7f\u7528\u65f6\u4e34\u65f6\u586b\u5199ak\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> 10\u3001\u4f7f\u7528--disableChecksum \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--disableChecksum\u5173\u95ed\u68c0\u67e5\u6587\u4ef6checksum\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --disableChecksum 11\u3001\u4f7f\u7528--deleteOnSuccess \u00a7 \u6709\u65f6\u5019\u6211\u4eec\u662f\u60f3\u8981\u79fb\u52a8\u6570\u636e\u800c\u4e0d\u662f\u590d\u5236\u6570\u636e\uff0c\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 --deleteOnSuccess \u9009\u9879\u3002\u6b64\u9009\u9879\u7c7b\u4f3c\u4e8emv\u64cd\u4f5c\uff0c\u9996\u5148\u590d\u5236\u6587\u4ef6\uff0c\u7136\u540e\u4ece\u6e90\u4f4d\u7f6e\u5220\u9664\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --deleteOnSuccess --parallelism 20 12\u3001\u4f7f\u7528--enableTransaction \u00a7 JindoDistCp\u9ed8\u8ba4\u4f7f\u7528task\u7ea7\u522b\u5b8c\u6574\u6027\uff0c\u5982\u60a8\u9700\u8981\u4fdd\u8bc1Job\u7ea7\u522b\u7684\u5b8c\u6574\u6027\u4ee5\u53ca\u4fdd\u8bc1Job\u4e4b\u95f4\u7684\u4e8b\u52a1\u652f\u6301\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528--enableTransaction\u53c2\u6570\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --enableTransaction --parallelism 20 13\u3001\u4f7f\u7528--ignore \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7ignore\u53c2\u6570\uff0c\u5ffd\u7565\u6570\u636e\u8fc1\u79fb\u671f\u95f4\u53d1\u751f\u7684\u5f02\u5e38\uff0c\u76f8\u5173\u62a5\u9519\u4e0d\u4f1a\u4e2d\u65ad\u4efb\u52a1\uff0c\u5e76\u6700\u7ec8\u4ee5DistCp Counter\u7684\u5f62\u5f0f\u900f\u51fa\u3002\uff08\u5982\u679c\u5f00\u542fCMS\u7684\u8bdd\uff0c\u4e5f\u4f1a\u4ee5\u6307\u5b9a\u65b9\u5f0f\u8fdb\u884c\u901a\u77e5\u3002\uff09 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --ignore --parallelism 20 14\u3001\u4f7f\u7528--diff \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5b8c\u6210\u540e\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9asrc\u548cdest\u6765\u67e5\u770b\u5f53\u524ddistcp\u7684\u6587\u4ef6\u5dee\u5f02, \u5982\u679csrc\u7684\u6587\u4ef6\u672a\u80fd\u540c\u6b65\u5230dest\u4e0a\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\u3002\u5982\u60a8\u7684distcp\u4efb\u52a1\u5305\u542b\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u5219--diff\u4e0d\u80fd\u663e\u793a\u6b63\u786e\u7684\u6587\u4ef6\u5dee\u5f02\uff0c\u56e0\u4e3a\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u4f1a\u6539\u53d8\u6587\u4ef6\u7684\u5927\u5c0f\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --diff \u5982\u679c\u6587\u4ef6\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter DIFF_FILES=1 \u5982\u679c\u60a8\u7684--dest\u4e3aHDFS\u8def\u5f84\uff0c\u73b0\u652f\u6301 /path \uff0c hdfs://hostname:ip/path \uff0c hdfs://headerIp:ip/path \u7684\u5199\u6cd5\uff0c\u6682\u4e0d\u652f\u6301 hdfs:///path \uff0c hdfs:/path \u548c\u5176\u4ed6\u81ea\u5b9a\u4e49\u5199\u6cd5\u3002 \u5982\u679c\u60f3\u67e5\u770b\u6587\u4ef6\u5143\u6570\u636e\u7684\u5dee\u5f02\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 --diff --preserveMeta \u547d\u4ee4\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --diff --preserveMeta \u5982\u679c\u6587\u4ef6\u5143\u6570\u636e\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5143\u6570\u636e\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter META_FILE_DIFF=1 15\u3001\u4f7f\u7528--update \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7update\u53c2\u6570\u4f7f\u7528\u589e\u91cf\u540c\u6b65\u529f\u80fd\uff0c\u8df3\u8fc7\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u548c\u76ee\u5f55\uff0c\u76f4\u63a5\u5c06src\u4e2d\u65b0\u589e\u6216\u53d1\u751f\u6539\u53d8\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u540c\u6b65\u5230dest\u4e0a\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --update --parallelism 20 16\u3001\u4f7f\u7528--preserveMeta \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/tmp --preserveMeta 17\u3001\u4f7f\u7528--jobBatch \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--jobBatch\u6765\u6307\u5b9a\u6bcf\u4e2adistcp job\u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10000\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --jobBatch 50000","title":"\u4f7f\u7528 JindoDistCp"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#jindodistcp","text":"","title":"\u4f7f\u7528 JindoDistCp"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#_1","text":"JindoDistCp\uff08\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u5de5\u5177\uff09\u662f\u7528\u4e8e\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u62f7\u8d1d\u6587\u4ef6\u7684\u5de5\u5177\u3002 \u5b83\u4f7f\u7528 MapReduce \u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3a MapReduce \u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002\u5168\u91cf\u652f\u6301 HDFS->OSS-HDFS \u670d\u52a1\uff0cHDFS->HDFS\uff0cOSS-HDFS \u670d\u52a1->HDFS\uff0cOSS-HDFS \u670d\u52a1->OSS-HDFS \u670d\u52a1\uff0c\u63d0\u4f9b\u591a\u79cd\u4e2a\u6027\u5316\u62f7\u8d1d\u53c2\u6570\u548c\u591a\u79cd\u62f7\u8d1d\u7b56\u7565\u3002\u91cd\u70b9\u4f18\u5316 HDFS \u5230 OSS-HDFS \u670d\u52a1\u7684\u6570\u636e\u62f7\u8d1d\uff0c\u901a\u8fc7\u5b9a\u5236\u5316CopyCommitter\uff0c\u5b9e\u73b0No-Rename\u62f7\u8d1d\uff0c\u5e76\u4fdd\u8bc1\u6570\u636e\u62f7\u8d1d\u843d\u5730\u7684\u4e00\u81f4\u6027\u3002\u529f\u80fd\u5168\u91cf\u5bf9\u9f50S3 DistCp\u548cHDFS DistCp\uff0c\u6027\u80fd\u8f83HDFS DistCp\u6709\u8f83\u5927\u63d0\u5347\uff0c\u76ee\u6807\u63d0\u4f9b\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177\u3002 \u60a8\u53ef\u4ee5\u4f7f\u7528 JindoDistCp \u8fdb\u884c HDFS \u548c OSS-HDFS \u670d\u52a1\u4e4b\u95f4\u7684\u6570\u636e\u8fc1\u79fb\uff0c\u76f8\u5bf9\u4e8e Hadoop \u793e\u533a DistCp \u5b9e\u73b0\uff0c\u60a8\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"\u4ecb\u7ecd"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#_2","text":"JindoDistCp \u63d0\u4f9b jar \u5305\u5f62\u5f0f\u4f7f\u7528\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 hadoop jar \u547d\u4ee4\u914d\u5408\u4e00\u7cfb\u5217\u53c2\u6570\u6765\u5b8c\u6210 distcp \u64cd\u4f5c\u3002","title":"\u4f7f\u7528\u6307\u5357"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#1-src-dest","text":"--src \u8868\u793a\u6307\u5b9a\u6e90\u6587\u4ef6\u7684\u8def\u5f84 --dest \u8868\u793a\u76ee\u6807\u6587\u4ef6\u7684\u8def\u5f84 \u5982\u4eceHDFS\u4e0a/opt/tmp\u76ee\u5f55\u62f7\u8d1d\u5230OSS-HDFS \u670d\u52a1 bucket\uff0c\u53ef\u4ee5\u6267\u884c\u5982\u4e0b\u8bed\u53e5 hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/tmp \u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9adest\u8def\u5f84\u6765\u786e\u5b9a\u62f7\u8d1d\u540e\u7684\u6587\u4ef6\u5c42\u6b21\uff0c\u5982\u60a8\u9700\u8981\u5c06/opt/tmp\u4e0b\u7684\u6587\u4ef6\u62f7\u8d1d\u5230yang-hhht\u8fd9\u4e2abucket\u4e0b\u7684tmp\u76ee\u5f55\u4e0b\uff0c\u5219\u53ef\u4ee5\u4f7f\u7528\u4e0a\u8ff0\u8bed\u53e5\u6765\u5b8c\u6210\u3002\u6b64\u5904\u548cHadoop\u7684distcp\u884c\u4e3a\u6709\u6240\u4e0d\u540c\uff0cJindoDistCp\u4f1a\u9ed8\u8ba4\u5c06src\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6587\u4ef6\u62f7\u8d1d\u5230\u60a8\u6307\u5b9adest\u8def\u5f84\u4e0b\uff0c\u5e76\u4e0d\u5305\u62ec\u5f53\u524d\u7684\u6839\u76ee\u5f55\u540d\u79f0\uff0c\u60a8\u53ef\u4ee5\u5728dest\u4e2d\u6307\u5b9a\u62f7\u8d1d\u8def\u5f84\u7684\u6839\u76ee\u5f55\uff0c\u5982\u679c\u4e0d\u5b58\u5728\u4f1a\u81ea\u52a8\u521b\u5efa\u3002 \u6ce8\u610f\uff1a\u5982\u9700\u62f7\u8d1d\u5355\u4e2a\u6587\u4ef6, dest\u9700\u6307\u5b9a\u4e3a\u76ee\u5f55\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /test.txt --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/tmp","title":"1\u3001\u4f7f\u7528--src\u548c--dest (\u5fc5\u9009)"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#2-bandwidth","text":"\u5728\u60a8\u7684distcp\u8fc7\u7a0b\u4e2d\uff0c\u60a8\u53ef\u4ee5\u6307\u5b9a\u672c\u6b21distcp\u4efb\u52a1\u6240\u7528\u7684\u5355\u673a\u5e26\u5bbd(\u4ee5MB\u4e3a\u5355\u4f4d)\uff0c\u907f\u514d\u5355\u673a\u5360\u7528\u8fc7\u5927\u5e26\u5bbd \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --bandWidth 6","title":"2\u3001\u4f7f\u7528--bandWidth"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#3-codec","text":"\u539f\u59cb\u6587\u4ef6\u901a\u5e38\u4ee5\u672a\u538b\u7f29\u7684\u6587\u672c\u683c\u5f0f\u8fdb\u5165OSS\u6216\u8005HDFS\u3002\u65e0\u8bba\u662f\u5b58\u50a8\u6210\u672c\u8fd8\u662f\u5bf9\u8be5\u6570\u636e\u8fdb\u884c\u5206\u6790\uff0c\u6b64\u683c\u5f0f\u90fd\u4e0d\u751a\u7406\u60f3\u3002JindoDistCp \u53ef\u4ee5\u4f7f\u7528 --codec \u9009\u9879\u5e2e\u52a9\u60a8\u5728\u7ebf\u9ad8\u6548\u5730\u5b58\u50a8\u6570\u636e\u548c\u538b\u7f29\u6587\u4ef6 \u547d\u4ee4\u793a\u4f8b\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --codec gz --parallelism 20 JindoDistCp \u7684\u5f53\u524d\u7248\u672c\u652f\u6301\u7f16\u89e3\u7801\u5668 gzip\u3001gz\u3001lzo\u3001lzop \u548c snappy \u4ee5\u53ca\u5173\u952e\u5b57 none \u548c keep \uff08\u9ed8\u8ba4\u503c\uff09\u3002\u8fd9\u4e9b\u5173\u952e\u5b57\u542b\u4e49\u5982\u4e0b\uff1a \"none\" \u2013 \u4fdd\u5b58\u4e3a\u672a\u538b\u7f29\u7684\u6587\u4ef6\u3002\u5982\u679c\u6587\u4ef6\u5df2\u538b\u7f29\uff0c\u5219 JindoDistCp \u4f1a\u5c06\u5176\u89e3\u538b\u7f29\u3002 \"keep\" \u2013 \u4e0d\u66f4\u6539\u6587\u4ef6\u538b\u7f29\u5f62\u6001\uff0c\u6309\u539f\u6837\u590d\u5236\u3002 \u6211\u4eec\u68c0\u67e5\u4e00\u4e0b\u76ee\u6807\u6587\u4ef6\u5939\u4e2d\u7684\u6587\u4ef6\uff0c\u8fd9\u4e9b\u6587\u4ef6\u73b0\u5728\u5df2\u7ecf\u7528 gz \u7f16\u89e3\u7801\u5668\u538b\u7f29\u4e86\uff1a [root@emr-header-1 opt]# hdfs dfs -ls oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03 Found 6 items -rw-rw-rw- 1 938 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/000151.sst.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/1.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/2.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/OPTIONS-000109.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/emp01.txt.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table/2017-02-01/03/emp06.txt.gz \u5982\u60a8\u5728\u5f00\u6e90Hadoop\u96c6\u7fa4\u73af\u5883\u4e2d\u4f7f\u7528lzo\u538b\u7f29\u529f\u80fd\uff0c\u5219\u60a8\u9700\u8981\u53bb\u5b89\u88c5gplcompression\u7684native\u5e93\u548chadoop-lzo\u5305\uff0c\u5982\u60a8\u7f3a\u5c11\u76f8\u5173\u73af\u5883\uff0c\u5efa\u8bae\u4f7f\u7528\u5176\u4ed6\u538b\u7f29\u65b9\u5f0f\u8fdb\u884c\u538b\u7f29\u3002","title":"3\u3001\u4f7f\u7528--codec"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#4-filters","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--filters\u6765\u6307\u5b9a\u6709\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --filters filter.txt \u5047\u8bbe filter.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u4e0d\u4f1a\u5c06\u8def\u5f84\u4e2d\u5e26\u6709 test \u7684\u6587\u4ef6\u62f7\u8d1d\u5230OSS\u3002","title":"4\u3001\u4f7f\u7528--filters"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#5-srcprefixesfile","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--srcPrefixesFile\u6765\u6307\u5b9a\u6709\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --srcPrefixesFile prefixes.txt \u5047\u8bbe prefixes.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u53ea\u6709\u8def\u5f84\u4e2d\u5e26\u6709 test \u7684\u6587\u4ef6\u624d\u4f1a\u62f7\u8d1d\u5230OSS\u3002","title":"5\u3001\u4f7f\u7528--srcPrefixesFile"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#6-parallelism","text":"parallelism \u53c2\u6570\u7528\u6765\u6307\u5b9aMR\u4efb\u52a1\u91cc\u7684 mapreduce.job.maps \u53c2\u6570\uff0c\u60a8\u53ef\u4ee5\u6839\u636e\u96c6\u7fa4\u7684\u8d44\u6e90\u60c5\u51b5\u81ea\u5b9a\u4e49 parallelism \u7684\u5927\u5c0f\u6765\u63a7\u5236 distcp \u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\u3002 \u5982\u4eceHDFS\u4e0a/opt/tmp\u76ee\u5f55\u62f7\u8d1d\u5230OSS bucket\uff0c\u53ef\u4ee5\u6267\u884c hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/tmp --parallelism 20","title":"6\u3001\u4f7f\u7528--parallelism"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#7-taskbatch","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--taskBatch\u6765\u6307\u5b9a\u6bcf\u4e2adistcp task\u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --taskBatch 10","title":"7\u3001\u4f7f\u7528--taskBatch"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#8-tmp","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7tmp\u53c2\u6570\u6307\u5b9a\u4e00\u4e2a\u4e34\u65f6\u76ee\u5f55\u5728hdfs\u4e0a\u4f20\u5b58\u653e\u4e34\u65f6\u6570\u636e\uff0c\u9ed8\u8ba4\u503c\u4e3a/tmp\uff0c\u5373hdfs:///tmp/\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --parallelism 20 --tmp /tmp","title":"8\u3001\u4f7f\u7528--tmp"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#9-oss-hdfs-ak","text":"\u5728 EMR \u5916\u6216\u8005\u514d\u5bc6\u670d\u52a1\u51fa\u73b0\u95ee\u9898\u7684\u60c5\u51b5\u4e0b\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9aAK\u6765\u83b7\u5f97\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u6743\u9650\u3002\u60a8\u53ef\u4ee5\u5728\u547d\u4ee4\u4e2d\u4f7f\u7528 --hadoopConf \u9009\u9879\u6765\u6307\u5b9aAK\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism 20 \u60a8\u4e5f\u53ef\u4ee5\u5c06 OSS-HDFS \u670d\u52a1\u7684 ak\u3001secret \u9884\u5148\u914d\u7f6e\u5728 hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u907f\u514d\u6bcf\u6b21\u4f7f\u7528\u65f6\u4e34\u65f6\u586b\u5199ak\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration>","title":"9\u3001\u4f7f\u7528 OSS-HDFS \u670d\u52a1 AK"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#10-disablechecksum","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--disableChecksum\u5173\u95ed\u68c0\u67e5\u6587\u4ef6checksum\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --disableChecksum","title":"10\u3001\u4f7f\u7528--disableChecksum"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#11-deleteonsuccess","text":"\u6709\u65f6\u5019\u6211\u4eec\u662f\u60f3\u8981\u79fb\u52a8\u6570\u636e\u800c\u4e0d\u662f\u590d\u5236\u6570\u636e\uff0c\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 --deleteOnSuccess \u9009\u9879\u3002\u6b64\u9009\u9879\u7c7b\u4f3c\u4e8emv\u64cd\u4f5c\uff0c\u9996\u5148\u590d\u5236\u6587\u4ef6\uff0c\u7136\u540e\u4ece\u6e90\u4f4d\u7f6e\u5220\u9664\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --deleteOnSuccess --parallelism 20","title":"11\u3001\u4f7f\u7528--deleteOnSuccess"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#12-enabletransaction","text":"JindoDistCp\u9ed8\u8ba4\u4f7f\u7528task\u7ea7\u522b\u5b8c\u6574\u6027\uff0c\u5982\u60a8\u9700\u8981\u4fdd\u8bc1Job\u7ea7\u522b\u7684\u5b8c\u6574\u6027\u4ee5\u53ca\u4fdd\u8bc1Job\u4e4b\u95f4\u7684\u4e8b\u52a1\u652f\u6301\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528--enableTransaction\u53c2\u6570\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --enableTransaction --parallelism 20","title":"12\u3001\u4f7f\u7528--enableTransaction"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#13-ignore","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7ignore\u53c2\u6570\uff0c\u5ffd\u7565\u6570\u636e\u8fc1\u79fb\u671f\u95f4\u53d1\u751f\u7684\u5f02\u5e38\uff0c\u76f8\u5173\u62a5\u9519\u4e0d\u4f1a\u4e2d\u65ad\u4efb\u52a1\uff0c\u5e76\u6700\u7ec8\u4ee5DistCp Counter\u7684\u5f62\u5f0f\u900f\u51fa\u3002\uff08\u5982\u679c\u5f00\u542fCMS\u7684\u8bdd\uff0c\u4e5f\u4f1a\u4ee5\u6307\u5b9a\u65b9\u5f0f\u8fdb\u884c\u901a\u77e5\u3002\uff09 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --ignore --parallelism 20","title":"13\u3001\u4f7f\u7528--ignore"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#14-diff","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5b8c\u6210\u540e\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9asrc\u548cdest\u6765\u67e5\u770b\u5f53\u524ddistcp\u7684\u6587\u4ef6\u5dee\u5f02, \u5982\u679csrc\u7684\u6587\u4ef6\u672a\u80fd\u540c\u6b65\u5230dest\u4e0a\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\u3002\u5982\u60a8\u7684distcp\u4efb\u52a1\u5305\u542b\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u5219--diff\u4e0d\u80fd\u663e\u793a\u6b63\u786e\u7684\u6587\u4ef6\u5dee\u5f02\uff0c\u56e0\u4e3a\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u4f1a\u6539\u53d8\u6587\u4ef6\u7684\u5927\u5c0f\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --diff \u5982\u679c\u6587\u4ef6\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter DIFF_FILES=1 \u5982\u679c\u60a8\u7684--dest\u4e3aHDFS\u8def\u5f84\uff0c\u73b0\u652f\u6301 /path \uff0c hdfs://hostname:ip/path \uff0c hdfs://headerIp:ip/path \u7684\u5199\u6cd5\uff0c\u6682\u4e0d\u652f\u6301 hdfs:///path \uff0c hdfs:/path \u548c\u5176\u4ed6\u81ea\u5b9a\u4e49\u5199\u6cd5\u3002 \u5982\u679c\u60f3\u67e5\u770b\u6587\u4ef6\u5143\u6570\u636e\u7684\u5dee\u5f02\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 --diff --preserveMeta \u547d\u4ee4\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --diff --preserveMeta \u5982\u679c\u6587\u4ef6\u5143\u6570\u636e\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5143\u6570\u636e\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter META_FILE_DIFF=1","title":"14\u3001\u4f7f\u7528--diff"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#15-update","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7update\u53c2\u6570\u4f7f\u7528\u589e\u91cf\u540c\u6b65\u529f\u80fd\uff0c\u8df3\u8fc7\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u548c\u76ee\u5f55\uff0c\u76f4\u63a5\u5c06src\u4e2d\u65b0\u589e\u6216\u53d1\u751f\u6539\u53d8\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u540c\u6b65\u5230dest\u4e0a\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --update --parallelism 20","title":"15\u3001\u4f7f\u7528--update"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#16-preservemeta","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/tmp --preserveMeta","title":"16\u3001\u4f7f\u7528--preserveMeta"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to/#17-jobbatch","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--jobBatch\u6765\u6307\u5b9a\u6bcf\u4e2adistcp job\u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10000\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --jobBatch 50000","title":"17\u3001\u4f7f\u7528--jobBatch"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_api/","text":"JindoDistCp API \u4f7f\u7528\u8bf4\u660e \u00a7 \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 \u5b89\u88c5\u4f9d\u8d56\u5305 \u00a7 \u76ee\u524d JindoSDK/JindoDistCP \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u4ee54.6.12\u7248\u672c\u4e3a\u4f8b\uff0c\u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoDistCP \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-distcp --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop with provided scope, your runtime should install hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> </dependencies> </project> \u793a\u4f8b\u4ee3\u7801 \u00a7 \u4ee5\u62f7\u8d1d HDFS \u6587\u4ef6\u5939\u5230\u963f\u91cc\u4e91 OSS \u4e3a\u4f8b\u3002 import com.aliyun.jindodata.distcp.util.DistCpCounter; import com.aliyun.jindodata.distjob.request.DistJobResponse; import org.apache.hadoop.conf.Configuration; import java.io.IOException; public class DistcpExample { static public void main(String[] args) throws IOException { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. conf.set(\"fs.oss.credentials.provider\", \"com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider\"); conf.set(\"fs.oss.accessKeyId\", \"XXX\"); conf.set(\"fs.oss.accessKeySecret\", \"XXX\"); DistCpRequest request = new DistCpRequest(); request.setSrcDir(\"hdfs:///user/root/random-data\"); request.setDestDir(\"oss://dls-bucket.cn-xxx.oss-dls.aliyuncs.com/dest\"); request.setWorkTempDir(\"hdfs:///user/root/tmp\"); request.setConf(conf); // Jindo DistCp Service DistCpService service = new DistCpService(); service.buildAllJobs(request); service.runAllJobs(); DistJobResponse response = service.collectAllJobResult(); printResponse(response); } static public void printResponse(DistJobResponse response) { System.out.println(\"ErrorCode: \" + response.getErrorCode()); System.out.println(\"ErrorMsg: \" + response.getErrorMsg()); System.out.println(\"FileCopied: \" + response.getCount(DistCpCounter.FILES_COPIED.getName())); System.out.println(\"BytesCopied: \" + response.getCount(DistCpCounter.BYTES_COPIED.getName())); } } \u8bf7\u6c42\u53c2\u6570 \u00a7 \u76f8\u5173\u65b9\u6cd5\u7684\u5177\u4f53\u63cf\u8ff0\u53ef\u53c2\u8003 \u300a\u4f7f\u7528Jindo DistCp\u300b \u65b9\u6cd5 \u53c2\u6570\u7c7b\u578b \u8bf4\u660e \u9ed8\u8ba4\u503c \u517c\u5bb9 void setSrc(String src) \u5fc5\u9009 \u8bbe\u7f6e\u6e90\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setDest(String dest) \u5fc5\u9009 \u8bbe\u7f6e\u76ee\u6807\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setConf(Configuration conf) \u5fc5\u9009 \u8bbe\u7f6e Configuration - 4.3.0+ void setMode(DistCpMode mode) \u53ef\u9009 \u8bbe\u7f6eDistCp\u6a21\u5f0f\uff0c\u652f\u6301\u7684\u6a21\u5f0f\u6709 COPY\uff08\u62f7\u8d1d\uff09 DIFF\uff08\u5dee\u5f02\u6bd4\u8f83\uff09 UPADTE\uff08\u589e\u91cf\u62f7\u8d1d\uff09 DistCpMode.COPY 4.3.0+ void setNumWorkers(int numWorkers) \u53ef\u9009 \u8bbe\u7f6eDistCp\u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\uff0c\u5bf9\u5e94MR\u4efb\u52a1\u4e2d\u7684 mapreduce.job.maps 10 4.3.0+ void setBandWidthLimit(int bandWidthLimit) \u53ef\u9009 \u8bbe\u7f6e\u5355\u4e2a\u8282\u70b9\u7684\u5e26\u5bbd\u9650\u5236\uff0c\u5355\u4f4dM -1 4.3.0+ void setWorkTempDir(String workTempDir) \u53ef\u9009 \u8bbe\u7f6e\u4e34\u65f6\u76ee\u5f55 /tmp 4.3.0+ void setOutputCodec(String outputCodec) \u53ef\u9009 \u8bbe\u7f6e\u538b\u7f29\u7c7b\u578b\uff0c\u652f\u6301\u7f16\u89e3\u7801\u5668\u6709 gzip\u3001gz\u3001lzo\u3001lzop\u3001lzop\u3001snappy keep\uff08\u4e0d\u66f4\u6539\u538b\u7f29\u7c7b\u578b\uff09 4.3.0+ void setEnableCMS(boolean enableCMS) \u53ef\u9009 \u662f\u5426\u5f00\u542f\u76d1\u63a7\u544a\u8b66 false 4.3.0+ void setDisableChecksum(boolean disableChecksum) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5173\u95edChecksum\u68c0\u67e5 false 4.3.0+ void setDeleteOnSuccess(boolean deleteOnSuccess) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5220\u9664\u6e90\u6587\u4ef6\uff0c\u7528\u4e8e\u79fb\u52a8\u6570\u636e false 4.3.0+ void setIncludeMatcherFile(String includeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setExcludeMatcherFile(String excludeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setIgnoreFailures(boolean ignoreFailures) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5ffd\u7565\u62f7\u8d1d\u4efb\u52a1\u4e2d\u629b\u51fa\u7684\u5f02\u5e38\uff0c\u907f\u514d\u4e2d\u65ad\u4efb\u52a1 False 4.3.0+ void setStorageClass(String storageClass) \u53ef\u9009 \u8bbe\u7f6e\u76ee\u6807\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301Standard\u3001IA\u3001Archive\u3001ColdArchive Standard 4.3.0+ void setEnableTransaction(boolean enableTransaction) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4e8b\u52a1\uff0c\u4ee5\u4fdd\u8bc1Job\u7ea7\u522b\u7684\u539f\u5b50\u6027 false 4.3.0+ void setEnablePreserveMetaStatus(boolean enablePreserveMetaStatus) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f false 4.4.0+ \u54cd\u5e94\u53c2\u6570 \u00a7 \u65b9\u6cd5 \u8bf4\u660e \u9ed8\u8ba4\u503c int getErrorCode() \u83b7\u53d6\u9519\u8bef\u7801 0 String getErrorMsg() \u83b7\u53d6\u9519\u8bef\u539f\u56e0 - long getStartTime() \u83b7\u53d6\u4efb\u52a1\u7684\u5f00\u59cb\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - long getEndTime() \u83b7\u53d6\u4efb\u52a1\u7684\u7ed3\u675f\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - getCount(String key) \u83b7\u53d6\u8ba1\u6570\u5668\u503c\uff0ckey\u4e3a\u8ba1\u6570\u5668\u540d\u79f0\uff0c - \u8ba1\u6570\u5668\u8bf4\u660e \u00a7 \u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf BYTES_EXPECTED \u9884\u671f\u7684copy\u5b57\u8282\u6570 FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 BYTES_COPIED copy\u6210\u529f\u7684\u5b57\u8282\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"JindoDistCp API \u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_api/#jindodistcp-api","text":"","title":"JindoDistCp API \u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_api/#jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"\u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_api/#_1","text":"\u76ee\u524d JindoSDK/JindoDistCP \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u4ee54.6.12\u7248\u672c\u4e3a\u4f8b\uff0c\u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoDistCP \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-distcp --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop with provided scope, your runtime should install hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> </dependencies> </project>","title":"\u5b89\u88c5\u4f9d\u8d56\u5305"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_api/#_2","text":"\u4ee5\u62f7\u8d1d HDFS \u6587\u4ef6\u5939\u5230\u963f\u91cc\u4e91 OSS \u4e3a\u4f8b\u3002 import com.aliyun.jindodata.distcp.util.DistCpCounter; import com.aliyun.jindodata.distjob.request.DistJobResponse; import org.apache.hadoop.conf.Configuration; import java.io.IOException; public class DistcpExample { static public void main(String[] args) throws IOException { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. conf.set(\"fs.oss.credentials.provider\", \"com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider\"); conf.set(\"fs.oss.accessKeyId\", \"XXX\"); conf.set(\"fs.oss.accessKeySecret\", \"XXX\"); DistCpRequest request = new DistCpRequest(); request.setSrcDir(\"hdfs:///user/root/random-data\"); request.setDestDir(\"oss://dls-bucket.cn-xxx.oss-dls.aliyuncs.com/dest\"); request.setWorkTempDir(\"hdfs:///user/root/tmp\"); request.setConf(conf); // Jindo DistCp Service DistCpService service = new DistCpService(); service.buildAllJobs(request); service.runAllJobs(); DistJobResponse response = service.collectAllJobResult(); printResponse(response); } static public void printResponse(DistJobResponse response) { System.out.println(\"ErrorCode: \" + response.getErrorCode()); System.out.println(\"ErrorMsg: \" + response.getErrorMsg()); System.out.println(\"FileCopied: \" + response.getCount(DistCpCounter.FILES_COPIED.getName())); System.out.println(\"BytesCopied: \" + response.getCount(DistCpCounter.BYTES_COPIED.getName())); } }","title":"\u793a\u4f8b\u4ee3\u7801"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_api/#_3","text":"\u76f8\u5173\u65b9\u6cd5\u7684\u5177\u4f53\u63cf\u8ff0\u53ef\u53c2\u8003 \u300a\u4f7f\u7528Jindo DistCp\u300b \u65b9\u6cd5 \u53c2\u6570\u7c7b\u578b \u8bf4\u660e \u9ed8\u8ba4\u503c \u517c\u5bb9 void setSrc(String src) \u5fc5\u9009 \u8bbe\u7f6e\u6e90\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setDest(String dest) \u5fc5\u9009 \u8bbe\u7f6e\u76ee\u6807\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setConf(Configuration conf) \u5fc5\u9009 \u8bbe\u7f6e Configuration - 4.3.0+ void setMode(DistCpMode mode) \u53ef\u9009 \u8bbe\u7f6eDistCp\u6a21\u5f0f\uff0c\u652f\u6301\u7684\u6a21\u5f0f\u6709 COPY\uff08\u62f7\u8d1d\uff09 DIFF\uff08\u5dee\u5f02\u6bd4\u8f83\uff09 UPADTE\uff08\u589e\u91cf\u62f7\u8d1d\uff09 DistCpMode.COPY 4.3.0+ void setNumWorkers(int numWorkers) \u53ef\u9009 \u8bbe\u7f6eDistCp\u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\uff0c\u5bf9\u5e94MR\u4efb\u52a1\u4e2d\u7684 mapreduce.job.maps 10 4.3.0+ void setBandWidthLimit(int bandWidthLimit) \u53ef\u9009 \u8bbe\u7f6e\u5355\u4e2a\u8282\u70b9\u7684\u5e26\u5bbd\u9650\u5236\uff0c\u5355\u4f4dM -1 4.3.0+ void setWorkTempDir(String workTempDir) \u53ef\u9009 \u8bbe\u7f6e\u4e34\u65f6\u76ee\u5f55 /tmp 4.3.0+ void setOutputCodec(String outputCodec) \u53ef\u9009 \u8bbe\u7f6e\u538b\u7f29\u7c7b\u578b\uff0c\u652f\u6301\u7f16\u89e3\u7801\u5668\u6709 gzip\u3001gz\u3001lzo\u3001lzop\u3001lzop\u3001snappy keep\uff08\u4e0d\u66f4\u6539\u538b\u7f29\u7c7b\u578b\uff09 4.3.0+ void setEnableCMS(boolean enableCMS) \u53ef\u9009 \u662f\u5426\u5f00\u542f\u76d1\u63a7\u544a\u8b66 false 4.3.0+ void setDisableChecksum(boolean disableChecksum) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5173\u95edChecksum\u68c0\u67e5 false 4.3.0+ void setDeleteOnSuccess(boolean deleteOnSuccess) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5220\u9664\u6e90\u6587\u4ef6\uff0c\u7528\u4e8e\u79fb\u52a8\u6570\u636e false 4.3.0+ void setIncludeMatcherFile(String includeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setExcludeMatcherFile(String excludeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setIgnoreFailures(boolean ignoreFailures) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5ffd\u7565\u62f7\u8d1d\u4efb\u52a1\u4e2d\u629b\u51fa\u7684\u5f02\u5e38\uff0c\u907f\u514d\u4e2d\u65ad\u4efb\u52a1 False 4.3.0+ void setStorageClass(String storageClass) \u53ef\u9009 \u8bbe\u7f6e\u76ee\u6807\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301Standard\u3001IA\u3001Archive\u3001ColdArchive Standard 4.3.0+ void setEnableTransaction(boolean enableTransaction) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4e8b\u52a1\uff0c\u4ee5\u4fdd\u8bc1Job\u7ea7\u522b\u7684\u539f\u5b50\u6027 false 4.3.0+ void setEnablePreserveMetaStatus(boolean enablePreserveMetaStatus) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f false 4.4.0+","title":"\u8bf7\u6c42\u53c2\u6570"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_api/#_4","text":"\u65b9\u6cd5 \u8bf4\u660e \u9ed8\u8ba4\u503c int getErrorCode() \u83b7\u53d6\u9519\u8bef\u7801 0 String getErrorMsg() \u83b7\u53d6\u9519\u8bef\u539f\u56e0 - long getStartTime() \u83b7\u53d6\u4efb\u52a1\u7684\u5f00\u59cb\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - long getEndTime() \u83b7\u53d6\u4efb\u52a1\u7684\u7ed3\u675f\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - getCount(String key) \u83b7\u53d6\u8ba1\u6570\u5668\u503c\uff0ckey\u4e3a\u8ba1\u6570\u5668\u540d\u79f0\uff0c -","title":"\u54cd\u5e94\u53c2\u6570"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_api/#_5","text":"\u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf BYTES_EXPECTED \u9884\u671f\u7684copy\u5b57\u8282\u6570 FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 BYTES_COPIED copy\u6210\u529f\u7684\u5b57\u8282\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"\u8ba1\u6570\u5668\u8bf4\u660e"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/","text":"JindoDistCp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66 \u00a7 JindoDistCp\uff08\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u5de5\u5177\uff09\u662f\u7528\u4e8e\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u62f7\u8d1d\u6587\u4ef6\u7684\u5de5\u5177\u3002\u4f7f\u7528MapReduce\u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3amap/reduce\u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002 CMS\uff08\u4e91\u76d1\u63a7\u670d\u52a1\uff09\uff0c\u53ef\u7528\u4e8e\u6536\u96c6\u963f\u91cc\u4e91\u8d44\u6e90\u7684\u76d1\u63a7\u6307\u6807\u6216\u7528\u6237\u81ea\u5b9a\u4e49\u7684\u76d1\u63a7\u6307\u6807\uff0c\u63a2\u6d4b\u670d\u52a1\u53ef\u7528\u6027\uff0c\u4ee5\u53ca\u9488\u5bf9\u6307\u6807\u8bbe\u7f6e\u8b66\u62a5\u3002\u4f7f\u60a8\u5168\u9762\u4e86\u89e3\u963f\u91cc\u4e91\u4e0a\u8d44\u6e90\u7684\u4f7f\u7528\u60c5\u51b5\u548c\u4e1a\u52a1\u8fd0\u884c\u72b6\u51b5\uff0c\u5e76\u53ca\u65f6\u5bf9\u6545\u969c\u8d44\u6e90\u8fdb\u884c\u5904\u7406\uff0c\u4fdd\u8bc1\u4e1a\u52a1\u6b63\u5e38\u8fd0\u884c\u3002 \u60a8\u53ef\u4ee5\u6307\u5b9a\u672c\u6b21JindoDistCp\u4efb\u52a1\u7ed3\u675f\u540e\u662f\u5426\u9700\u8981\u5f00\u542fCMS\uff0c\u4ee5\u4e0a\u62a5\u4efb\u52a1\u5931\u8d25\u4fe1\u606f\uff0c\u5e76\u4f7f\u7528CMS\u63a7\u5236\u53f0\u6765\u914d\u7f6e\u544a\u8b66\u529f\u80fd\u3002\u544a\u8b66\u6761\u4ef6\u53c2\u89c1 \u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668 \u3002 \u6b65\u9aa4 \u00a7 1. \u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u548c\u62a5\u8b66\u8054\u7cfb\u7ec4 \u00a7 \u5177\u4f53\u8bf7\u53c2\u8003\u6587\u6863 \u300a\u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u6216\u62a5\u8b66\u8054\u7cfb\u7ec4\u300b 2. \u83b7\u53d6\u62a5\u8b66Token \u00a7 \u70b9\u51fb\u5de6\u4fa7\u5bfc\u822a\u680f\u201c\u62a5\u8b66\u670d\u52a1\u201d->\"\u62a5\u8b66\u8054\u7cfb\u4eba\"->\u5207\u6362\u4e0a\u6d4b\u9875\u7b7e\u5230\"\u62a5\u8b66\u8054\u7cfb\u7ec4\"->\u9009\u62e9\u5bf9\u5e94\u7684\u62a5\u8b66\u8054\u7cfb\u7ec4->\u70b9\u51fb\u53f3\u4fa7\u63a5\u5165\u5916\u90e8\u62a5\u8b66 3. \u914d\u7f6e\u73af\u5883\u53d8\u91cf \u00a7 \u73af\u5883\u53d8\u91cf \u8bf4\u660e cmsAccessKeyId \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94AccessKey ID cmsAccessSecret \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94Access Key Secret cmsRegion \u8bbe\u7f6eCMS\u6240\u5728Region\uff0c\u9700\u8981\u4e0e\u5176\u4ed6\u76f8\u5173\u7684\u4e91\u4e0a\u8d44\u6e90\u4e00\u81f4\uff0c\u4f8b\u5982ECS\uff0cOSS\uff0cEMR\u6240\u5bf9\u5e94\u7684Region cmsLevel \u8bbe\u7f6e\u6807\u8bc6\u544a\u8b66\u901a\u77e5\u65b9\u5f0f\uff0c\u5982\u679c\u672a\u8bbe\u7f6e\uff0c\u9ed8\u8ba4\u4e3aWARN\u3002INFO\u4e3a \u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cWARN\u4e3a \u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cCRITICAL\u4e3a \u7535\u8bdd+\u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \u793a\u4f8b\u5982\u4e0b export cmsAccessKeyId=<your_key_id> export cmsAccessSecret=<your_key_secret> export cmsRegion=cn-hangzhou export cmsToken=<your_cms_token> export cmsLevel=WARN 4. \u5f00\u542fCMS\u5bf9\u5931\u8d25\u4efb\u52a1\u8fdb\u884c\u544a\u8b66 \u00a7 copy\u64cd\u4f5c \u00a7 hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table \\ --enableCMS diff\u64cd\u4f5c \u00a7 hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table \\ --diff \\ --enableCMS update\u64cd\u4f5c \u00a7 hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table \\ --update \\ --enableCMS \u544a\u8b66\u5185\u5bb9 \u00a7 \u544a\u8b66\u5185\u5bb9\u683c\u5f0f \u00a7 { \"JobTime\": \"2021-03-09 19:45:27~2021-03-09 19:45:28\", \"SrcPath\": \"hdfs://cluster-xxx:9000//data/incoming/hourly_table\", \"DestPath\": \"oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table\", \"Counter\": { \"COUNTER_1\": XXX, \"COUNTER_2\": XXX, ... } } ... \u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668 \u00a7 \u544a\u8b66\u5185\u5bb9\u4e2d\u7684counter\u8ba1\u6570\u5668\u6709\u4ee5\u4e0b\u51e0\u79cd\uff1a copy\u64cd\u4f5c \u00a7 \u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 diff\u64cd\u4f5c \u00a7 \u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"JindoDistCp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#jindodistcpcms","text":"JindoDistCp\uff08\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u5de5\u5177\uff09\u662f\u7528\u4e8e\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u62f7\u8d1d\u6587\u4ef6\u7684\u5de5\u5177\u3002\u4f7f\u7528MapReduce\u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3amap/reduce\u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002 CMS\uff08\u4e91\u76d1\u63a7\u670d\u52a1\uff09\uff0c\u53ef\u7528\u4e8e\u6536\u96c6\u963f\u91cc\u4e91\u8d44\u6e90\u7684\u76d1\u63a7\u6307\u6807\u6216\u7528\u6237\u81ea\u5b9a\u4e49\u7684\u76d1\u63a7\u6307\u6807\uff0c\u63a2\u6d4b\u670d\u52a1\u53ef\u7528\u6027\uff0c\u4ee5\u53ca\u9488\u5bf9\u6307\u6807\u8bbe\u7f6e\u8b66\u62a5\u3002\u4f7f\u60a8\u5168\u9762\u4e86\u89e3\u963f\u91cc\u4e91\u4e0a\u8d44\u6e90\u7684\u4f7f\u7528\u60c5\u51b5\u548c\u4e1a\u52a1\u8fd0\u884c\u72b6\u51b5\uff0c\u5e76\u53ca\u65f6\u5bf9\u6545\u969c\u8d44\u6e90\u8fdb\u884c\u5904\u7406\uff0c\u4fdd\u8bc1\u4e1a\u52a1\u6b63\u5e38\u8fd0\u884c\u3002 \u60a8\u53ef\u4ee5\u6307\u5b9a\u672c\u6b21JindoDistCp\u4efb\u52a1\u7ed3\u675f\u540e\u662f\u5426\u9700\u8981\u5f00\u542fCMS\uff0c\u4ee5\u4e0a\u62a5\u4efb\u52a1\u5931\u8d25\u4fe1\u606f\uff0c\u5e76\u4f7f\u7528CMS\u63a7\u5236\u53f0\u6765\u914d\u7f6e\u544a\u8b66\u529f\u80fd\u3002\u544a\u8b66\u6761\u4ef6\u53c2\u89c1 \u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668 \u3002","title":"JindoDistCp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#1","text":"\u5177\u4f53\u8bf7\u53c2\u8003\u6587\u6863 \u300a\u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u6216\u62a5\u8b66\u8054\u7cfb\u7ec4\u300b","title":"1. \u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u548c\u62a5\u8b66\u8054\u7cfb\u7ec4"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#2-token","text":"\u70b9\u51fb\u5de6\u4fa7\u5bfc\u822a\u680f\u201c\u62a5\u8b66\u670d\u52a1\u201d->\"\u62a5\u8b66\u8054\u7cfb\u4eba\"->\u5207\u6362\u4e0a\u6d4b\u9875\u7b7e\u5230\"\u62a5\u8b66\u8054\u7cfb\u7ec4\"->\u9009\u62e9\u5bf9\u5e94\u7684\u62a5\u8b66\u8054\u7cfb\u7ec4->\u70b9\u51fb\u53f3\u4fa7\u63a5\u5165\u5916\u90e8\u62a5\u8b66","title":"2. \u83b7\u53d6\u62a5\u8b66Token"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#3","text":"\u73af\u5883\u53d8\u91cf \u8bf4\u660e cmsAccessKeyId \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94AccessKey ID cmsAccessSecret \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94Access Key Secret cmsRegion \u8bbe\u7f6eCMS\u6240\u5728Region\uff0c\u9700\u8981\u4e0e\u5176\u4ed6\u76f8\u5173\u7684\u4e91\u4e0a\u8d44\u6e90\u4e00\u81f4\uff0c\u4f8b\u5982ECS\uff0cOSS\uff0cEMR\u6240\u5bf9\u5e94\u7684Region cmsLevel \u8bbe\u7f6e\u6807\u8bc6\u544a\u8b66\u901a\u77e5\u65b9\u5f0f\uff0c\u5982\u679c\u672a\u8bbe\u7f6e\uff0c\u9ed8\u8ba4\u4e3aWARN\u3002INFO\u4e3a \u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cWARN\u4e3a \u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cCRITICAL\u4e3a \u7535\u8bdd+\u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \u793a\u4f8b\u5982\u4e0b export cmsAccessKeyId=<your_key_id> export cmsAccessSecret=<your_key_secret> export cmsRegion=cn-hangzhou export cmsToken=<your_cms_token> export cmsLevel=WARN","title":"3. \u914d\u7f6e\u73af\u5883\u53d8\u91cf"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#4-cms","text":"","title":"4. \u5f00\u542fCMS\u5bf9\u5931\u8d25\u4efb\u52a1\u8fdb\u884c\u544a\u8b66"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#copy","text":"hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table \\ --enableCMS","title":"copy\u64cd\u4f5c"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#diff","text":"hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table \\ --diff \\ --enableCMS","title":"diff\u64cd\u4f5c"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#update","text":"hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table \\ --update \\ --enableCMS","title":"update\u64cd\u4f5c"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#_2","text":"","title":"\u544a\u8b66\u5185\u5bb9"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#_3","text":"{ \"JobTime\": \"2021-03-09 19:45:27~2021-03-09 19:45:28\", \"SrcPath\": \"hdfs://cluster-xxx:9000//data/incoming/hourly_table\", \"DestPath\": \"oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table\", \"Counter\": { \"COUNTER_1\": XXX, \"COUNTER_2\": XXX, ... } } ...","title":"\u544a\u8b66\u5185\u5bb9\u683c\u5f0f"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#_4","text":"\u544a\u8b66\u5185\u5bb9\u4e2d\u7684counter\u8ba1\u6570\u5668\u6709\u4ee5\u4e0b\u51e0\u79cd\uff1a","title":"\u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#copy_1","text":"\u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570","title":"copy\u64cd\u4f5c"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_how_to_cms/#diff_1","text":"\u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"diff\u64cd\u4f5c"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/","text":"\u6570\u636e\u5728 OSS-HDFS \u670d\u52a1\u4e0d\u540c bucket \u4e4b\u95f4\u8fc1\u79fb \u00a7 \u4f7f\u7528\u524d\u987b\u77e5 \u00a7 \u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\uff0c\u4e5f\u53ef \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988 1\u3001\u62f7\u8d1d\u6570\u636e\u5230 OSS-HDFS \u670d\u52a1\u4e0a \u00a7 \u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS-HDFS \u670d\u52a1\u4e0a\u3002 \u5982\u679c\u60a8\u7684\u4e24\u4e2a\u8981\u62f7\u8d1d\u7684 bucket \u90fd\u5728\u540c\u4e00\u4e2a region \u4e0b\uff0c\u4e14\u53ef\u7528\u540c\u4e00\u4e2a ossKey \u548c ossSecret \u8fdb\u884c\u8bfb\u5199\uff0c\u60a8\u53ef\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket.cn-xxx.oss-dls.aliyuncs.com/ --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src OSS-HDFS \u670d\u52a1\u7684\u6e90\u8def\u5f84\u3002 oss://srcBucket.cn-xxx.oss-dls.aliyuncs.com/ --dest OSS-HDFS \u670d\u52a1\u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/ --hadoopConf \u6307\u5b9a OSS-HDFS \u670d\u52a1\u7684 Access Key ID , Access Key Secret * \u914d\u7f6e OSS-HDFS \u670d\u52a1\u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS-HDFS \u670d\u52a1\u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10 \u5982\u679c\u60a8\u7684 src \u548c dest \u7684 bucket \u4e0d\u5728\u540c\u4e00\u4e2a region \u6216\u8005\u4e0d\u80fd\u4f7f\u7528\u540c\u4e00\u4e2a ossKey \u548c ossSecret \u8fdb\u884c\u8bfb\u5199\uff0c\u90a3\u4e48\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcbucket.oss-cn-xxx.aliyuncs.com/ --dest oss://destBucket.oss-cn-xxx.aliyuncs.com/ --hadoopConf fs.oss.bucket.srcbucket.accessKeyId=yourkey --hadoopConf fs.oss.bucket.srcbucket.accessKeySecret=yoursecret --hadoopConf --hadoopConf fs.oss.bucket.destBucket.accessKeyId=yourkey --hadoopConf fs.oss.bucket.destBucket.accessKeySecret=yoursecret --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --hadoopConf \u7ed9 src \u548c dest \u7684 bucket \u6307\u5b9a\u4e0d\u540c Access Key ID , Access Key Secret * \u914d\u7f6e XXX bucket \u7684 AccessKeyId: --hadoopConf fs.oss.bucket.XXX.accessKeyId=yourkey * \u914d\u7f6e OSS-HDFS \u670d\u52a1\u7684 AccessKeySecret: --hadoopConf fs.oss.bucket.XXX.accessKeySecret=yoursecret 2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6 \u00a7 \u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53ea Copy \u5269\u4e0b\u672a Copy \u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a \u4f7f\u7528 --update \u547d\u4ee4\uff0c\u83b7\u5f97\u589e\u91cf\u7684\u6587\u4ef6\u5217\u8868 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket.oss-cn-xxx.aliyuncs.com/ --dest oss://destBucket.oss-cn-xxx.aliyuncs.com/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --update --parallelism 20 3\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9 \u00a7 \u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket.oss-cn-xxx.aliyuncs.com/ --dest oss://destBucket.oss-cn-xxx.aliyuncs.com/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB 4\u3001\u514d\u5bc6\u53ca\u5bc6\u94a5\u56fa\u5b9a\u5b58\u50a8 \u00a7 \u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS-HDFS \u670d\u52a1\u7684 AccessKey/AccessSecret \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662fJindoDistCp\u53ef\u4ee5\u5c06\u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> 5\u3001\u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u00a7 \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 JindoFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 6\u3001\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f \u00a7 \u4f7f\u7528preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket.oss-cn-xxx.aliyuncs.com/ --dest oss://destBucket.oss-cn-xxx.aliyuncs.com/ --preserveMeta 7\u3001\u5176\u4ed6\u529f\u80fd \u00a7 \u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 * JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e","title":"\u6570\u636e\u5728 OSS-HDFS \u670d\u52a1\u4e0d\u540c bucket \u4e4b\u95f4\u8fc1\u79fb"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/#oss-hdfs-bucket","text":"","title":"\u6570\u636e\u5728 OSS-HDFS \u670d\u52a1\u4e0d\u540c bucket \u4e4b\u95f4\u8fc1\u79fb"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/#_1","text":"\u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\uff0c\u4e5f\u53ef \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988","title":"\u4f7f\u7528\u524d\u987b\u77e5"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/#1-oss-hdfs","text":"\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS-HDFS \u670d\u52a1\u4e0a\u3002 \u5982\u679c\u60a8\u7684\u4e24\u4e2a\u8981\u62f7\u8d1d\u7684 bucket \u90fd\u5728\u540c\u4e00\u4e2a region \u4e0b\uff0c\u4e14\u53ef\u7528\u540c\u4e00\u4e2a ossKey \u548c ossSecret \u8fdb\u884c\u8bfb\u5199\uff0c\u60a8\u53ef\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket.cn-xxx.oss-dls.aliyuncs.com/ --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src OSS-HDFS \u670d\u52a1\u7684\u6e90\u8def\u5f84\u3002 oss://srcBucket.cn-xxx.oss-dls.aliyuncs.com/ --dest OSS-HDFS \u670d\u52a1\u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/ --hadoopConf \u6307\u5b9a OSS-HDFS \u670d\u52a1\u7684 Access Key ID , Access Key Secret * \u914d\u7f6e OSS-HDFS \u670d\u52a1\u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS-HDFS \u670d\u52a1\u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10 \u5982\u679c\u60a8\u7684 src \u548c dest \u7684 bucket \u4e0d\u5728\u540c\u4e00\u4e2a region \u6216\u8005\u4e0d\u80fd\u4f7f\u7528\u540c\u4e00\u4e2a ossKey \u548c ossSecret \u8fdb\u884c\u8bfb\u5199\uff0c\u90a3\u4e48\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcbucket.oss-cn-xxx.aliyuncs.com/ --dest oss://destBucket.oss-cn-xxx.aliyuncs.com/ --hadoopConf fs.oss.bucket.srcbucket.accessKeyId=yourkey --hadoopConf fs.oss.bucket.srcbucket.accessKeySecret=yoursecret --hadoopConf --hadoopConf fs.oss.bucket.destBucket.accessKeyId=yourkey --hadoopConf fs.oss.bucket.destBucket.accessKeySecret=yoursecret --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --hadoopConf \u7ed9 src \u548c dest \u7684 bucket \u6307\u5b9a\u4e0d\u540c Access Key ID , Access Key Secret * \u914d\u7f6e XXX bucket \u7684 AccessKeyId: --hadoopConf fs.oss.bucket.XXX.accessKeyId=yourkey * \u914d\u7f6e OSS-HDFS \u670d\u52a1\u7684 AccessKeySecret: --hadoopConf fs.oss.bucket.XXX.accessKeySecret=yoursecret","title":"1\u3001\u62f7\u8d1d\u6570\u636e\u5230 OSS-HDFS \u670d\u52a1\u4e0a"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/#2","text":"\u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53ea Copy \u5269\u4e0b\u672a Copy \u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a","title":"2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/#-update","text":"hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket.oss-cn-xxx.aliyuncs.com/ --dest oss://destBucket.oss-cn-xxx.aliyuncs.com/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --update --parallelism 20","title":"\u4f7f\u7528 --update \u547d\u4ee4\uff0c\u83b7\u5f97\u589e\u91cf\u7684\u6587\u4ef6\u5217\u8868"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/#3yarn","text":"\u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket.oss-cn-xxx.aliyuncs.com/ --dest oss://destBucket.oss-cn-xxx.aliyuncs.com/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB","title":"3\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/#4","text":"\u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS-HDFS \u670d\u52a1\u7684 AccessKey/AccessSecret \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662fJindoDistCp\u53ef\u4ee5\u5c06\u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration>","title":"4\u3001\u514d\u5bc6\u53ca\u5bc6\u94a5\u56fa\u5b9a\u5b58\u50a8"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/#5-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 JindoFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002","title":"5\u3001\u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/#6","text":"\u4f7f\u7528preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket.oss-cn-xxx.aliyuncs.com/ --dest oss://destBucket.oss-cn-xxx.aliyuncs.com/ --preserveMeta","title":"6\u3001\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_dls_to_dls/#7","text":"\u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 * JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e","title":"7\u3001\u5176\u4ed6\u529f\u80fd"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/","text":"\u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\u4e0a \u00a7 \u4f7f\u7528\u524d\u987b\u77e5 \u00a7 \u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\uff0c\u4e5f\u53ef \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988 1\u3001\u62f7\u8d1d\u6570\u636e\u5230\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a \u00a7 \u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06 HDFS \u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS-HDFS \u670d\u52a1\u4e0a hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src HDFS \u7684\u6e90\u8def\u5f84\u3002 /data --dest OSS-HDFS \u670d\u52a1\u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/ --hadoopConf \u6307\u5b9a OSS-HDFS \u670d\u52a1\u7684 Access Key ID , Access Key Secret * \u914d\u7f6e OSS-HDFS \u670d\u52a1\u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS-HDFS \u670d\u52a1 \u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10 2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6 \u00a7 \u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53eaCopy\u5269\u4e0b\u672aCopy\u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a \u4f7f\u7528 --update \u547d\u4ee4\uff0c\u83b7\u5f97\u589e\u91cf\u7684\u6587\u4ef6\u5217\u8868 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --update --parallelism 20 3\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9 \u00a7 \u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB 4\u3001\u514d\u5bc6\u53ca\u5bc6\u94a5\u56fa\u5b9a\u5b58\u50a8 \u00a7 \u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS-HDFS \u670d\u52a1 AccessKey/AccessSecret \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662fJindoDistCp\u53ef\u4ee5\u5c06 \u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> 5\u3001\u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u00a7 \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 JindoFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u6ce8\u610f \uff1a JindoSDK 4.4.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u9ed8\u8ba4\u4f7f\u7528\u57df\u540d\u5206\u79bb\u7684\u65b9\u5f0f\uff0c\u6570\u636e\u8bfb\u5199\u57df\u540d\u9ed8\u8ba4\u4f7f\u7528\u5185\u7f51\u6807\u51c6 OSS \u57df\u540d\uff0c\u5982\u679c\u5728\u975e\u963f\u91cc\u4e91\u5185\u7f51\u73af\u5883\u6267\u884c distcp \u547d\u4ee4\uff0c\u5219\u9700\u8981\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u914d\u7f6e\u6570\u636e\u6d41\u8bbf\u95ee\u7684\u516c\u7f51 OSS endpoint\u3002 <configuration> <property> <name>fs.oss.data.endpoint</name> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> 6\u3001\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f \u00a7 \u4f7f\u7528preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --preserveMeta 7\u3001\u5176\u4ed6\u529f\u80fd \u00a7 \u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 * JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e","title":"\u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\u4e0a"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/#hdfs-oss-hdfs","text":"","title":"\u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\u4e0a"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/#_1","text":"\u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\uff0c\u4e5f\u53ef \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988","title":"\u4f7f\u7528\u524d\u987b\u77e5"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/#1-oss-hdfs-jindofs","text":"\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06 HDFS \u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS-HDFS \u670d\u52a1\u4e0a hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src HDFS \u7684\u6e90\u8def\u5f84\u3002 /data --dest OSS-HDFS \u670d\u52a1\u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/ --hadoopConf \u6307\u5b9a OSS-HDFS \u670d\u52a1\u7684 Access Key ID , Access Key Secret * \u914d\u7f6e OSS-HDFS \u670d\u52a1\u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS-HDFS \u670d\u52a1 \u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10","title":"1\u3001\u62f7\u8d1d\u6570\u636e\u5230\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/#2","text":"\u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53eaCopy\u5269\u4e0b\u672aCopy\u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a","title":"2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/#-update","text":"hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --update --parallelism 20","title":"\u4f7f\u7528 --update \u547d\u4ee4\uff0c\u83b7\u5f97\u589e\u91cf\u7684\u6587\u4ef6\u5217\u8868"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/#3yarn","text":"\u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB","title":"3\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/#4","text":"\u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS-HDFS \u670d\u52a1 AccessKey/AccessSecret \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662fJindoDistCp\u53ef\u4ee5\u5c06 \u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration>","title":"4\u3001\u514d\u5bc6\u53ca\u5bc6\u94a5\u56fa\u5b9a\u5b58\u50a8"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/#5-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 JindoFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u6ce8\u610f \uff1a JindoSDK 4.4.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u9ed8\u8ba4\u4f7f\u7528\u57df\u540d\u5206\u79bb\u7684\u65b9\u5f0f\uff0c\u6570\u636e\u8bfb\u5199\u57df\u540d\u9ed8\u8ba4\u4f7f\u7528\u5185\u7f51\u6807\u51c6 OSS \u57df\u540d\uff0c\u5982\u679c\u5728\u975e\u963f\u91cc\u4e91\u5185\u7f51\u73af\u5883\u6267\u884c distcp \u547d\u4ee4\uff0c\u5219\u9700\u8981\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u914d\u7f6e\u6570\u636e\u6d41\u8bbf\u95ee\u7684\u516c\u7f51 OSS endpoint\u3002 <configuration> <property> <name>fs.oss.data.endpoint</name> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration>","title":"5\u3001\u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/#6","text":"\u4f7f\u7528preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --preserveMeta","title":"6\u3001\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_on_hdfs_to_dls/#7","text":"\u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 * JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e","title":"7\u3001\u5176\u4ed6\u529f\u80fd"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_overview/","text":"JindoDistCp \u6570\u636e\u8fc1\u79fb \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 JDK 1.8\u53ca\u4ee5\u4e0a Hadoop 2.3+\u7248\u672c\uff0c\u8bf7\u4e0b\u8f7d\u6700\u65b0\u7248\u7684 jindo-distcp-tool-x.x.x.jar \uff0c\u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz \u5185\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 tools/ \u76ee\u5f55\u4e0b\u627e\u5230\uff0c \u4e0b\u8f7d\u9875\u9762 \u3002 \u8be5\u7248\u672c\u57fa\u4e8e native \u4ee3\u7801\u5b9e\u73b0\uff0c\u652f\u6301\u529f\u80fd\u8f83\u4e30\u5bcc (2.3 \u4ee5\u524d\u7248\u672c\u6682\u672a\u6d4b\u8bd5\uff0c\u5982\u6709\u95ee\u9898\u8bf7 \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988) JindoDistCp \u4f7f\u7528\u573a\u666f \u00a7 \u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\u4e0a \u6570\u636e\u5728 OSS-HDFS \u670d\u52a1\u4e0d\u540c bucket \u4e4b\u95f4\u8fc1\u79fb JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u00a7 \u6211\u4eec\u603b\u7ed3\u4e86\u4e00\u4e9b\u5728\u4e0d\u540c\u73af\u5883\u4f7f\u7528 JindoDistCp \u7528\u6237\u53ef\u80fd\u4f1a\u9047\u5230\u7684\u4e00\u4e9b\u95ee\u9898\uff0c\u60a8\u53ef\u4ee5\u53c2\u7167\u4e0b\u6587\u8fdb\u884c\u5e38\u89c1\u95ee\u9898\u6392\u67e5\u548c\u89e3\u51b3 * JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 JindoDistCp \u7cfb\u5217\u6587\u7ae0 \u00a7 \u300a\u518d\u51fa\u738b\u724c\uff1a\u963f\u91cc\u4e91 JindoDistCp \u5168\u9762\u5f00\u653e\u4f7f\u7528\uff0c\u6210\u4e3a\u963f\u91cc\u4e91\u6570\u636e\u8fc1\u79fb\u5229\u5668\u300b JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e (\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177)","title":"JindoDistCp \u6570\u636e\u8fc1\u79fb"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_overview/#jindodistcp","text":"","title":"JindoDistCp \u6570\u636e\u8fc1\u79fb"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_overview/#_1","text":"JDK 1.8\u53ca\u4ee5\u4e0a Hadoop 2.3+\u7248\u672c\uff0c\u8bf7\u4e0b\u8f7d\u6700\u65b0\u7248\u7684 jindo-distcp-tool-x.x.x.jar \uff0c\u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz \u5185\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 tools/ \u76ee\u5f55\u4e0b\u627e\u5230\uff0c \u4e0b\u8f7d\u9875\u9762 \u3002 \u8be5\u7248\u672c\u57fa\u4e8e native \u4ee3\u7801\u5b9e\u73b0\uff0c\u652f\u6301\u529f\u80fd\u8f83\u4e30\u5bcc (2.3 \u4ee5\u524d\u7248\u672c\u6682\u672a\u6d4b\u8bd5\uff0c\u5982\u6709\u95ee\u9898\u8bf7 \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988)","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_overview/#jindodistcp_1","text":"\u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\u4e0a \u6570\u636e\u5728 OSS-HDFS \u670d\u52a1\u4e0d\u540c bucket \u4e4b\u95f4\u8fc1\u79fb","title":"JindoDistCp \u4f7f\u7528\u573a\u666f"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_overview/#jindodistcp_2","text":"\u6211\u4eec\u603b\u7ed3\u4e86\u4e00\u4e9b\u5728\u4e0d\u540c\u73af\u5883\u4f7f\u7528 JindoDistCp \u7528\u6237\u53ef\u80fd\u4f1a\u9047\u5230\u7684\u4e00\u4e9b\u95ee\u9898\uff0c\u60a8\u53ef\u4ee5\u53c2\u7167\u4e0b\u6587\u8fdb\u884c\u5e38\u89c1\u95ee\u9898\u6392\u67e5\u548c\u89e3\u51b3 * JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357","title":"JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357"},{"location":"backup/jindofs/jindo_distcp/jindo_distcp_overview/#jindodistcp_3","text":"\u300a\u518d\u51fa\u738b\u724c\uff1a\u963f\u91cc\u4e91 JindoDistCp \u5168\u9762\u5f00\u653e\u4f7f\u7528\uff0c\u6210\u4e3a\u963f\u91cc\u4e91\u6570\u636e\u8fc1\u79fb\u5229\u5668\u300b JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e (\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177)","title":"JindoDistCp \u7cfb\u5217\u6587\u7ae0"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/","text":"\u4f7f\u7528 JindoFuse \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u901a\u8fc7 JindoFuse \u63d0\u4f9b POSIX \u652f\u6301\u3002JindoFuse \u53ef\u4ee5\u628a OSS-HDFS \u670d\u52a1\u4e0a\u7684\u6587\u4ef6\u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c\u8be5\u670d\u52a1\u4e0a\u7684\u6587\u4ef6\u3002 \u57fa\u672c\u4f7f\u7528 \u00a7 \u73af\u5883\u51c6\u5907 \u00a7 # CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev \u914d\u7f6e\u5ba2\u6237\u7aef \u00a7 \u914d\u7f6e\u76ee\u5f55 \u00a7 \u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf \u914d\u7f6e\u6587\u4ef6 \u00a7 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u5f00\u542fHDFS\u670d\u52a1\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> \u514d\u5bc6\u8bbf\u95ee \u00a7 \u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7\u89d2\u8272\u6388\u6743\u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u5f00\u542fHDFS\u670d\u52a1\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE fs.oss.provider.format = JSON \u6302\u8f7d JindoFuse \u00a7 \u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002 \u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 mkdir -p <mount_point> \u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 jindo-fuse <mount_point> -ouri=[<oss_path>] -ouri \u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684 oss \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a Bucket \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002 \u8bbf\u95ee JindoFuse \u00a7 \u5982\u679c\u5c06 OSS-HDFS \u670d\u52a1\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1 ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/ \u5378\u8f7d JindoFuse \u00a7 \u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point> \u81ea\u52a8\u5378\u8f7d JindoFuse \u00a7 \u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u7279\u6027\u652f\u6301 \u00a7 \u76ee\u524d JindoFuse \u5df2\u7ecf\u652f\u6301\u4ee5\u4e0b POSIX API\uff1a \u7279\u6027 \u8bf4\u660e getattr() \u67e5\u8be2\u6587\u4ef6\u5c5e\u6027\uff0c\u7c7b\u4f3c ls mkdir() \u521b\u5efa\u76ee\u5f55\uff0c\u7c7b\u4f3c mkdir rmdir() \u5220\u9664\u76ee\u5f55\uff0c\u7c7b\u4f3c rm -rf unlink() \u5220\u9664\u6587\u4ef6\uff0c\u7c7b\u4f3c unlink rename() \u91cd\u547d\u540d\uff0c\u7c7b\u4f3c mv read() \u987a\u5e8f\u8bfb\u53d6 pread() \u968f\u673a\u8bfb write() \u987a\u5e8f\u5199 pwrite() \u968f\u673a\u5199 flush() \u5237\u65b0\u5185\u5b58\u5230\u5185\u6838\u7f13\u51b2\u533a fsync() \u5237\u65b0\u5185\u5b58\u5230\u78c1\u76d8 release() \u5173\u95ed\u6587\u4ef6 readdir() \u8bfb\u53d6\u76ee\u5f55 create() \u521b\u5efa\u6587\u4ef6 open() O_APPEND \u901a\u8fc7\u8ffd\u52a0\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 open() O_TRUNC \u901a\u8fc7\u8986\u76d6\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 ftruncate() \u5bf9\u6253\u5f00\u7684\u6587\u4ef6\u8fdb\u884c\u622a\u65ad truncate() \u5bf9\u672a\u6253\u5f00\u7684\u6587\u4ef6\u8fdb\u884c\u622a\u65ad\uff0c\u7c7b\u4f3c truncate -s lseek() \u6307\u5b9a\u6253\u5f00\u6587\u4ef6\u4e2d\u7684\u8bfb\u5199\u4f4d\u7f6e\u3002 chmod() \u4fee\u6539\u6587\u4ef6\u6743\u9650\uff0c\u7c7b\u4f3c chmod access() \u67e5\u8be2\u6587\u4ef6\u6743\u9650 utimes() \u4fee\u6539\u6587\u4ef6\u7684\u5b58\u53d6\u65f6\u95f4\u548c\u66f4\u6539\u65f6\u95f4(\u4ec5\u652f\u6301\u4fee\u6539\u4e3a\u66f4\u65b0\u7684\u65f6\u95f4) setxattr() \u4fee\u6539\u6587\u4ef6 xattr \u5c5e\u6027 getxattr() \u83b7\u53d6\u6587\u4ef6 xattr \u5c5e\u6027 listxattr() \u5217\u4e3e\u6587\u4ef6 xattr \u5c5e\u6027 removexattr() \u5220\u9664\u6587\u4ef6 xattr \u5c5e\u6027 lock() \u652f\u6301 posix \u9501\uff0c\u7c7b\u4f3c fcntl fallocate() \u4e3a\u6587\u4ef6\u9884\u5206\u914d\u7269\u7406\u7a7a\u95f4 symlink() \u521b\u5efa\u8f6f\u8fde\u63a5\uff0c\u76ee\u524d\u4ec5\u652f\u6301 OSS-HDFS \u5185\u90e8\u4f7f\u7528\uff0c\u4e14\u4e0d\u652f\u6301\u7f13\u5b58\u52a0\u901f readlink() \u8bfb\u53d6\u8f6f\u8fde\u63a5 \u9ad8\u9636\u4f7f\u7528 \u00a7 \u6302\u8f7d\u9009\u9879 \u00a7 \u53c2\u6570\u540d\u79f0 \u5fc5\u9009 \u53c2\u6570\u8bf4\u660e \u4f7f\u7528\u8303\u4f8b uri \u2713 \u914d\u7f6e\u9700\u8981\u6620\u5c04\u7684 oss \u8def\u5f84\u3002\u8def\u5f84\u53ef\u4ee5\u662f\u6839\u76ee\u5f55\uff0c\u4e5f\u53ef\u4ee5\u662f\u5b50\u76ee\u5f55\u3002\u4f8b\u5982\uff1aoss://bucket.endpoint/ \u6216 oss://bucket.endpoint/subdir\u3002 -ouri=oss://examplebucket.cn-beijing.oss-dls.aliyuncs.com/ f \u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u9ed8\u8ba4\u4f7f\u7528\u5b88\u62a4\u8fdb\u7a0b\u65b9\u5f0f\u540e\u53f0\u542f\u52a8\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -f d \u4f7f\u7528 Debug \u6a21\u5f0f\uff0c\u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -d auto_unmount fuse\u8fdb\u7a0b\u9000\u51fa\u540e\u81ea\u52a8umount\u6302\u8f7d\u8282\u70b9\u3002 -oauto_unmount ro \u53ea\u8bfb\u6302\u8f7d\uff0c\u542f\u7528\u53c2\u6570\u540e\u4e0d\u5141\u8bb8\u5199\u64cd\u4f5c\u3002 -oro direct_io \u5f00\u542f\u540e\uff0c\u8bfb\u5199\u6587\u4ef6\u53ef\u4ee5\u7ed5\u8fc7page cache\u3002 -odirect_io kernel_cache \u5f00\u542f\u540e\uff0c\u5229\u7528\u5185\u6838\u7f13\u5b58\u4f18\u5316\u8bfb\u6027\u80fd\u3002 -okernel_cache auto_cache \u9ed8\u8ba4\u5f00\u542f\uff0c\u4e0ekernel_cache \u4e8c\u9009\u4e00\uff0c\u4e0ekernel_cache\u4e0d\u540c\u7684\u662f\uff0c\u5982\u679c\u6587\u4ef6\u5927\u5c0f\u6216\u4fee\u6539\u65f6\u95f4\u53d1\u751f\u53d8\u5316\uff0c\u7f13\u5b58\u5c31\u4f1a\u5931\u6548\u3002 entry_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u76ee\u5f55\u6761\u76ee\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oentry_timeout=60 attr_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oattr_timeout=60 negative_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u5931\u8d25\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -onegative_timeout=0 jindo_entry_size \u9ed8\u8ba4\u503c\uff0c5000\u3002\u76ee\u5f55\u6761\u76ee\u7f13\u5b58\u6570\u91cf\uff0c\u7528\u4e8e\u4f18\u5316readdir\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -ojindo_entry_size=5000 jindo_attr_size \u9ed8\u8ba4\u503c\uff0c50000\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u6570\u91cf\uff0c\u7528\u4e8e\u4f18\u5316getattr\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -ojindo_attr_sizet=50000 max_idle_threads \u9ed8\u8ba4\u503c\uff0c10\u3002\u6700\u5927\u7a7a\u95f2\u7ebf\u7a0b\u6570\u3002 -omax_idle_threads=10 metrics_port \u9ed8\u8ba4\u503c\uff0c9090\u3002\u5f00\u542fhttp\u7aef\u53e3\uff0c\u8f93\u51fametrics\uff0c\u5982http://localhost:9090/brpc_metrics -ometrics_port=9090 enable_pread \u5f00\u542f\u540e\uff0c\u4f7f\u7528pread\u63a5\u53e3\u8bfb\u53d6\u6587\u4ef6\u3002 -oenable_pread \u914d\u7f6e\u9009\u9879 \u00a7 \u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u914d\u7f6e\u8282 \u8bf4\u660e logger.dir /tmp/fuse-log common \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false common \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false common \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\u3002 \u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002 logger.verbose 0 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false common \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u5730\u5740\uff0c\u5982cn-xxx.oss-dls.aliyuncs.com fs.oss.accessKeyId jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeySecret \u66f4\u591a\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<oss_path>] -ofs.oss.endpoint=[<your_endpoint>] -ofs.oss.accessKeyId=[<your_key_id>] -ofs.oss.accessKeySecret=[<your_key_secret>] \u5e38\u89c1\u95ee\u9898 \u00a7 Input/Output error \u00a7 \u4e0d\u50cf\u4f7f\u7528 JindoSDK \u8c03\u7528 API \u53ef\u4ee5\u83b7\u53d6\u66f4\u4e3a\u5177\u4f53\u7684 ErrorMsg\uff0cJindoFuse \u53ea\u80fd\u663e\u793a\u64cd\u4f5c\u7cfb\u7edf\u9884\u8bbe\u7684\u9519\u8bef\u4fe1\u606f\uff0c\u6bd4\u5982\u4ee5\u4e0b\u9519\u8bef\u5c31\u975e\u5e38\u5e38\u89c1\uff1a $ ls /mnt/oss/ ls: /mnt/oss/: Input/output error \u5982\u679c\u9700\u8981\u5b9a\u4f4d\u5177\u4f53\u7684\u9519\u8bef\u539f\u56e0\uff0c\u53ef\u4ee5\u6839\u636e JindoSDK \u914d\u7f6e\u4e2d\u7684 logger.dir \uff0c\u5728\u6307\u5b9a\u8def\u5f84\u4e0b\u7684 jindosdk.log \u6587\u4ef6\u4e2d\uff0c\u5bfb\u627e\u5177\u4f53\u7684\u9519\u8bef\u3002 \u4e0b\u9762\u5c55\u793a\u7684\u8fd9\u4e2a\u9519\u8bef\u5c31\u6765\u81ea jindosdk.log \uff0c\u6839\u636e\u62a5\u9519\u4fe1\u606f\u53ef\u89c1\uff0c\u8fd9\u4e2a\u9519\u8bef\u662f\u5e38\u89c1\u7684\u9274\u6743\u95ee\u9898\u3002 EMMDD HH:mm:ss jindofs_connectivity.cpp:13] Please check your Endpoint/Bucket/RoleArn. Failed test connectivity, operation: mkdir, errMsg: [RequestId]: 618B8183343EA53531C62B74 [HostId]: oss-cn-shanghai-internal.aliyuncs.com [ErrorMessage]: [E1010]HTTP/1.1 403 Forbidden ...","title":"\u4f7f\u7528 JindoFuse \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#jindofuse-oss-hdfs-jindofs","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u901a\u8fc7 JindoFuse \u63d0\u4f9b POSIX \u652f\u6301\u3002JindoFuse \u53ef\u4ee5\u628a OSS-HDFS \u670d\u52a1\u4e0a\u7684\u6587\u4ef6\u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c\u8be5\u670d\u52a1\u4e0a\u7684\u6587\u4ef6\u3002","title":"\u4f7f\u7528 JindoFuse \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_1","text":"","title":"\u57fa\u672c\u4f7f\u7528"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_2","text":"# CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev","title":"\u73af\u5883\u51c6\u5907"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_3","text":"","title":"\u914d\u7f6e\u5ba2\u6237\u7aef"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_4","text":"\u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf","title":"\u914d\u7f6e\u76ee\u5f55"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_5","text":"\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u5f00\u542fHDFS\u670d\u52a1\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret>","title":"\u914d\u7f6e\u6587\u4ef6"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_6","text":"\u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7\u89d2\u8272\u6388\u6743\u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u5f00\u542fHDFS\u670d\u52a1\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE fs.oss.provider.format = JSON","title":"\u514d\u5bc6\u8bbf\u95ee"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#jindofuse","text":"\u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002","title":"\u6302\u8f7d JindoFuse"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_7","text":"mkdir -p <mount_point>","title":"\u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#fuse","text":"jindo-fuse <mount_point> -ouri=[<oss_path>] -ouri \u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684 oss \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a Bucket \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002","title":"\u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#jindofuse_1","text":"\u5982\u679c\u5c06 OSS-HDFS \u670d\u52a1\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1 ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/","title":"\u8bbf\u95ee JindoFuse"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#jindofuse_2","text":"\u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point>","title":"\u5378\u8f7d JindoFuse"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#jindofuse_3","text":"\u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002","title":"\u81ea\u52a8\u5378\u8f7d JindoFuse"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_8","text":"\u76ee\u524d JindoFuse \u5df2\u7ecf\u652f\u6301\u4ee5\u4e0b POSIX API\uff1a \u7279\u6027 \u8bf4\u660e getattr() \u67e5\u8be2\u6587\u4ef6\u5c5e\u6027\uff0c\u7c7b\u4f3c ls mkdir() \u521b\u5efa\u76ee\u5f55\uff0c\u7c7b\u4f3c mkdir rmdir() \u5220\u9664\u76ee\u5f55\uff0c\u7c7b\u4f3c rm -rf unlink() \u5220\u9664\u6587\u4ef6\uff0c\u7c7b\u4f3c unlink rename() \u91cd\u547d\u540d\uff0c\u7c7b\u4f3c mv read() \u987a\u5e8f\u8bfb\u53d6 pread() \u968f\u673a\u8bfb write() \u987a\u5e8f\u5199 pwrite() \u968f\u673a\u5199 flush() \u5237\u65b0\u5185\u5b58\u5230\u5185\u6838\u7f13\u51b2\u533a fsync() \u5237\u65b0\u5185\u5b58\u5230\u78c1\u76d8 release() \u5173\u95ed\u6587\u4ef6 readdir() \u8bfb\u53d6\u76ee\u5f55 create() \u521b\u5efa\u6587\u4ef6 open() O_APPEND \u901a\u8fc7\u8ffd\u52a0\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 open() O_TRUNC \u901a\u8fc7\u8986\u76d6\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 ftruncate() \u5bf9\u6253\u5f00\u7684\u6587\u4ef6\u8fdb\u884c\u622a\u65ad truncate() \u5bf9\u672a\u6253\u5f00\u7684\u6587\u4ef6\u8fdb\u884c\u622a\u65ad\uff0c\u7c7b\u4f3c truncate -s lseek() \u6307\u5b9a\u6253\u5f00\u6587\u4ef6\u4e2d\u7684\u8bfb\u5199\u4f4d\u7f6e\u3002 chmod() \u4fee\u6539\u6587\u4ef6\u6743\u9650\uff0c\u7c7b\u4f3c chmod access() \u67e5\u8be2\u6587\u4ef6\u6743\u9650 utimes() \u4fee\u6539\u6587\u4ef6\u7684\u5b58\u53d6\u65f6\u95f4\u548c\u66f4\u6539\u65f6\u95f4(\u4ec5\u652f\u6301\u4fee\u6539\u4e3a\u66f4\u65b0\u7684\u65f6\u95f4) setxattr() \u4fee\u6539\u6587\u4ef6 xattr \u5c5e\u6027 getxattr() \u83b7\u53d6\u6587\u4ef6 xattr \u5c5e\u6027 listxattr() \u5217\u4e3e\u6587\u4ef6 xattr \u5c5e\u6027 removexattr() \u5220\u9664\u6587\u4ef6 xattr \u5c5e\u6027 lock() \u652f\u6301 posix \u9501\uff0c\u7c7b\u4f3c fcntl fallocate() \u4e3a\u6587\u4ef6\u9884\u5206\u914d\u7269\u7406\u7a7a\u95f4 symlink() \u521b\u5efa\u8f6f\u8fde\u63a5\uff0c\u76ee\u524d\u4ec5\u652f\u6301 OSS-HDFS \u5185\u90e8\u4f7f\u7528\uff0c\u4e14\u4e0d\u652f\u6301\u7f13\u5b58\u52a0\u901f readlink() \u8bfb\u53d6\u8f6f\u8fde\u63a5","title":"\u7279\u6027\u652f\u6301"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_9","text":"","title":"\u9ad8\u9636\u4f7f\u7528"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_10","text":"\u53c2\u6570\u540d\u79f0 \u5fc5\u9009 \u53c2\u6570\u8bf4\u660e \u4f7f\u7528\u8303\u4f8b uri \u2713 \u914d\u7f6e\u9700\u8981\u6620\u5c04\u7684 oss \u8def\u5f84\u3002\u8def\u5f84\u53ef\u4ee5\u662f\u6839\u76ee\u5f55\uff0c\u4e5f\u53ef\u4ee5\u662f\u5b50\u76ee\u5f55\u3002\u4f8b\u5982\uff1aoss://bucket.endpoint/ \u6216 oss://bucket.endpoint/subdir\u3002 -ouri=oss://examplebucket.cn-beijing.oss-dls.aliyuncs.com/ f \u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u9ed8\u8ba4\u4f7f\u7528\u5b88\u62a4\u8fdb\u7a0b\u65b9\u5f0f\u540e\u53f0\u542f\u52a8\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -f d \u4f7f\u7528 Debug \u6a21\u5f0f\uff0c\u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -d auto_unmount fuse\u8fdb\u7a0b\u9000\u51fa\u540e\u81ea\u52a8umount\u6302\u8f7d\u8282\u70b9\u3002 -oauto_unmount ro \u53ea\u8bfb\u6302\u8f7d\uff0c\u542f\u7528\u53c2\u6570\u540e\u4e0d\u5141\u8bb8\u5199\u64cd\u4f5c\u3002 -oro direct_io \u5f00\u542f\u540e\uff0c\u8bfb\u5199\u6587\u4ef6\u53ef\u4ee5\u7ed5\u8fc7page cache\u3002 -odirect_io kernel_cache \u5f00\u542f\u540e\uff0c\u5229\u7528\u5185\u6838\u7f13\u5b58\u4f18\u5316\u8bfb\u6027\u80fd\u3002 -okernel_cache auto_cache \u9ed8\u8ba4\u5f00\u542f\uff0c\u4e0ekernel_cache \u4e8c\u9009\u4e00\uff0c\u4e0ekernel_cache\u4e0d\u540c\u7684\u662f\uff0c\u5982\u679c\u6587\u4ef6\u5927\u5c0f\u6216\u4fee\u6539\u65f6\u95f4\u53d1\u751f\u53d8\u5316\uff0c\u7f13\u5b58\u5c31\u4f1a\u5931\u6548\u3002 entry_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u76ee\u5f55\u6761\u76ee\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oentry_timeout=60 attr_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oattr_timeout=60 negative_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u5931\u8d25\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -onegative_timeout=0 jindo_entry_size \u9ed8\u8ba4\u503c\uff0c5000\u3002\u76ee\u5f55\u6761\u76ee\u7f13\u5b58\u6570\u91cf\uff0c\u7528\u4e8e\u4f18\u5316readdir\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -ojindo_entry_size=5000 jindo_attr_size \u9ed8\u8ba4\u503c\uff0c50000\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u6570\u91cf\uff0c\u7528\u4e8e\u4f18\u5316getattr\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -ojindo_attr_sizet=50000 max_idle_threads \u9ed8\u8ba4\u503c\uff0c10\u3002\u6700\u5927\u7a7a\u95f2\u7ebf\u7a0b\u6570\u3002 -omax_idle_threads=10 metrics_port \u9ed8\u8ba4\u503c\uff0c9090\u3002\u5f00\u542fhttp\u7aef\u53e3\uff0c\u8f93\u51fametrics\uff0c\u5982http://localhost:9090/brpc_metrics -ometrics_port=9090 enable_pread \u5f00\u542f\u540e\uff0c\u4f7f\u7528pread\u63a5\u53e3\u8bfb\u53d6\u6587\u4ef6\u3002 -oenable_pread","title":"\u6302\u8f7d\u9009\u9879"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_11","text":"\u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u914d\u7f6e\u8282 \u8bf4\u660e logger.dir /tmp/fuse-log common \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false common \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false common \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\u3002 \u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002 logger.verbose 0 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false common \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u5730\u5740\uff0c\u5982cn-xxx.oss-dls.aliyuncs.com fs.oss.accessKeyId jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeySecret \u66f4\u591a\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<oss_path>] -ofs.oss.endpoint=[<your_endpoint>] -ofs.oss.accessKeyId=[<your_key_id>] -ofs.oss.accessKeySecret=[<your_key_secret>]","title":"\u914d\u7f6e\u9009\u9879"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#_12","text":"","title":"\u5e38\u89c1\u95ee\u9898"},{"location":"backup/jindofs/jindo_fuse/jindo_fuse_on_dls/#inputoutput-error","text":"\u4e0d\u50cf\u4f7f\u7528 JindoSDK \u8c03\u7528 API \u53ef\u4ee5\u83b7\u53d6\u66f4\u4e3a\u5177\u4f53\u7684 ErrorMsg\uff0cJindoFuse \u53ea\u80fd\u663e\u793a\u64cd\u4f5c\u7cfb\u7edf\u9884\u8bbe\u7684\u9519\u8bef\u4fe1\u606f\uff0c\u6bd4\u5982\u4ee5\u4e0b\u9519\u8bef\u5c31\u975e\u5e38\u5e38\u89c1\uff1a $ ls /mnt/oss/ ls: /mnt/oss/: Input/output error \u5982\u679c\u9700\u8981\u5b9a\u4f4d\u5177\u4f53\u7684\u9519\u8bef\u539f\u56e0\uff0c\u53ef\u4ee5\u6839\u636e JindoSDK \u914d\u7f6e\u4e2d\u7684 logger.dir \uff0c\u5728\u6307\u5b9a\u8def\u5f84\u4e0b\u7684 jindosdk.log \u6587\u4ef6\u4e2d\uff0c\u5bfb\u627e\u5177\u4f53\u7684\u9519\u8bef\u3002 \u4e0b\u9762\u5c55\u793a\u7684\u8fd9\u4e2a\u9519\u8bef\u5c31\u6765\u81ea jindosdk.log \uff0c\u6839\u636e\u62a5\u9519\u4fe1\u606f\u53ef\u89c1\uff0c\u8fd9\u4e2a\u9519\u8bef\u662f\u5e38\u89c1\u7684\u9274\u6743\u95ee\u9898\u3002 EMMDD HH:mm:ss jindofs_connectivity.cpp:13] Please check your Endpoint/Bucket/RoleArn. Failed test connectivity, operation: mkdir, errMsg: [RequestId]: 618B8183343EA53531C62B74 [HostId]: oss-cn-shanghai-internal.aliyuncs.com [ErrorMessage]: [E1010]HTTP/1.1 403 Forbidden ...","title":"Input/Output error"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/","text":"\u4f7f\u7528 Jindo Tensorflow Collector \u8bbf\u95ee OSS-HDFS \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u901a\u8fc7 Jindo Tensorflow Collector \u63d0\u4f9b\u4f7f\u7528 tensorflow gfile \u8bbf\u95ee OSS-HDFS \u7684\u652f\u6301\u3002 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u4fee\u6539\u914d\u7f6e \u00a7 2.1 \u5b89\u88c5 tensorflow \u00a7 \u76ee\u524d\u4ec5\u652f\u6301 tensorflow 1.15 \u6216 tensorflow 2.8 \u7248\u672c\uff0c\u4ee5\u4e0b\u4ee5 tensorflow 2.8 \u4e3a\u4f8b\u3002 pip3 install tensorflow==2.8 2.2 \u6dfb\u52a0\u73af\u5883\u53d8\u91cf \u00a7 2.2.1 JindoSDK \u73af\u5883\u53d8\u91cf \u00a7 export LD_LIBRARY_PATH=\"/path/to/jindosdk-x.x.x/lib/native\":$LD_LIBRARY_PATH export JINDOSDK_CONF_DIR=/path/to/jindosdk-x.x.x/conf 2.2.2 \u4f9d\u8d56\u7684 Tensorflow \u73af\u5883\u53d8\u91cf \u00a7 \u540c\u65f6\uff0cLD_LIBRARY_PATH \u9700\u8981\u6dfb\u52a0 tensorflow \u6240\u5728\u7684\u4f9d\u8d56\u8def\u5f84\uff0c\u4e0d\u540c\u7684 python \u73af\u5883\u5b89\u88c5\u8def\u5f84\u53ef\u80fd\u4e0d\u540c\uff0c\u5982\uff1a # tensorflow1.15 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow_core/\":$LD_LIBRARY_PATH # tensorflow2.8 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow/\":$LD_LIBRARY_PATH 2.3 \u914d\u7f6e\u6587\u4ef6 \u00a7 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \u3002\u53ef\u4ee5\u9009\u62e9\u56fa\u5b9a\u6216\u8005\u514d\u5bc6\u4e24\u79cd\u914d\u7f6e\u65b9\u5f0f\u3002 2.3.1 \u56fa\u5b9a AK \u8bbf\u95ee \u00a7 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/tfio-log [jindosdk] # \u5df2\u5f00\u542fHDFS\u670d\u52a1\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> \u914d\u7f6e\u9009\u9879 \u00a7 \u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u914d\u7f6e\u8282 \u8bf4\u660e logger.dir /tmp/tfio-log common \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false common \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false common \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\u3002 \u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002 logger.verbose 0 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false common \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u5730\u5740\uff0c\u5982cn-xxx.oss-dls.aliyuncs.com fs.oss.accessKeyId jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeySecret \u66f4\u591a\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 2.3.2 \u514d\u5bc6\u8bbf\u95ee \u00a7 \u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7\u89d2\u8272\u6388\u6743\u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u5f00\u542fHDFS\u670d\u52a1\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE fs.oss.provider.format = JSON 3. \u4f7f\u7528 Jindo Tensorflow \u00a7 \u4ee5\u52a0\u8f7d jindo-tensorflow2.8 \u4e3a\u4f8b 3.1 \u901a\u8fc7 load_library \u6dfb\u52a0 collector \u00a7 import tensorflow as tf tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\") 3.2 \u901a\u8fc7 gfile \u8bbf\u95ee oss-hdfs \u00a7 \u4e3a\u907f\u514d\u4e0e tfio \u4e2d\u7684 oss \u524d\u7f00\u51b2\u7a81\uff0cjindo-tensorflow \u4f7f\u7528 joss \u524d\u7f00\u3002 # \u76ee\u5f55\u64cd\u4f5c tf.io.gfile.mkdir(\"joss://<bucket>.<endpoint>/test_mkdir\") content = tf.io.gfile.listdir(\"joss://<bucket>.<endpoint>/test_mkdir\") # \u8bfb gmnist = tf.io.gfile.GFile(\"joss://<bucket>.<endpoint>/mnist.npz\", \"rb\") gmnist_file = gmnist.read() # \u5199 fh = gfile.Open(f, mode=\"w\") content = \"file content\" fh.write(content) fh.close() 3.3 \u8bfb\u53d6 TFRecordDataset \u00a7 tfRecordJfsPath = \"joss://<bucket>.<endpoint>/test.tfrecord\" filenames = [tfRecordJfsPath] raw_dataset = tf.data.TFRecordDataset(filenames) 3.4 \u5b8c\u6574\u793a\u4f8b \u00a7 import tensorflow as tf import numpy as np tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\") gmnist = tf.io.gfile.GFile(\"joss://<bucket>.<endpoint>/mnist.npz\", 'rb') print(gmnist.size()) with np.load(gmnist) as f: x_train, y_train = f['x_train'], f['y_train'] x_test, y_test = f['x_test'], f['y_test'] model = tf.keras.models.Sequential([ tf.keras.layers.Flatten(input_shape=(28, 28)), tf.keras.layers.Dense(128, activation='relu'), tf.keras.layers.Dropout(0.2), tf.keras.layers.Dense(10, activation='softmax') ]) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) model.fit(x_train, y_train, epochs=5)","title":"\u4f7f\u7528 Jindo Tensorflow Collector \u8bbf\u95ee OSS-HDFS"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#jindo-tensorflow-collector-oss-hdfs","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u901a\u8fc7 Jindo Tensorflow Collector \u63d0\u4f9b\u4f7f\u7528 tensorflow gfile \u8bbf\u95ee OSS-HDFS \u7684\u652f\u6301\u3002","title":"\u4f7f\u7528 Jindo Tensorflow Collector \u8bbf\u95ee OSS-HDFS"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#2","text":"","title":"2. \u4fee\u6539\u914d\u7f6e"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#21-tensorflow","text":"\u76ee\u524d\u4ec5\u652f\u6301 tensorflow 1.15 \u6216 tensorflow 2.8 \u7248\u672c\uff0c\u4ee5\u4e0b\u4ee5 tensorflow 2.8 \u4e3a\u4f8b\u3002 pip3 install tensorflow==2.8","title":"2.1 \u5b89\u88c5 tensorflow"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#22","text":"","title":"2.2 \u6dfb\u52a0\u73af\u5883\u53d8\u91cf"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#221-jindosdk","text":"export LD_LIBRARY_PATH=\"/path/to/jindosdk-x.x.x/lib/native\":$LD_LIBRARY_PATH export JINDOSDK_CONF_DIR=/path/to/jindosdk-x.x.x/conf","title":"2.2.1 JindoSDK \u73af\u5883\u53d8\u91cf"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#222-tensorflow","text":"\u540c\u65f6\uff0cLD_LIBRARY_PATH \u9700\u8981\u6dfb\u52a0 tensorflow \u6240\u5728\u7684\u4f9d\u8d56\u8def\u5f84\uff0c\u4e0d\u540c\u7684 python \u73af\u5883\u5b89\u88c5\u8def\u5f84\u53ef\u80fd\u4e0d\u540c\uff0c\u5982\uff1a # tensorflow1.15 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow_core/\":$LD_LIBRARY_PATH # tensorflow2.8 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow/\":$LD_LIBRARY_PATH","title":"2.2.2 \u4f9d\u8d56\u7684 Tensorflow \u73af\u5883\u53d8\u91cf"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#23","text":"\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \u3002\u53ef\u4ee5\u9009\u62e9\u56fa\u5b9a\u6216\u8005\u514d\u5bc6\u4e24\u79cd\u914d\u7f6e\u65b9\u5f0f\u3002","title":"2.3 \u914d\u7f6e\u6587\u4ef6"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#231-ak","text":"\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/tfio-log [jindosdk] # \u5df2\u5f00\u542fHDFS\u670d\u52a1\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret>","title":"2.3.1 \u56fa\u5b9a AK \u8bbf\u95ee"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#_1","text":"\u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u914d\u7f6e\u8282 \u8bf4\u660e logger.dir /tmp/tfio-log common \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false common \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false common \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\u3002 \u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002 logger.verbose 0 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false common \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u5730\u5740\uff0c\u5982cn-xxx.oss-dls.aliyuncs.com fs.oss.accessKeyId jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeySecret \u66f4\u591a\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002","title":"\u914d\u7f6e\u9009\u9879"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#232","text":"\u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7\u89d2\u8272\u6388\u6743\u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u5f00\u542fHDFS\u670d\u52a1\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE fs.oss.provider.format = JSON","title":"2.3.2 \u514d\u5bc6\u8bbf\u95ee"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#3-jindo-tensorflow","text":"\u4ee5\u52a0\u8f7d jindo-tensorflow2.8 \u4e3a\u4f8b","title":"3. \u4f7f\u7528 Jindo Tensorflow"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#31-load_library-collector","text":"import tensorflow as tf tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\")","title":"3.1 \u901a\u8fc7 load_library \u6dfb\u52a0 collector"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#32-gfile-oss-hdfs","text":"\u4e3a\u907f\u514d\u4e0e tfio \u4e2d\u7684 oss \u524d\u7f00\u51b2\u7a81\uff0cjindo-tensorflow \u4f7f\u7528 joss \u524d\u7f00\u3002 # \u76ee\u5f55\u64cd\u4f5c tf.io.gfile.mkdir(\"joss://<bucket>.<endpoint>/test_mkdir\") content = tf.io.gfile.listdir(\"joss://<bucket>.<endpoint>/test_mkdir\") # \u8bfb gmnist = tf.io.gfile.GFile(\"joss://<bucket>.<endpoint>/mnist.npz\", \"rb\") gmnist_file = gmnist.read() # \u5199 fh = gfile.Open(f, mode=\"w\") content = \"file content\" fh.write(content) fh.close()","title":"3.2 \u901a\u8fc7 gfile \u8bbf\u95ee oss-hdfs"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#33-tfrecorddataset","text":"tfRecordJfsPath = \"joss://<bucket>.<endpoint>/test.tfrecord\" filenames = [tfRecordJfsPath] raw_dataset = tf.data.TFRecordDataset(filenames)","title":"3.3 \u8bfb\u53d6 TFRecordDataset"},{"location":"backup/jindofs/jindo_tensorflow/jindo_tensorflow_on_dls/#34","text":"import tensorflow as tf import numpy as np tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\") gmnist = tf.io.gfile.GFile(\"joss://<bucket>.<endpoint>/mnist.npz\", 'rb') print(gmnist.size()) with np.load(gmnist) as f: x_train, y_train = f['x_train'], f['y_train'] x_test, y_test = f['x_test'], f['y_test'] model = tf.keras.models.Sequential([ tf.keras.layers.Flatten(input_shape=(28, 28)), tf.keras.layers.Dense(128, activation='relu'), tf.keras.layers.Dropout(0.2), tf.keras.layers.Dense(10, activation='softmax') ]) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) model.fit(x_train, y_train, epochs=5)","title":"3.4 \u5b8c\u6574\u793a\u4f8b"},{"location":"backup/jindofs/jindotable/table_moveto/","text":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 HDFS \u53d7\u9650\u4e8e\u96c6\u7fa4\u89c4\u6a21\u548c\u6210\u672c\u56e0\u7d20\uff0c\u65e0\u6cd5\u65e0\u9650\u6269\u5c55\uff0c\u5bb9\u91cf\u5b58\u5728\u74f6\u9888\u3002\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\u53ef\u4ee5\u4f5c\u4e3a HDFS \u7684\u66ff\u4ee3\u6216\u8865\u5145\uff0c\u6269\u5c55\u4e91\u4e0a Hadoop \u5e73\u53f0\u7684\u5b58\u50a8\u80fd\u529b\u3002 \u672c\u5de5\u5177\u7528\u4e8e\u628a Hive \u6570\u636e\u6839\u636e\u5206\u533a\u952e\u89c4\u5219\u7b5b\u9009\uff0c\u5728 HDFS \u548c \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\u4e4b\u95f4\u8f6c\u79fb\u5206\u533a\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u90e8\u7f72 JindoSDK \u00a7 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8 \u3002 \u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883 \u00a7 \u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e \u914d\u7f6e MoveTo \u5de5\u5177\u5728 HDFS \u4e0b\u7684\u9501\u76ee\u5f55\uff1a \u00a7 \u5728 Hadoop \u914d\u7f6e\u6587\u4ef6 core-site.xml \u6216 hdfs-site.xml \uff08\u4efb\u4e00\u5373\u53ef\uff0c\u5728 $HADOOP_CONF_DIR \u76ee\u5f55\u4e0b\uff09\u65b0\u589e\u914d\u7f6e\u9879\uff1a jindotable.moveto.tablelock.base.dir \u8be5\u914d\u7f6e\u7684\u503c\u5e94\u6307\u5411\u4e00\u4e2a HDFS \u76ee\u5f55\uff0c\u76ee\u7684\u662f\u5b58\u653e MoveTo \u5de5\u5177\u5728\u8fd0\u884c\u65f6\u81ea\u52a8\u521b\u5efa\u7684\u9501\u6587\u4ef6\u3002\u9700\u786e\u4fdd\u8be5\u76ee\u5f55\u53ea\u4f1a\u88ab MoveTo \u5de5\u5177\u8bbf\u95ee\uff0c\u5e76\u4e14\u6709\u8bbf\u95ee\u6743\u9650\u3002\u5982\u679c\u4e0d\u914d\u7f6e\uff0c\u5219\u4f7f\u7528\u7f3a\u7701\u503c hdfs:///tmp/jindotable-lock/ \uff0c\u65e0\u6743\u9650\u5219\u62a5\u9519\u3002 \u4f7f\u7528\u8bf4\u660e \u00a7 \u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f \u00a7 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help moveTo \u53c2\u6570\u8bf4\u660e \u00a7 jindotable -moveTo -t <dbName.tableName> -d <destination path> [-c \"<condition>\" | -fullTable] [-b/-before <before days>] [-p/-parallel <parallelism>] [-s/-storagePolicy <OSS storage policy>] [-o/-overWrite] [-r/-removeSource] [-skipTrash] [-e/-explain] [-q/-queue <yarn queue>] [-w/-workingDir <working directory>][-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u8981\u79fb\u52a8\u7684\u8868\u3002 \u662f -d <destination path> \u76ee\u6807\u8def\u5f84\uff0c\u4e3a\u8868\u7ea7\u522b\u7684\u8def\u5f84\uff0c\u5206\u533a\u8def\u5f84\u4f1a\u5728\u8fd9\u4e2a\u8def\u5f84\u4e0b\u81ea\u52a8\u521b\u5efa\u3002 \u662f -c \"<condition>\" / -fullTable \u5206\u533a\u8fc7\u6ee4\u6761\u4ef6\u8868\u8fbe\u5f0f\uff0c\u652f\u6301\u57fa\u672c\u8fd0\u7b97\u7b26\uff0c\u4e0d\u652f\u6301udf\u3002 \u5426 -b/before <before days> \u6839\u636e\u5206\u533a\u521b\u5efa\u65f6\u95f4\uff0c\u521b\u5efa\u65f6\u95f4\u8d85\u8fc7\u7ed9\u5b9a\u5929\u6570\u7684\u5206\u533a\u624d\u8fdb\u884c\u79fb\u52a8\u3002 \u5426 -p/-parallel <parallelism> \u6574\u4e2amoveTo\u4efb\u52a1\u7684\u6700\u5927task\u5e76\u53d1\u5ea6\uff0c\u9ed8\u8ba4\u4e3a1\u3002 \u5426 -s/-storagePolicy <OSS storage policy> OSS-HDFS \u670d\u52a1\u4e0d\u652f\u6301\u8be5\u53c2\u6570 \u5426 -o/-overWrite \u662f\u5426\u8986\u76d6\u6700\u7ec8\u76ee\u5f55\u3002\u5206\u533a\u7ea7\u522b\u8986\u76d6\uff0c\u4e0d\u4f1a\u8986\u76d6\u672c\u6b21\u79fb\u52a8\u4e0d\u6d89\u53ca\u7684\u5206\u533a\u3002 \u5426 -r/-removeSource \u662f\u5426\u5728\u79fb\u52a8\u5b8c\u6210\u540e\u5220\u9664\u6e90\u8def\u5f84\u3002 \u5426 -skipTrash \u5982\u679c\u5220\u9664\u6e90\u8def\u5f84\uff0c\u662f\u5426\u8df3\u8fc7Trash\u3002 \u5426 -e/-explain \u5982\u679c\u6307\u5b9aexplain\u6a21\u5f0f\uff0c\u4e0d\u4f1a\u89e6\u53d1\u5b9e\u9645\u64cd\u4f5c\uff0c\u4ec5\u6253\u5370\u4f1a\u540c\u6b65\u7684\u5206\u533a\u3002 \u5426 -q/-queue <yarn queue \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684yarn\u961f\u5217\u3002 \u5426 -w/-workingDir \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684\u5de5\u4f5c\u4e34\u65f6\u76ee\u5f55\u3002 \u5426 -l/-logDir <log directory> \u672c\u5730\u65e5\u5fd7\u76ee\u5f55\uff0c\u9ed8\u8ba4\u4e3a /tmp/<current user>/ \u5426 \u4f7f\u7528\u793a\u4f8b \u00a7 \u6709\u4e00\u4e2a HDFS \u4e0a\u7684 Hive \u5206\u533a\u8868\uff0c\u5982\u4e0b\u56fe\u6240\u793a\uff1a \u60f3\u628a bbb \u548c ccc \u5206\u533a\u79fb\u52a8\u5230 OSS-HDFS \u670d\u52a1\uff0c\u5148\u7528 explain \u6a21\u5f0f\u770b\u770b\u4f1a\u79fb\u52a8\u7684\u5206\u533a\u662f\u5426\u7b26\u5408\u9884\u671f\uff0c\u53c2\u6570\u4e3a -e \u6216 -explain\uff1a \u53bb\u6389\u53c2\u6570 -e\uff0c\u771f\u6b63\u79fb\u52a8\u5206\u533a\uff1a \u6267\u884c\u5b8c\u6210\u540e\uff0c\u68c0\u67e5\u6570\u636e\uff0c\u6570\u636e\u5df2\u7ecf\u5728 OSS-HDFS \u670d\u52a1\uff1a \u518d\u79fb\u56de HDFS\uff0c\u7ed3\u679c\u5931\u8d25\u4e86\uff0c\u4f1a\u663e\u793a\u5931\u8d25\u539f\u56e0\uff0c\u539f\u6765\u662f\u76ee\u6807\u76ee\u5f55\u975e\u7a7a\uff0c\u63d0\u793a\u53ef\u4ee5\u4f7f\u7528 -overWrite \u6e05\u7a7a\u76ee\u6807\u76ee\u5f55\uff1a \u4f7f\u7528 -overWrite \u5f3a\u5236\u8986\u76d6\uff1a \u5f02\u5e38\u72b6\u6001\u5904\u7406 \u00a7 \u4e3a\u4e86\u5c3d\u53ef\u80fd\u4fdd\u8bc1\u6570\u636e\u5b89\u5168\uff0c\u9632\u6b62\u6c61\u67d3\uff0c\u8be5\u547d\u4ee4\u4f1a\u81ea\u52a8\u68c0\u67e5\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u786e\u4fdd\u4e0d\u5b58\u5728\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u76ee\u5f55\u62f7\u8d1d\u3002 \u5982\u679c\u65e0\u6cd5\u6392\u9664\uff0c\u5219\u8be5\u8868\u6216\u5206\u533a\u7684\u62f7\u8d1d\u547d\u4ee4\u5c06\u6267\u884c\u5931\u8d25\u3002 \u6b64\u65f6\uff0c\u9700\u8981\u4e3b\u52a8\u5e72\u9884\uff0c\u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u6b63\u5728\u6267\u884c\u62f7\u8d1d\u4e4b\u540e\uff0c\u6e05\u7406\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u5219\u53ef\u4ee5\u91cd\u65b0\u6267\u884c\u8be5\u547d\u4ee4\u3002 \u5bf9\u4e8e\u975e\u5206\u533a\u8868\u7684\u62f7\u8d1d\u6216\u8fc1\u79fb\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u5373\u4e3a\u8868\u7ea7\u522b\u76ee\u5f55\uff1b\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u4e3a\u5f85\u62f7\u8d1d\u6216\u79fb\u52a8\u5206\u533a\u7684\u5206\u533a\u7ea7\u76ee\u5f55\uff0c \u53ea\u9700\u5bf9\u5f85\u62f7\u8d1d\u7684\u5206\u533a\u8fdb\u884c\u6e05\u7406\u3002 \u51fa\u73b0\u539f\u56e0 \u00a7 \u5982\u679c\u8be5\u547d\u4ee4\u88ab\u5f02\u5e38\u4e2d\u6b62\uff0c\u5219\u53ef\u80fd\u51fa\u73b0\u9700\u8981\u4e3b\u52a8\u5e72\u9884\u7684\u60c5\u51b5\u3002\u6b64\u65f6\u62f7\u8d1d\u8fd8\u672a\u5b8c\u6210\uff0c\u6e90\u6570\u636e\u4e0e\u8868\u7684\u5143\u4fe1\u606f\u5747\u672a\u6539\u53d8\uff0c \u6570\u636e\u4ecd\u5904\u4e8e\u5b89\u5168\u72b6\u6001\u3002\u5e38\u89c1\u7684\u5f02\u5e38\u4e2d\u6b62\u6709\u4e24\u79cd\u60c5\u51b5\uff1a * \u7528\u6237\u5728\u547d\u4ee4\u5c1a\u672a\u7ed3\u675f\u65f6\uff0c\u4e3b\u52a8\u6740\u6389\u547d\u4ee4\u8fdb\u7a0b * \u7531\u4e8e\u5185\u5b58\u6ea2\u51fa\u7b49\u5f02\u5e38\uff0c\u8fdb\u7a0b\u5f02\u5e38\u4e2d\u6b62 \u5982\u679c\u62f7\u8d1d\u8868\u6216\u5206\u533a\u65f6\u62f7\u8d1d\u5931\u8d25\u5e76\u663e\u793a Conflicts found \uff0c\u5219\u5c5e\u4e8e\u5f02\u5e38\u72b6\u6001\uff0c\u9700\u4e3b\u52a8\u5e72\u9884\u3002 \u5904\u7406\u65b9\u6cd5 \u00a7 \u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u7684\u76ee\u6807\u8def\u5f84\u62f7\u8d1d\u6570\u636e\u3002\u7279\u522b\u662f\uff0c\u786e\u4fdd\u6ca1\u6709 DistCp/JindoDistCp \u7b49\u5206\u5e03\u5f0f\u62f7\u8d1d\u547d\u4ee4\u3002 \u5220\u9664\u76ee\u7684\u5730\u76ee\u5f55\u3002\u5bf9\u4e8e\u975e\u5206\u533a\u8868\uff0c\u5220\u9664\u8868\u4e00\u7ea7\u76ee\u5f55\u3002\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u5220\u9664\u5b58\u5728\u51b2\u7a81\u7684\u5206\u533a\u7ea7\u76ee\u5f55\u3002\u5207\u52ff\u5220\u9664\u6e90\u76ee\u5f55\u3002","title":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/jindotable/table_moveto/#jindotable-hive-oss-hdfs-jindofs","text":"HDFS \u53d7\u9650\u4e8e\u96c6\u7fa4\u89c4\u6a21\u548c\u6210\u672c\u56e0\u7d20\uff0c\u65e0\u6cd5\u65e0\u9650\u6269\u5c55\uff0c\u5bb9\u91cf\u5b58\u5728\u74f6\u9888\u3002\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\u53ef\u4ee5\u4f5c\u4e3a HDFS \u7684\u66ff\u4ee3\u6216\u8865\u5145\uff0c\u6269\u5c55\u4e91\u4e0a Hadoop \u5e73\u53f0\u7684\u5b58\u50a8\u80fd\u529b\u3002 \u672c\u5de5\u5177\u7528\u4e8e\u628a Hive \u6570\u636e\u6839\u636e\u5206\u533a\u952e\u89c4\u5219\u7b5b\u9009\uff0c\u5728 HDFS \u548c \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\u4e4b\u95f4\u8f6c\u79fb\u5206\u533a\u3002","title":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/jindotable/table_moveto/#_1","text":"","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/jindofs/jindotable/table_moveto/#jindosdk","text":"\u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8 \u3002","title":"\u5df2\u90e8\u7f72 JindoSDK"},{"location":"backup/jindofs/jindotable/table_moveto/#hadoop-hive","text":"\u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e","title":"\u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883"},{"location":"backup/jindofs/jindotable/table_moveto/#moveto-hdfs","text":"\u5728 Hadoop \u914d\u7f6e\u6587\u4ef6 core-site.xml \u6216 hdfs-site.xml \uff08\u4efb\u4e00\u5373\u53ef\uff0c\u5728 $HADOOP_CONF_DIR \u76ee\u5f55\u4e0b\uff09\u65b0\u589e\u914d\u7f6e\u9879\uff1a jindotable.moveto.tablelock.base.dir \u8be5\u914d\u7f6e\u7684\u503c\u5e94\u6307\u5411\u4e00\u4e2a HDFS \u76ee\u5f55\uff0c\u76ee\u7684\u662f\u5b58\u653e MoveTo \u5de5\u5177\u5728\u8fd0\u884c\u65f6\u81ea\u52a8\u521b\u5efa\u7684\u9501\u6587\u4ef6\u3002\u9700\u786e\u4fdd\u8be5\u76ee\u5f55\u53ea\u4f1a\u88ab MoveTo \u5de5\u5177\u8bbf\u95ee\uff0c\u5e76\u4e14\u6709\u8bbf\u95ee\u6743\u9650\u3002\u5982\u679c\u4e0d\u914d\u7f6e\uff0c\u5219\u4f7f\u7528\u7f3a\u7701\u503c hdfs:///tmp/jindotable-lock/ \uff0c\u65e0\u6743\u9650\u5219\u62a5\u9519\u3002","title":"\u914d\u7f6e MoveTo \u5de5\u5177\u5728 HDFS \u4e0b\u7684\u9501\u76ee\u5f55\uff1a"},{"location":"backup/jindofs/jindotable/table_moveto/#_2","text":"","title":"\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindotable/table_moveto/#_3","text":"\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help moveTo","title":"\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f"},{"location":"backup/jindofs/jindotable/table_moveto/#_4","text":"jindotable -moveTo -t <dbName.tableName> -d <destination path> [-c \"<condition>\" | -fullTable] [-b/-before <before days>] [-p/-parallel <parallelism>] [-s/-storagePolicy <OSS storage policy>] [-o/-overWrite] [-r/-removeSource] [-skipTrash] [-e/-explain] [-q/-queue <yarn queue>] [-w/-workingDir <working directory>][-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u8981\u79fb\u52a8\u7684\u8868\u3002 \u662f -d <destination path> \u76ee\u6807\u8def\u5f84\uff0c\u4e3a\u8868\u7ea7\u522b\u7684\u8def\u5f84\uff0c\u5206\u533a\u8def\u5f84\u4f1a\u5728\u8fd9\u4e2a\u8def\u5f84\u4e0b\u81ea\u52a8\u521b\u5efa\u3002 \u662f -c \"<condition>\" / -fullTable \u5206\u533a\u8fc7\u6ee4\u6761\u4ef6\u8868\u8fbe\u5f0f\uff0c\u652f\u6301\u57fa\u672c\u8fd0\u7b97\u7b26\uff0c\u4e0d\u652f\u6301udf\u3002 \u5426 -b/before <before days> \u6839\u636e\u5206\u533a\u521b\u5efa\u65f6\u95f4\uff0c\u521b\u5efa\u65f6\u95f4\u8d85\u8fc7\u7ed9\u5b9a\u5929\u6570\u7684\u5206\u533a\u624d\u8fdb\u884c\u79fb\u52a8\u3002 \u5426 -p/-parallel <parallelism> \u6574\u4e2amoveTo\u4efb\u52a1\u7684\u6700\u5927task\u5e76\u53d1\u5ea6\uff0c\u9ed8\u8ba4\u4e3a1\u3002 \u5426 -s/-storagePolicy <OSS storage policy> OSS-HDFS \u670d\u52a1\u4e0d\u652f\u6301\u8be5\u53c2\u6570 \u5426 -o/-overWrite \u662f\u5426\u8986\u76d6\u6700\u7ec8\u76ee\u5f55\u3002\u5206\u533a\u7ea7\u522b\u8986\u76d6\uff0c\u4e0d\u4f1a\u8986\u76d6\u672c\u6b21\u79fb\u52a8\u4e0d\u6d89\u53ca\u7684\u5206\u533a\u3002 \u5426 -r/-removeSource \u662f\u5426\u5728\u79fb\u52a8\u5b8c\u6210\u540e\u5220\u9664\u6e90\u8def\u5f84\u3002 \u5426 -skipTrash \u5982\u679c\u5220\u9664\u6e90\u8def\u5f84\uff0c\u662f\u5426\u8df3\u8fc7Trash\u3002 \u5426 -e/-explain \u5982\u679c\u6307\u5b9aexplain\u6a21\u5f0f\uff0c\u4e0d\u4f1a\u89e6\u53d1\u5b9e\u9645\u64cd\u4f5c\uff0c\u4ec5\u6253\u5370\u4f1a\u540c\u6b65\u7684\u5206\u533a\u3002 \u5426 -q/-queue <yarn queue \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684yarn\u961f\u5217\u3002 \u5426 -w/-workingDir \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684\u5de5\u4f5c\u4e34\u65f6\u76ee\u5f55\u3002 \u5426 -l/-logDir <log directory> \u672c\u5730\u65e5\u5fd7\u76ee\u5f55\uff0c\u9ed8\u8ba4\u4e3a /tmp/<current user>/ \u5426","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"backup/jindofs/jindotable/table_moveto/#_5","text":"\u6709\u4e00\u4e2a HDFS \u4e0a\u7684 Hive \u5206\u533a\u8868\uff0c\u5982\u4e0b\u56fe\u6240\u793a\uff1a \u60f3\u628a bbb \u548c ccc \u5206\u533a\u79fb\u52a8\u5230 OSS-HDFS \u670d\u52a1\uff0c\u5148\u7528 explain \u6a21\u5f0f\u770b\u770b\u4f1a\u79fb\u52a8\u7684\u5206\u533a\u662f\u5426\u7b26\u5408\u9884\u671f\uff0c\u53c2\u6570\u4e3a -e \u6216 -explain\uff1a \u53bb\u6389\u53c2\u6570 -e\uff0c\u771f\u6b63\u79fb\u52a8\u5206\u533a\uff1a \u6267\u884c\u5b8c\u6210\u540e\uff0c\u68c0\u67e5\u6570\u636e\uff0c\u6570\u636e\u5df2\u7ecf\u5728 OSS-HDFS \u670d\u52a1\uff1a \u518d\u79fb\u56de HDFS\uff0c\u7ed3\u679c\u5931\u8d25\u4e86\uff0c\u4f1a\u663e\u793a\u5931\u8d25\u539f\u56e0\uff0c\u539f\u6765\u662f\u76ee\u6807\u76ee\u5f55\u975e\u7a7a\uff0c\u63d0\u793a\u53ef\u4ee5\u4f7f\u7528 -overWrite \u6e05\u7a7a\u76ee\u6807\u76ee\u5f55\uff1a \u4f7f\u7528 -overWrite \u5f3a\u5236\u8986\u76d6\uff1a","title":"\u4f7f\u7528\u793a\u4f8b"},{"location":"backup/jindofs/jindotable/table_moveto/#_6","text":"\u4e3a\u4e86\u5c3d\u53ef\u80fd\u4fdd\u8bc1\u6570\u636e\u5b89\u5168\uff0c\u9632\u6b62\u6c61\u67d3\uff0c\u8be5\u547d\u4ee4\u4f1a\u81ea\u52a8\u68c0\u67e5\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u786e\u4fdd\u4e0d\u5b58\u5728\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u76ee\u5f55\u62f7\u8d1d\u3002 \u5982\u679c\u65e0\u6cd5\u6392\u9664\uff0c\u5219\u8be5\u8868\u6216\u5206\u533a\u7684\u62f7\u8d1d\u547d\u4ee4\u5c06\u6267\u884c\u5931\u8d25\u3002 \u6b64\u65f6\uff0c\u9700\u8981\u4e3b\u52a8\u5e72\u9884\uff0c\u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u6b63\u5728\u6267\u884c\u62f7\u8d1d\u4e4b\u540e\uff0c\u6e05\u7406\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u5219\u53ef\u4ee5\u91cd\u65b0\u6267\u884c\u8be5\u547d\u4ee4\u3002 \u5bf9\u4e8e\u975e\u5206\u533a\u8868\u7684\u62f7\u8d1d\u6216\u8fc1\u79fb\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u5373\u4e3a\u8868\u7ea7\u522b\u76ee\u5f55\uff1b\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u4e3a\u5f85\u62f7\u8d1d\u6216\u79fb\u52a8\u5206\u533a\u7684\u5206\u533a\u7ea7\u76ee\u5f55\uff0c \u53ea\u9700\u5bf9\u5f85\u62f7\u8d1d\u7684\u5206\u533a\u8fdb\u884c\u6e05\u7406\u3002","title":"\u5f02\u5e38\u72b6\u6001\u5904\u7406"},{"location":"backup/jindofs/jindotable/table_moveto/#_7","text":"\u5982\u679c\u8be5\u547d\u4ee4\u88ab\u5f02\u5e38\u4e2d\u6b62\uff0c\u5219\u53ef\u80fd\u51fa\u73b0\u9700\u8981\u4e3b\u52a8\u5e72\u9884\u7684\u60c5\u51b5\u3002\u6b64\u65f6\u62f7\u8d1d\u8fd8\u672a\u5b8c\u6210\uff0c\u6e90\u6570\u636e\u4e0e\u8868\u7684\u5143\u4fe1\u606f\u5747\u672a\u6539\u53d8\uff0c \u6570\u636e\u4ecd\u5904\u4e8e\u5b89\u5168\u72b6\u6001\u3002\u5e38\u89c1\u7684\u5f02\u5e38\u4e2d\u6b62\u6709\u4e24\u79cd\u60c5\u51b5\uff1a * \u7528\u6237\u5728\u547d\u4ee4\u5c1a\u672a\u7ed3\u675f\u65f6\uff0c\u4e3b\u52a8\u6740\u6389\u547d\u4ee4\u8fdb\u7a0b * \u7531\u4e8e\u5185\u5b58\u6ea2\u51fa\u7b49\u5f02\u5e38\uff0c\u8fdb\u7a0b\u5f02\u5e38\u4e2d\u6b62 \u5982\u679c\u62f7\u8d1d\u8868\u6216\u5206\u533a\u65f6\u62f7\u8d1d\u5931\u8d25\u5e76\u663e\u793a Conflicts found \uff0c\u5219\u5c5e\u4e8e\u5f02\u5e38\u72b6\u6001\uff0c\u9700\u4e3b\u52a8\u5e72\u9884\u3002","title":"\u51fa\u73b0\u539f\u56e0"},{"location":"backup/jindofs/jindotable/table_moveto/#_8","text":"\u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u7684\u76ee\u6807\u8def\u5f84\u62f7\u8d1d\u6570\u636e\u3002\u7279\u522b\u662f\uff0c\u786e\u4fdd\u6ca1\u6709 DistCp/JindoDistCp \u7b49\u5206\u5e03\u5f0f\u62f7\u8d1d\u547d\u4ee4\u3002 \u5220\u9664\u76ee\u7684\u5730\u76ee\u5f55\u3002\u5bf9\u4e8e\u975e\u5206\u533a\u8868\uff0c\u5220\u9664\u8868\u4e00\u7ea7\u76ee\u5f55\u3002\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u5220\u9664\u5b58\u5728\u51b2\u7a81\u7684\u5206\u533a\u7ea7\u76ee\u5f55\u3002\u5207\u52ff\u5220\u9664\u6e90\u76ee\u5f55\u3002","title":"\u5904\u7406\u65b9\u6cd5"},{"location":"backup/jindofs/jindotable/table_shell/","text":"\u4f7f\u7528 JindoTable \u547d\u4ee4\u5b9e\u73b0 Hive \u8868\u548c\u5206\u533a\u7684\u6570\u636e\u6cbb\u7406 \u00a7 JindoTable \u63d0\u4f9b\u591a\u4e2a\u547d\u4ee4\u884c\u547d\u4ee4\uff0c\u53ef\u4ee5\u4fbf\u6377\u67e5\u8be2 Hive \u8868\u548c\u5206\u533a\u7684\u57fa\u672c\u4fe1\u606f\u6216\u7a7a\u95f4\u5360\u7528\uff0c \u6216\u8005\u5b9e\u73b0\u6570\u636e\u7684\u8fc1\u79fb\u3001\u5f52\u6863\u3002\u672c\u6587\u4ecb\u7ecd\u8fd9\u4e9b\u547d\u4ee4\u884c\u547d\u4ee4\u7684\u7528\u6cd5\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u90e8\u7f72 JindoSDK \u00a7 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8 \u3002 \u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883 \u00a7 \u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e \u4f7f\u7528\u8bf4\u660e \u00a7 \u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f \u00a7 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u67e5\u770b JindoTable \u652f\u6301\u7684\u6240\u6709\u547d\u4ee4\uff1a jindotable -help \u5982\u679c\u60f3\u67e5\u770b\u547d\u4ee4\u7684\u5177\u4f53\u7528\u6cd5\uff0c\u6267\u884c\uff1a jindotable -help <cmd> \u5176\u4e2d\uff0c <cmd> \u53ef\u4ee5\u662f showTable \u3001 moveTo \u3001 archiveTable \u7b49\u3002 showTable \u547d\u4ee4 \u00a7 jindotable -showTable -t <dbName.tableName> \u5176\u4e2d <dbName.tableName> \u4e3a\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a \u5e93\u540d.\u8868\u540d \u529f\u80fd\uff1a\u5982\u679c\u662f\u5206\u533a\u8868\uff0c\u5219\u5c55\u793a\u6240\u6709\u5206\u533a\uff1b\u5982\u679c\u662f\u975e\u5206\u533a\u8868\uff0c\u5219\u8fd4\u56de\u8868\u7684\u5b58\u50a8\u60c5\u51b5\u3002 \u793a\u4f8b\uff1a\u5c55\u793a db1.t1 \u5206\u533a\u8868\u7684\u6240\u6709\u5206\u533a\u3002 jindo table -showTable -t db1.t1 showPartition \u547d\u4ee4 \u00a7 jindotable -showPartition -t <dbName.tableName> -p <partitionSpec> \u5176\u4e2d <dbName.tableName> \u4e3a\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a \u5e93\u540d.\u8868\u540d \uff1b <partitionSpec> \u4e3a\u5206\u533a\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a col1=v1,col2=v2 \u529f\u80fd\uff1a\u8fd4\u56de\u5206\u533a\u7684\u5b58\u50a8\u60c5\u51b5\u3002 \u793a\u4f8b\uff1a\u8fd4\u56de\u5206\u533a\u8868 db1.t1 \u5728 2020-10-12 \u65e5\u7684\u5b58\u50a8\u60c5\u51b5\u3002 jindo table -showPartition -t db1.t1 -p date=2020-10-12 listTables \u547d\u4ee4 \u00a7 jindotable -listTables [-db] <dbName> \u5176\u4e2d <dbName> \u4e3a\u6570\u636e\u5e93\u540d\u79f0\uff0c\u53ef\u7701\u7565\uff0c\u7701\u7565\u5219\u4e3a\u9ed8\u8ba4\u6570\u636e\u5e93\u3002 \u529f\u80fd\uff1a\u5c55\u793a\u6307\u5b9a\u6570\u636e\u5e93\u4e2d\u7684\u6240\u6709\u8868\u3002\u4e0d\u6307\u5b9a -db \u65f6\u9ed8\u8ba4\u5c55\u793a default \u5e93\u4e2d\u7684\u8868\u3002 \u793a\u4f8b\uff1a\u5217\u51fa\u6570\u636e\u5e93 db1 \u4e2d\u7684\u8868\u3002 jindo table -listTables -db db1 moveTo \u547d\u4ee4 \u00a7 \u8bf7\u53c2\u8003 \u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1 archiveTable/unarchiveTable \u547d\u4ee4 \u00a7 \u6682\u4e0d\u652f\u6301\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff0c\u5bf9\u4e8e OSS \u670d\u52a1\uff0c\u8bf7\u53c2\u8003 JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a cacheTable/uncacheTable \u547d\u4ee4 \u00a7 \u4ec5\u652f\u6301\u914d\u5907 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7684\u573a\u666f\uff0c\u8bf7\u53c2\u8003 \u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e","title":"\u4f7f\u7528 JindoTable \u547d\u4ee4\u5b9e\u73b0 Hive \u8868\u548c\u5206\u533a\u7684\u6570\u636e\u6cbb\u7406"},{"location":"backup/jindofs/jindotable/table_shell/#jindotable-hive","text":"JindoTable \u63d0\u4f9b\u591a\u4e2a\u547d\u4ee4\u884c\u547d\u4ee4\uff0c\u53ef\u4ee5\u4fbf\u6377\u67e5\u8be2 Hive \u8868\u548c\u5206\u533a\u7684\u57fa\u672c\u4fe1\u606f\u6216\u7a7a\u95f4\u5360\u7528\uff0c \u6216\u8005\u5b9e\u73b0\u6570\u636e\u7684\u8fc1\u79fb\u3001\u5f52\u6863\u3002\u672c\u6587\u4ecb\u7ecd\u8fd9\u4e9b\u547d\u4ee4\u884c\u547d\u4ee4\u7684\u7528\u6cd5\u3002","title":"\u4f7f\u7528 JindoTable \u547d\u4ee4\u5b9e\u73b0 Hive \u8868\u548c\u5206\u533a\u7684\u6570\u636e\u6cbb\u7406"},{"location":"backup/jindofs/jindotable/table_shell/#_1","text":"","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/jindofs/jindotable/table_shell/#jindosdk","text":"\u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8 \u3002","title":"\u5df2\u90e8\u7f72 JindoSDK"},{"location":"backup/jindofs/jindotable/table_shell/#hadoop-hive","text":"\u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e","title":"\u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883"},{"location":"backup/jindofs/jindotable/table_shell/#_2","text":"","title":"\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/jindotable/table_shell/#_3","text":"\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u67e5\u770b JindoTable \u652f\u6301\u7684\u6240\u6709\u547d\u4ee4\uff1a jindotable -help \u5982\u679c\u60f3\u67e5\u770b\u547d\u4ee4\u7684\u5177\u4f53\u7528\u6cd5\uff0c\u6267\u884c\uff1a jindotable -help <cmd> \u5176\u4e2d\uff0c <cmd> \u53ef\u4ee5\u662f showTable \u3001 moveTo \u3001 archiveTable \u7b49\u3002","title":"\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f"},{"location":"backup/jindofs/jindotable/table_shell/#showtable","text":"jindotable -showTable -t <dbName.tableName> \u5176\u4e2d <dbName.tableName> \u4e3a\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a \u5e93\u540d.\u8868\u540d \u529f\u80fd\uff1a\u5982\u679c\u662f\u5206\u533a\u8868\uff0c\u5219\u5c55\u793a\u6240\u6709\u5206\u533a\uff1b\u5982\u679c\u662f\u975e\u5206\u533a\u8868\uff0c\u5219\u8fd4\u56de\u8868\u7684\u5b58\u50a8\u60c5\u51b5\u3002 \u793a\u4f8b\uff1a\u5c55\u793a db1.t1 \u5206\u533a\u8868\u7684\u6240\u6709\u5206\u533a\u3002 jindo table -showTable -t db1.t1","title":"showTable \u547d\u4ee4"},{"location":"backup/jindofs/jindotable/table_shell/#showpartition","text":"jindotable -showPartition -t <dbName.tableName> -p <partitionSpec> \u5176\u4e2d <dbName.tableName> \u4e3a\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a \u5e93\u540d.\u8868\u540d \uff1b <partitionSpec> \u4e3a\u5206\u533a\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a col1=v1,col2=v2 \u529f\u80fd\uff1a\u8fd4\u56de\u5206\u533a\u7684\u5b58\u50a8\u60c5\u51b5\u3002 \u793a\u4f8b\uff1a\u8fd4\u56de\u5206\u533a\u8868 db1.t1 \u5728 2020-10-12 \u65e5\u7684\u5b58\u50a8\u60c5\u51b5\u3002 jindo table -showPartition -t db1.t1 -p date=2020-10-12","title":"showPartition \u547d\u4ee4"},{"location":"backup/jindofs/jindotable/table_shell/#listtables","text":"jindotable -listTables [-db] <dbName> \u5176\u4e2d <dbName> \u4e3a\u6570\u636e\u5e93\u540d\u79f0\uff0c\u53ef\u7701\u7565\uff0c\u7701\u7565\u5219\u4e3a\u9ed8\u8ba4\u6570\u636e\u5e93\u3002 \u529f\u80fd\uff1a\u5c55\u793a\u6307\u5b9a\u6570\u636e\u5e93\u4e2d\u7684\u6240\u6709\u8868\u3002\u4e0d\u6307\u5b9a -db \u65f6\u9ed8\u8ba4\u5c55\u793a default \u5e93\u4e2d\u7684\u8868\u3002 \u793a\u4f8b\uff1a\u5217\u51fa\u6570\u636e\u5e93 db1 \u4e2d\u7684\u8868\u3002 jindo table -listTables -db db1","title":"listTables \u547d\u4ee4"},{"location":"backup/jindofs/jindotable/table_shell/#moveto","text":"\u8bf7\u53c2\u8003 \u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1","title":"moveTo \u547d\u4ee4"},{"location":"backup/jindofs/jindotable/table_shell/#archivetableunarchivetable","text":"\u6682\u4e0d\u652f\u6301\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff0c\u5bf9\u4e8e OSS \u670d\u52a1\uff0c\u8bf7\u53c2\u8003 JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a","title":"archiveTable/unarchiveTable \u547d\u4ee4"},{"location":"backup/jindofs/jindotable/table_shell/#cachetableuncachetable","text":"\u4ec5\u652f\u6301\u914d\u5907 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7684\u573a\u666f\uff0c\u8bf7\u53c2\u8003 \u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e","title":"cacheTable/uncacheTable \u547d\u4ee4"},{"location":"backup/jindofs/presto/jindosdk_on_presto/","text":"Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u6570\u636e \u00a7 Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002 \u6b65\u9aa4 \u00a7 1. \u5b89\u88c5 jar \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/ 2. \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> 3. \u914d\u7f6e OSS Access Key \u00a7 \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002 4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u00a7 \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 5. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 \u4f7f\u7528\u793a\u4f8b \u00a7 \u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236\u53f0 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='oss://<Bucket>.<Endpoint>/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofs/presto/jindosdk_on_presto/#presto-jindosdk-oss-hdfs","text":"Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002","title":"Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofs/presto/jindosdk_on_presto/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofs/presto/jindosdk_on_presto/#1-jar","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/","title":"1. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofs/presto/jindosdk_on_presto/#2-jindosdk-oss","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration>","title":"2. \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b"},{"location":"backup/jindofs/presto/jindosdk_on_presto/#3-oss-access-key","text":"\u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002","title":"3. \u914d\u7f6e OSS Access Key"},{"location":"backup/jindofs/presto/jindosdk_on_presto/#4-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002","title":"4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint"},{"location":"backup/jindofs/presto/jindosdk_on_presto/#5-presto","text":"","title":"5. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"backup/jindofs/presto/jindosdk_on_presto/#_2","text":"\u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236\u53f0 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='oss://<Bucket>.<Endpoint>/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"\u4f7f\u7528\u793a\u4f8b"},{"location":"backup/jindofs/security/jindosdk_credential_provider_bucket_dls/","text":"JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u6309 bucket \u914d\u7f6e\u4f7f\u7528\u8bf4\u660e \u00a7 \u6309\u7167 bucket \u914d\u7f6e Credential Provider \u00a7 <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleAliyunCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.AliyunCredentialsProvider\u7684\u5b9e\u73b0\u7c7b\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u8bf4\u660e XXX \u4e3a OSS-HDFS \u670d\u52a1 bucket \u540d\u79f0\u3002 Credential Provider \u7c7b\u578b \u00a7 1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Secret</value> </property> <property> <name>fs.oss.bucket.XXX.securityToken</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration> 2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Secret</value> </property> </configuration> 3. EnvironmentVariableCredentialsProvider \u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6 AccessKey\u3002 \u00a7 \u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Id OSS_ACCESS_KEY_SECRET OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Secret OSS_SECURITY_TOKEN OSS-HDFS \u670d\u52a1 bucket \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548Token\u65f6\u9700\u8981\u3002 4. CommonCredentialsProvider \u4e3a\u901a\u7528\u914d\u7f6e\u3002 \u00a7 \u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration>","title":"JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u6309 bucket \u914d\u7f6e\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/security/jindosdk_credential_provider_bucket_dls/#jindosdk-oss-hdfs-credential-provider-bucket","text":"","title":"JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u6309 bucket \u914d\u7f6e\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofs/security/jindosdk_credential_provider_bucket_dls/#bucket-credential-provider","text":"<configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleAliyunCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.AliyunCredentialsProvider\u7684\u5b9e\u73b0\u7c7b\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u8bf4\u660e XXX \u4e3a OSS-HDFS \u670d\u52a1 bucket \u540d\u79f0\u3002","title":"\u6309\u7167 bucket \u914d\u7f6e Credential Provider"},{"location":"backup/jindofs/security/jindosdk_credential_provider_bucket_dls/#credential-provider","text":"","title":"Credential Provider \u7c7b\u578b"},{"location":"backup/jindofs/security/jindosdk_credential_provider_bucket_dls/#1-temporarycredentialsprovider-accesskey-securitytoken-oss-hdfs","text":"\u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Secret</value> </property> <property> <name>fs.oss.bucket.XXX.securityToken</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration>","title":"1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u60c5\u51b5\u3002"},{"location":"backup/jindofs/security/jindosdk_credential_provider_bucket_dls/#2-simplecredentialsprovider-accesskey-oss-hdfs","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Secret</value> </property> </configuration>","title":"2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u60c5\u51b5\u3002"},{"location":"backup/jindofs/security/jindosdk_credential_provider_bucket_dls/#3-environmentvariablecredentialsprovider-accesskey","text":"\u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Id OSS_ACCESS_KEY_SECRET OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Secret OSS_SECURITY_TOKEN OSS-HDFS \u670d\u52a1 bucket \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548Token\u65f6\u9700\u8981\u3002","title":"3. EnvironmentVariableCredentialsProvider \u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6 AccessKey\u3002"},{"location":"backup/jindofs/security/jindosdk_credential_provider_bucket_dls/#4-commoncredentialsprovider","text":"\u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS-HDFS \u670d\u52a1 bucket \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration>","title":"4. CommonCredentialsProvider \u4e3a\u901a\u7528\u914d\u7f6e\u3002"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/","text":"JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u00a7 \u57fa\u672c\u914d\u7f6e\u65b9\u5f0f \u00a7 \u60a8\u53ef\u4ee5\u5c06\u5df2\u5f00\u542f\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 \u7684 Access Key ID \u3001 Access Secret \u3001 Endpoint \u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \uff0c\u914d\u7f6e\u9879\u5982\u4e0b\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>cn-xxx.oss-dls.aliyuncs.com</value> </property> </configuration> \u4f7f\u7528 Hadoop Credential Providers \u5b58\u50a8 AccessKey \u4fe1\u606f \u00a7 \u4e0a\u9762\u7684 fs.oss.accessKeyId \u548c fs.oss.accessKeySecret \u5c06\u4f1a\u660e\u6587\u914d\u7f6e\u5728 core-site.xml \u4e2d\uff0c\u53ef\u4ee5\u5c06\u5176\u4ee5\u52a0\u5bc6\u5bf9\u65b9\u5f0f\u5b58\u50a8\u81f3 Hadoop Credential Providers\u6587\u4ef6\u4e2d\u3002 \u4f7f\u7528Hadoop\u63d0\u4f9b\u7684\u547d\u4ee4\uff0c\u5b58\u50a8AccessKey\u548cSecurityToken\u4fe1\u606f\u81f3Credential\u6587\u4ef6\u4e2d\u3002\u547d\u4ee4\u683c\u5f0f\u5982\u4e0b\u3002 hadoop credential <subcommand> [options] \u4f8b\u5982\uff0c\u5b58\u50a8AccessKey\u548cToken\u4fe1\u606f\u81f3JECKS\u6587\u4ef6\u4e2d\uff0c\u9664\u4e86\u4f7f\u7528\u6587\u4ef6\u6743\u9650\u4fdd\u62a4\u8be5\u6587\u4ef6\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u6307\u5b9a\u5bc6\u7801\u52a0\u5bc6\u5b58\u50a8\u4fe1\u606f\uff0c\u5982\u679c\u4e0d\u6307\u5b9a\u5bc6\u7801\u5219\u4f7f\u7528\u9ed8\u8ba4\u5b57\u7b26\u4e32\u52a0\u5bc6\u3002 hadoop credential create fs.oss.accessKeyId -value AAA -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.accessKeySecret -value BBB -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.securityToken -value CCC -provider jceks://file/root/oss.jceks \u751f\u6210Credential\u6587\u4ef6\u540e\uff0c\u60a8\u9700\u8981\u914d\u7f6e\u4e0b\u9762\u7684\u53c2\u6570\u6765\u6307\u5b9aProvider\u7684\u7c7b\u578b\u548c\u4f4d\u7f6e\u3002 <configuration> <property> <name>fs.oss.security.credential.provider.path</name> <value>jceks://file/root/oss.jceks</value> <description>\u914d\u7f6e\u5b58\u50a8AK\u7684Credential\u6587\u4ef6\u3002\u4f8b\u5982\uff0cjceks://file/${user.home}/oss.jceks\u4e3aHOME\u4e0b\u7684oss.jceks\u6587\u4ef6</description> </property> </configuration> \u4f7f\u7528 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u00a7 \u9ed8\u8ba4\u4f1a\u914d\u7f6e SimpleCredentialsProvider\uff0cEnvironmentVariableCredentialsProvider\uff0cCommonCredentialsProvider \u8fd9\u4e09\u4e2aCredential Provider, \u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6 Credential \u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684 Credential\u3002 <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.JindoCredentialsProvider\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u60a8\u53ef\u4ee5\u6839\u636e\u60c5\u51b5\uff0c\u9009\u62e9\u4e0d\u540c\u7684 Credential Provider\uff0c\u652f\u6301\u5982\u4e0bProvider\uff1a 1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Secret</value> </property> <property> <name>fs.oss.securityToken</name> <value>OSS-HDFS \u670d\u52a1\u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration> 2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Secret</value> </property> </configuration> 3. EnvironmentVariableCredentialsProvider\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6AK\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS-HDFS \u670d\u52a1\u7684 AccessKey Id OSS_ACCESS_KEY_SECRET OSS-HDFS \u670d\u52a1\u7684 AccessKey Secret OSS_SECURITY_TOKEN OSS-HDFS \u670d\u52a1\u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002 4. CommonCredentialsProvider \u4e3a\u901a\u7528\u914d\u7f6e\u3002 \u00a7 \u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS-HDFS \u670d\u52a1\u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration> 5. CustomCredentialsProvider \u5bf9\u63a5\u5b9a\u5236\u7684\u514d\u5bc6\u670d\u52a1\u3002 \u00a7 \u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CustomCredentialsProvider</value> </property> </configuration> \u914d\u7f6e\u514d\u5bc6\u670d\u52a1\u5730\u5740 <configuration> <property> <name>aliyun.oss.provider.url</name> <value>\u514d\u5bc6\u670d\u52a1\u5730\u5740</value> </property> </configuration> aliyun.oss.provider.url \u652f\u6301 http(s) \u534f\u8bae \u548c secrets \u534f\u8bae\uff1a a. http(s) \u534f\u8bae http(s) \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a http://localhost:1234/sts , http \u514d\u5bc6\u534f\u8bae\u8981\u6c42\u8fd4\u56de\u7ed3\u679c\u4e3a Json \u683c\u5f0f\uff0c\u5982\u679c\u60a8\u9700\u8981\u5bf9\u63a5\u60a8\u7684 http \u514d\u5bc6\u670d\u52a1\uff0c\u514d\u5bc6\u670d\u52a1\u7684\u8bbe\u8ba1\u53c2\u8003 \u300a\u901a\u8fc7API\u4f7f\u7528\u5b9e\u4f8bRAM\u89d2\u8272#\u4e34\u65f6\u6388\u6743Token\u300b { \"AccessKeyId\" : \"XXXXXXXXX\", \"AccessKeySecret\" : \"XXXXXXXXX\", \"Expiration\" : \"2020-11-01T05:20:01Z\", \"SecurityToken\" : \"XXXXXXXXX\", \"LastUpdated\" : \"2020-10-31T23:20:01Z\", \"Code\" : \"Success\" } b. Secrets \u534f\u8bae Secrets \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a secrets:///local_path_prefix \uff0c\u5e38\u89c1\u4f7f\u7528\u4e8e k8s \u573a\u666f \uff0c \u5176\u4e2d local_path_prefix \u4e3a\u8def\u5f84\u524d\u7f00\uff0c\u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOssAccessKeyId /secret/JindoOssAccessKeySecret /secret/JindoOssSecurityToken \u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss/ \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOss/AccessKeyId /secret/JindoOss/AccessKeySecret /secret/JindoOss/SecurityToken JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 OSS-HDFS \u670d\u52a1 bucket \u914d\u7f6e\u4e0d\u540c\u7684 Credential Provider \u00a7 \u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u6309 bucket \u914d\u7f6e\u4f7f\u7528\u8bf4\u660e \u3002","title":"JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/#jindosdk-oss-hdfs-credential-provider","text":"","title":"JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/#_1","text":"\u60a8\u53ef\u4ee5\u5c06\u5df2\u5f00\u542f\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 \u7684 Access Key ID \u3001 Access Secret \u3001 Endpoint \u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \uff0c\u914d\u7f6e\u9879\u5982\u4e0b\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>cn-xxx.oss-dls.aliyuncs.com</value> </property> </configuration>","title":"\u57fa\u672c\u914d\u7f6e\u65b9\u5f0f"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/#hadoop-credential-providers-accesskey","text":"\u4e0a\u9762\u7684 fs.oss.accessKeyId \u548c fs.oss.accessKeySecret \u5c06\u4f1a\u660e\u6587\u914d\u7f6e\u5728 core-site.xml \u4e2d\uff0c\u53ef\u4ee5\u5c06\u5176\u4ee5\u52a0\u5bc6\u5bf9\u65b9\u5f0f\u5b58\u50a8\u81f3 Hadoop Credential Providers\u6587\u4ef6\u4e2d\u3002 \u4f7f\u7528Hadoop\u63d0\u4f9b\u7684\u547d\u4ee4\uff0c\u5b58\u50a8AccessKey\u548cSecurityToken\u4fe1\u606f\u81f3Credential\u6587\u4ef6\u4e2d\u3002\u547d\u4ee4\u683c\u5f0f\u5982\u4e0b\u3002 hadoop credential <subcommand> [options] \u4f8b\u5982\uff0c\u5b58\u50a8AccessKey\u548cToken\u4fe1\u606f\u81f3JECKS\u6587\u4ef6\u4e2d\uff0c\u9664\u4e86\u4f7f\u7528\u6587\u4ef6\u6743\u9650\u4fdd\u62a4\u8be5\u6587\u4ef6\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u6307\u5b9a\u5bc6\u7801\u52a0\u5bc6\u5b58\u50a8\u4fe1\u606f\uff0c\u5982\u679c\u4e0d\u6307\u5b9a\u5bc6\u7801\u5219\u4f7f\u7528\u9ed8\u8ba4\u5b57\u7b26\u4e32\u52a0\u5bc6\u3002 hadoop credential create fs.oss.accessKeyId -value AAA -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.accessKeySecret -value BBB -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.securityToken -value CCC -provider jceks://file/root/oss.jceks \u751f\u6210Credential\u6587\u4ef6\u540e\uff0c\u60a8\u9700\u8981\u914d\u7f6e\u4e0b\u9762\u7684\u53c2\u6570\u6765\u6307\u5b9aProvider\u7684\u7c7b\u578b\u548c\u4f4d\u7f6e\u3002 <configuration> <property> <name>fs.oss.security.credential.provider.path</name> <value>jceks://file/root/oss.jceks</value> <description>\u914d\u7f6e\u5b58\u50a8AK\u7684Credential\u6587\u4ef6\u3002\u4f8b\u5982\uff0cjceks://file/${user.home}/oss.jceks\u4e3aHOME\u4e0b\u7684oss.jceks\u6587\u4ef6</description> </property> </configuration>","title":"\u4f7f\u7528 Hadoop Credential Providers \u5b58\u50a8 AccessKey \u4fe1\u606f"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/#jindosdk-oss-hdfs-credential-provider_1","text":"\u9ed8\u8ba4\u4f1a\u914d\u7f6e SimpleCredentialsProvider\uff0cEnvironmentVariableCredentialsProvider\uff0cCommonCredentialsProvider \u8fd9\u4e09\u4e2aCredential Provider, \u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6 Credential \u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684 Credential\u3002 <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.JindoCredentialsProvider\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u60a8\u53ef\u4ee5\u6839\u636e\u60c5\u51b5\uff0c\u9009\u62e9\u4e0d\u540c\u7684 Credential Provider\uff0c\u652f\u6301\u5982\u4e0bProvider\uff1a","title":"\u4f7f\u7528 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/#1-temporarycredentialsprovider-accesskey-securitytoken-oss","text":"\u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Secret</value> </property> <property> <name>fs.oss.securityToken</name> <value>OSS-HDFS \u670d\u52a1\u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration>","title":"1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/#2-simplecredentialsprovider-accesskey-oss","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Secret</value> </property> </configuration>","title":"2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/#3-environmentvariablecredentialsproviderak","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS-HDFS \u670d\u52a1\u7684 AccessKey Id OSS_ACCESS_KEY_SECRET OSS-HDFS \u670d\u52a1\u7684 AccessKey Secret OSS_SECURITY_TOKEN OSS-HDFS \u670d\u52a1\u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002","title":"3. EnvironmentVariableCredentialsProvider\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6AK\u3002"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/#4-commoncredentialsprovider","text":"\u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS-HDFS \u670d\u52a1\u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS-HDFS \u670d\u52a1\u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration>","title":"4. CommonCredentialsProvider \u4e3a\u901a\u7528\u914d\u7f6e\u3002"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/#5-customcredentialsprovider","text":"\u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CustomCredentialsProvider</value> </property> </configuration> \u914d\u7f6e\u514d\u5bc6\u670d\u52a1\u5730\u5740 <configuration> <property> <name>aliyun.oss.provider.url</name> <value>\u514d\u5bc6\u670d\u52a1\u5730\u5740</value> </property> </configuration> aliyun.oss.provider.url \u652f\u6301 http(s) \u534f\u8bae \u548c secrets \u534f\u8bae\uff1a a. http(s) \u534f\u8bae http(s) \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a http://localhost:1234/sts , http \u514d\u5bc6\u534f\u8bae\u8981\u6c42\u8fd4\u56de\u7ed3\u679c\u4e3a Json \u683c\u5f0f\uff0c\u5982\u679c\u60a8\u9700\u8981\u5bf9\u63a5\u60a8\u7684 http \u514d\u5bc6\u670d\u52a1\uff0c\u514d\u5bc6\u670d\u52a1\u7684\u8bbe\u8ba1\u53c2\u8003 \u300a\u901a\u8fc7API\u4f7f\u7528\u5b9e\u4f8bRAM\u89d2\u8272#\u4e34\u65f6\u6388\u6743Token\u300b { \"AccessKeyId\" : \"XXXXXXXXX\", \"AccessKeySecret\" : \"XXXXXXXXX\", \"Expiration\" : \"2020-11-01T05:20:01Z\", \"SecurityToken\" : \"XXXXXXXXX\", \"LastUpdated\" : \"2020-10-31T23:20:01Z\", \"Code\" : \"Success\" } b. Secrets \u534f\u8bae Secrets \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a secrets:///local_path_prefix \uff0c\u5e38\u89c1\u4f7f\u7528\u4e8e k8s \u573a\u666f \uff0c \u5176\u4e2d local_path_prefix \u4e3a\u8def\u5f84\u524d\u7f00\uff0c\u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOssAccessKeyId /secret/JindoOssAccessKeySecret /secret/JindoOssSecurityToken \u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss/ \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOss/AccessKeyId /secret/JindoOss/AccessKeySecret /secret/JindoOss/SecurityToken","title":"5. CustomCredentialsProvider \u5bf9\u63a5\u5b9a\u5236\u7684\u514d\u5bc6\u670d\u52a1\u3002"},{"location":"backup/jindofs/security/jindosdk_credential_provider_dls/#jindosdk-oss-hdfs-bucket-credential-provider","text":"\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u6309 bucket \u914d\u7f6e\u4f7f\u7528\u8bf4\u660e \u3002","title":"JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 OSS-HDFS \u670d\u52a1 bucket \u914d\u7f6e\u4e0d\u540c\u7684 Credential Provider"},{"location":"backup/jindofs/spark/jindosdk_ide_spark/","text":"Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 \u00a7 \u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindosdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.spark.sql.SparkSession; public class TestJindoSDK { public static void main(String[] args) throws Exception { SparkSession spark = SparkSession .builder() .config(\"spark.hadoop.fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\") .config(\"spark.hadoop.fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\") .config(\"spark.hadoop.fs.oss.accessKeyId\", \"xxx\") .config(\"spark.hadoop.fs.oss.accessKeySecret\", \"xxx\") // set accessKey, secret, endpoint and so on. .appName(\"TestJindoSDK\") .getOrCreate(); spark.read().parquet(\"oss://bucket.endpoint/xxx\").count(); spark.stop(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency>","title":"Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5"},{"location":"backup/jindofs/spark/jindosdk_ide_spark/#spark-jindosdk-ide","text":"\u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindosdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.spark.sql.SparkSession; public class TestJindoSDK { public static void main(String[] args) throws Exception { SparkSession spark = SparkSession .builder() .config(\"spark.hadoop.fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\") .config(\"spark.hadoop.fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\") .config(\"spark.hadoop.fs.oss.accessKeyId\", \"xxx\") .config(\"spark.hadoop.fs.oss.accessKeySecret\", \"xxx\") // set accessKey, secret, endpoint and so on. .appName(\"TestJindoSDK\") .getOrCreate(); spark.read().parquet(\"oss://bucket.endpoint/xxx\").count(); spark.stop(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency>","title":"Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5"},{"location":"backup/jindofs/spark/jindosdk_on_spark/","text":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e \u00a7 JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411Hadoop/Spark\u751f\u6001\u7684OSS\u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684Hadoop FileSystem\u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fddSpark\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/ 3. \u914d\u7f6e JindoSDK \u00a7 \u5168\u5c40\u914d\u7f6e, \u4fee\u6539 Spark \u4f7f\u7528\u7684 core-site.xml \u00a7 \u914d\u7f6e JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u4efb\u52a1\u7ea7\u522b\u914d\u7f6e \u00a7 \u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx 4. \u4f7f\u7528 Spark \u8bbf\u95ee OSS \u00a7 \u521b\u5efa\u8868 create table test_oss (c1 string) location \"oss://bucket.endpoint/dir\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss; 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofs/spark/jindosdk_on_spark/#spark-jindosdk-oss-hdfs-jindofs","text":"JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411Hadoop/Spark\u751f\u6001\u7684OSS\u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684Hadoop FileSystem\u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofs/spark/jindosdk_on_spark/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofs/spark/jindosdk_on_spark/#1-spark-hadoop","text":"\u786e\u4fddSpark\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"backup/jindofs/spark/jindosdk_on_spark/#2-spark-classpath-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/","title":"2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK"},{"location":"backup/jindofs/spark/jindosdk_on_spark/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofs/spark/jindosdk_on_spark/#spark-core-sitexml","text":"\u914d\u7f6e JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://dls-chenshi-test.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002","title":"\u5168\u5c40\u914d\u7f6e, \u4fee\u6539 Spark \u4f7f\u7528\u7684core-site.xml"},{"location":"backup/jindofs/spark/jindosdk_on_spark/#_2","text":"\u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx","title":"\u4efb\u52a1\u7ea7\u522b\u914d\u7f6e"},{"location":"backup/jindofs/spark/jindosdk_on_spark/#4-spark-oss","text":"\u521b\u5efa\u8868 create table test_oss (c1 string) location \"oss://bucket.endpoint/dir\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss;","title":"4. \u4f7f\u7528 Spark \u8bbf\u95ee OSS"},{"location":"backup/jindofs/spark/jindosdk_on_spark/#5","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofs/sqoop/kitesdk_on_sqoop/","text":"Sqoop \u4f7f\u7528 Kite SDK \u8bfb\u5199\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 \u5728\u96c6\u7fa4\u4e0a\u6709\u5f00\u6e90\u7248\u672c Sqoop \u8f6f\u4ef6\uff0c\u7248\u672c\u4e0d\u4f4e\u4e8e 1.4.7\u3002 Sqoop \u4f9d\u8d56\u7684 Hadoop \u73af\u5883\u53ef\u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff0c\u53ef\u53c2\u8003 Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u4f7f\u7528 Kite SDK \u8bfb\u5199 OSS-HDFS \u670d\u52a1 \u00a7 Sqoop \u672c\u8eab\u5e76\u4e0d\u652f\u6301\u5bf9 OSS \u7684\u8bfb\u5199\uff0c\u9700\u8981\u4f7f\u7528\u7b2c\u4e09\u65b9 Kite SDK \u8fdb\u884c URI \u7684\u8f6c\u6362\u4ece\u800c\u8fdb\u884c\u6570\u636e\u4ea4\u4e92\u3002 \u6b65\u9aa4 \u00a7 1. \u5b89\u88c5 jar \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 jar \u5305 kite-data-oss-3.4.0.jar ( \u4e0b\u8f7d\u9875\u9762 ))\uff0c\u5c06 sdk \u5305\u5b89\u88c5\u5230 Sqoop \u7684 classpath \u76ee\u5f55\u4e0b\u3002 cp ./kite-data-oss-3.4.0.jar <SQOOP_HOME>/lib/kite-data-oss-3.4.0.jar 2.\u6388\u4e88\u5bf9 JAR \u7684\u6743\u9650 \u00a7 sudo chmod 755 kite-data-oss-3.4.0.jar 3.\u4f7f\u7528 oss \u8fde\u63a5\u5668\u5bfc\u5165 mysql \u4e0a\u7684\u6570\u636e \u00a7 sqoop import --connect jdbc:mysql://<host>:<port>/database --username username --password password --table yourtable --target-dir \"oss://yourbucket/dir/\" --as-parquetfile -m 5","title":"Sqoop \u4f7f\u7528 Kite SDK \u8bfb\u5199\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/sqoop/kitesdk_on_sqoop/#sqoop-kite-sdk-oss-hdfs-jindofs","text":"","title":"Sqoop \u4f7f\u7528 Kite SDK \u8bfb\u5199\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofs/sqoop/kitesdk_on_sqoop/#_1","text":"\u5728\u96c6\u7fa4\u4e0a\u6709\u5f00\u6e90\u7248\u672c Sqoop \u8f6f\u4ef6\uff0c\u7248\u672c\u4e0d\u4f4e\u4e8e 1.4.7\u3002 Sqoop \u4f9d\u8d56\u7684 Hadoop \u73af\u5883\u53ef\u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff0c\u53ef\u53c2\u8003 Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS-HDFS \u670d\u52a1","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/jindofs/sqoop/kitesdk_on_sqoop/#kite-sdk-oss-hdfs","text":"Sqoop \u672c\u8eab\u5e76\u4e0d\u652f\u6301\u5bf9 OSS \u7684\u8bfb\u5199\uff0c\u9700\u8981\u4f7f\u7528\u7b2c\u4e09\u65b9 Kite SDK \u8fdb\u884c URI \u7684\u8f6c\u6362\u4ece\u800c\u8fdb\u884c\u6570\u636e\u4ea4\u4e92\u3002","title":"\u4f7f\u7528 Kite SDK \u8bfb\u5199 OSS-HDFS \u670d\u52a1"},{"location":"backup/jindofs/sqoop/kitesdk_on_sqoop/#_2","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofs/sqoop/kitesdk_on_sqoop/#1-jar","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 jar \u5305 kite-data-oss-3.4.0.jar ( \u4e0b\u8f7d\u9875\u9762 ))\uff0c\u5c06 sdk \u5305\u5b89\u88c5\u5230 Sqoop \u7684 classpath \u76ee\u5f55\u4e0b\u3002 cp ./kite-data-oss-3.4.0.jar <SQOOP_HOME>/lib/kite-data-oss-3.4.0.jar","title":"1. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofs/sqoop/kitesdk_on_sqoop/#2-jar","text":"sudo chmod 755 kite-data-oss-3.4.0.jar","title":"2.\u6388\u4e88\u5bf9 JAR \u7684\u6743\u9650"},{"location":"backup/jindofs/sqoop/kitesdk_on_sqoop/#3-oss-mysql","text":"sqoop import --connect jdbc:mysql://<host>:<port>/database --username username --password password --table yourtable --target-dir \"oss://yourbucket/dir/\" --as-parquetfile -m 5","title":"3.\u4f7f\u7528 oss \u8fde\u63a5\u5668\u5bfc\u5165 mysql \u4e0a\u7684\u6570\u636e"},{"location":"backup/jindofsx/jindofsx_cache/","text":"JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u00a7 \u7f13\u5b58\u52a0\u901f\u662fJindoFSx\u4e00\u4e2a\u6838\u5fc3\u529f\u80fd\uff0c\u5305\u62ec\u5143\u6570\u636e\u7f13\u5b58\u548c\u6570\u636e\u7f13\u5b58\uff0c\u53ef\u4ee5\u5bf9\u4e0d\u540c\u7684\u5b58\u50a8\u540e\u7aef\u63d0\u4f9b\u8bbf\u95ee\u52a0\u901f\u80fd\u529b\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u542f\u7528\u7f13\u5b58\u52a0\u901f\u529f\u80fd \u542f\u7528\u7f13\u5b58\u4f1a\u5229\u7528\u672c\u5730\u78c1\u76d8\u5bf9\u8bbf\u95ee\u7684\u70ed\u6570\u636e\u5757\u8fdb\u884c\u7f13\u5b58\uff0c\u9ed8\u8ba4\u72b6\u6001\u4e3a\u7981\u7528\uff0c\u5373\u6240\u6709OSS\u8bfb\u53d6\u90fd\u76f4\u63a5\u8bbf\u95eeOSS\u4e0a\u7684\u6570\u636e\u3002 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> <property> <!-- \u5143\u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.meta.cache.enable</name> <value>false</value> </property> <property> <!-- \u5c0f\u6587\u4ef6\u7f13\u5b58\u4f18\u5316\u5f00\u5173 --> <name>fs.jindofsx.slice.cache.enable</name> <value>false</value> </property> <property> <!-- \u5185\u5b58\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.ram.cache.enable</name> <value>false</value> </property> <property> <!-- \u77ed\u8def\u8bfb\u5f00\u5173 --> <name>fs.jindofsx.short.circuit.enable</name> <value>true</value> </property> </configuration> \u5b8c\u6210\u4ee5\u4e0a\u914d\u7f6e\u540e\uff0c\u4f5c\u4e1a\u8bfb\u53d6 OSS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002 \u6ce8\u610f\uff1a\u6b64\u914d\u7f6e\u4e3a\u5ba2\u6237\u7aef\u914d\u7f6e\uff0c\u4e0d\u9700\u8981\u91cd\u542f JindoFSx \u670d\u52a1\u3002 \u78c1\u76d8\u7a7a\u95f4\u6c34\u4f4d\u63a7\u5236 \u00a7 \u7f13\u5b58\u542f\u7528\u540e\uff0cJindoFSx \u670d\u52a1\u4f1a\u81ea\u52a8\u7ba1\u7406\u672c\u5730\u7f13\u5b58\u5907\u4efd\uff0c\u901a\u8fc7\u6c34\u4f4d\u6e05\u7406\u672c\u5730\u7f13\u5b58\uff0c\u8bf7\u60a8\u6839\u636e\u9700\u6c42\u914d\u7f6e\u4e00\u5b9a\u7684\u6bd4\u4f8b\u7528\u4e8e\u7f13\u5b58\u3002JindoFSx \u540e\u7aef\u57fa\u4e8e OSS\uff0c\u53ef\u4ee5\u63d0\u4f9b\u6d77\u91cf\u7684\u5b58\u50a8\uff0c\u4f46\u662f\u672c\u5730\u76d8\u7684\u5bb9\u91cf\u662f\u6709\u9650\u7684\uff0c\u56e0\u6b64 JindoFSx \u4f1a\u81ea\u52a8\u6dd8\u6c70\u672c\u5730\u8f83\u51b7\u7684\u6570\u636e\u5907\u4efd \u6211\u4eec\u63d0\u4f9b\u4e86 storage.watermark.high.ratio \u548c storage.watermark.low.ratio \u4e24\u4e2a\u53c2\u6570\u6765\u8c03\u8282\u672c\u5730\u5b58\u50a8\u7684\u4f7f\u7528\u5bb9\u91cf\uff0c\u503c\u5747\u4e3a0\uff5e1\u7684\u5c0f\u6570\uff0c\u8868\u793a\u4f7f\u7528\u78c1\u76d8\u7a7a\u95f4\u7684\u6bd4\u4f8b\u3002 \u4fee\u6539 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u5e76\u66f4\u65b0\u81f3\u6240\u6709\u670d\u52a1\u8282\u70b9\u3002 [jindofsx-storage]# Storage service \u914d\u7f6e storage.data-dirs=/mnt/disk1/jindofsx,/mnt/disk2/jindofsx # \u8868\u793a\u7528\u4ee5\u7f13\u5b58\u6570\u636e\u7684\u78c1\u76d8\u76ee\u5f55 storage.data-dirs.capacities=500G,500G # \u8868\u793a\u6bcf\u4e2a\u78c1\u76d8\u76ee\u5f55\u7684\u603b\u5bb9\u91cf storage.watermark.high.ratio=0.8 # \u8868\u793a\u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0a\u6c34\u4f4d\u6bd4\u4f8b\uff0c\u6bcf\u5757\u6570\u636e\u76d8\u7684 JindoFSx \u6570\u636e\u76ee\u5f55\u5360\u7528\u7684\u78c1\u76d8\u7a7a\u95f4\u5230\u8fbe\u4e0a\u6c34\u4f4d\u5373\u4f1a\u89e6\u53d1\u6e05\u7406\u3002\u9ed8\u8ba4\u503c\uff1a0.8\u3002 storage.watermark.low.ratio=0.6 # \u8868\u793a\u4f7f\u7528\u91cf\u7684\u4e0b\u6c34\u4f4d\u6bd4\u4f8b\uff0c\u89e6\u53d1\u6e05\u7406\u540e\u4f1a\u81ea\u52a8\u6e05\u7406\u51b7\u6570\u636e\uff0c\u5c06JindoFS\u6570\u636e\u76ee\u5f55\u5360\u7528\u7a7a\u95f4\u6e05\u7406\u5230\u4e0b\u6c34\u4f4d\u3002\u9ed8\u8ba4\u503c\uff1a0.6\u3002 \u8bf4\u660e: \u60a8\u53ef\u4ee5\u901a\u8fc7\u8bbe\u7f6e\u4e0a\u6c34\u4f4d\u6bd4\u4f8b\u8c03\u8282\u671f\u671b\u5206\u7ed9 JindoFSx \u7684\u78c1\u76d8\u7a7a\u95f4\uff0c\u4e0b\u6c34\u4f4d\u5fc5\u987b\u5c0f\u4e8e\u4e0a\u6c34\u4f4d\uff0c\u8bbe\u7f6e\u5408\u7406\u7684\u503c\u5373\u53ef\u3002 \u6ce8\u610f: \u914d\u7f6e\u5b8c\u6210\u540e\u9700\u91cd\u542f JindoFSx \u670d\u52a1\u3002","title":"JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofsx/jindofsx_cache/#jindofsx","text":"\u7f13\u5b58\u52a0\u901f\u662fJindoFSx\u4e00\u4e2a\u6838\u5fc3\u529f\u80fd\uff0c\u5305\u62ec\u5143\u6570\u636e\u7f13\u5b58\u548c\u6570\u636e\u7f13\u5b58\uff0c\u53ef\u4ee5\u5bf9\u4e0d\u540c\u7684\u5b58\u50a8\u540e\u7aef\u63d0\u4f9b\u8bbf\u95ee\u52a0\u901f\u80fd\u529b\u3002","title":"JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofsx/jindofsx_cache/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindofsx_cache/#jindosdk","text":"\u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u542f\u7528\u7f13\u5b58\u52a0\u901f\u529f\u80fd \u542f\u7528\u7f13\u5b58\u4f1a\u5229\u7528\u672c\u5730\u78c1\u76d8\u5bf9\u8bbf\u95ee\u7684\u70ed\u6570\u636e\u5757\u8fdb\u884c\u7f13\u5b58\uff0c\u9ed8\u8ba4\u72b6\u6001\u4e3a\u7981\u7528\uff0c\u5373\u6240\u6709OSS\u8bfb\u53d6\u90fd\u76f4\u63a5\u8bbf\u95eeOSS\u4e0a\u7684\u6570\u636e\u3002 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> <property> <!-- \u5143\u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.meta.cache.enable</name> <value>false</value> </property> <property> <!-- \u5c0f\u6587\u4ef6\u7f13\u5b58\u4f18\u5316\u5f00\u5173 --> <name>fs.jindofsx.slice.cache.enable</name> <value>false</value> </property> <property> <!-- \u5185\u5b58\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.ram.cache.enable</name> <value>false</value> </property> <property> <!-- \u77ed\u8def\u8bfb\u5f00\u5173 --> <name>fs.jindofsx.short.circuit.enable</name> <value>true</value> </property> </configuration> \u5b8c\u6210\u4ee5\u4e0a\u914d\u7f6e\u540e\uff0c\u4f5c\u4e1a\u8bfb\u53d6 OSS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002 \u6ce8\u610f\uff1a\u6b64\u914d\u7f6e\u4e3a\u5ba2\u6237\u7aef\u914d\u7f6e\uff0c\u4e0d\u9700\u8981\u91cd\u542f JindoFSx \u670d\u52a1\u3002","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/jindofsx_cache/#_2","text":"\u7f13\u5b58\u542f\u7528\u540e\uff0cJindoFSx \u670d\u52a1\u4f1a\u81ea\u52a8\u7ba1\u7406\u672c\u5730\u7f13\u5b58\u5907\u4efd\uff0c\u901a\u8fc7\u6c34\u4f4d\u6e05\u7406\u672c\u5730\u7f13\u5b58\uff0c\u8bf7\u60a8\u6839\u636e\u9700\u6c42\u914d\u7f6e\u4e00\u5b9a\u7684\u6bd4\u4f8b\u7528\u4e8e\u7f13\u5b58\u3002JindoFSx \u540e\u7aef\u57fa\u4e8e OSS\uff0c\u53ef\u4ee5\u63d0\u4f9b\u6d77\u91cf\u7684\u5b58\u50a8\uff0c\u4f46\u662f\u672c\u5730\u76d8\u7684\u5bb9\u91cf\u662f\u6709\u9650\u7684\uff0c\u56e0\u6b64 JindoFSx \u4f1a\u81ea\u52a8\u6dd8\u6c70\u672c\u5730\u8f83\u51b7\u7684\u6570\u636e\u5907\u4efd \u6211\u4eec\u63d0\u4f9b\u4e86 storage.watermark.high.ratio \u548c storage.watermark.low.ratio \u4e24\u4e2a\u53c2\u6570\u6765\u8c03\u8282\u672c\u5730\u5b58\u50a8\u7684\u4f7f\u7528\u5bb9\u91cf\uff0c\u503c\u5747\u4e3a0\uff5e1\u7684\u5c0f\u6570\uff0c\u8868\u793a\u4f7f\u7528\u78c1\u76d8\u7a7a\u95f4\u7684\u6bd4\u4f8b\u3002 \u4fee\u6539 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u5e76\u66f4\u65b0\u81f3\u6240\u6709\u670d\u52a1\u8282\u70b9\u3002 [jindofsx-storage]# Storage service \u914d\u7f6e storage.data-dirs=/mnt/disk1/jindofsx,/mnt/disk2/jindofsx # \u8868\u793a\u7528\u4ee5\u7f13\u5b58\u6570\u636e\u7684\u78c1\u76d8\u76ee\u5f55 storage.data-dirs.capacities=500G,500G # \u8868\u793a\u6bcf\u4e2a\u78c1\u76d8\u76ee\u5f55\u7684\u603b\u5bb9\u91cf storage.watermark.high.ratio=0.8 # \u8868\u793a\u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0a\u6c34\u4f4d\u6bd4\u4f8b\uff0c\u6bcf\u5757\u6570\u636e\u76d8\u7684 JindoFSx \u6570\u636e\u76ee\u5f55\u5360\u7528\u7684\u78c1\u76d8\u7a7a\u95f4\u5230\u8fbe\u4e0a\u6c34\u4f4d\u5373\u4f1a\u89e6\u53d1\u6e05\u7406\u3002\u9ed8\u8ba4\u503c\uff1a0.8\u3002 storage.watermark.low.ratio=0.6 # \u8868\u793a\u4f7f\u7528\u91cf\u7684\u4e0b\u6c34\u4f4d\u6bd4\u4f8b\uff0c\u89e6\u53d1\u6e05\u7406\u540e\u4f1a\u81ea\u52a8\u6e05\u7406\u51b7\u6570\u636e\uff0c\u5c06JindoFS\u6570\u636e\u76ee\u5f55\u5360\u7528\u7a7a\u95f4\u6e05\u7406\u5230\u4e0b\u6c34\u4f4d\u3002\u9ed8\u8ba4\u503c\uff1a0.6\u3002 \u8bf4\u660e: \u60a8\u53ef\u4ee5\u901a\u8fc7\u8bbe\u7f6e\u4e0a\u6c34\u4f4d\u6bd4\u4f8b\u8c03\u8282\u671f\u671b\u5206\u7ed9 JindoFSx \u7684\u78c1\u76d8\u7a7a\u95f4\uff0c\u4e0b\u6c34\u4f4d\u5fc5\u987b\u5c0f\u4e8e\u4e0a\u6c34\u4f4d\uff0c\u8bbe\u7f6e\u5408\u7406\u7684\u503c\u5373\u53ef\u3002 \u6ce8\u610f: \u914d\u7f6e\u5b8c\u6210\u540e\u9700\u91cd\u542f JindoFSx \u670d\u52a1\u3002","title":"\u78c1\u76d8\u7a7a\u95f4\u6c34\u4f4d\u63a7\u5236"},{"location":"backup/jindofsx/jindofsx_local_cache/","text":"\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u00a7 \u6570\u636e\u7f13\u5b58\u52a0\u901f\u4f5c\u4e3aJindoFSx\u7684\u6838\u5fc3\u529f\u80fd\u4e4b\u4e00\uff0c4.4.0\u7248\u672c\u5f15\u5165\u4e86\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\uff08Local cache\uff09\u7684\u7f13\u5b58\u4f7f\u7528\u6a21\u5f0f\uff0c\u63d0\u4f9b\u4e00\u79cd\u7eaf\u5ba2\u6237\u7aef\u7684\u7f13\u5b58\u52a0\u901f\u80fd\u529b\u3002 \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u7b80\u4ecb \u00a7 JindoFSx\u5ba2\u6237\u7aef\u63d0\u4f9b\u4e86\u4e24\u79cd\u6570\u636e\u7f13\u5b58\u4f7f\u7528\u6a21\u5f0f\uff0c\u5206\u5e03\u5f0f\u7f13\u5b58\uff08Dist cache\uff09\u548c\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\uff08Local cache\uff09\u3002\u5206\u5e03\u5f0f\u7f13\u5b58\u4f5c\u4e3aJindoFSx\u7684\u9ed8\u8ba4\u4f7f\u7528\u6a21\u5f0f\uff0c\u901a\u8fc7\u8bbf\u95eeJindoFSx\u5206\u5e03\u5f0f\u7f13\u5b58\u670d\u52a1\u5b9e\u73b0\u5bf9\u7f13\u5b58\u6570\u636e\u7684\u8bfb\u5199\uff1b 4.4.0\u989d\u5916\u5f15\u5165\u4e86\u672c\u5730\u7f13\u5b58\u7684\u4f7f\u7528\u6a21\u5f0f\uff0c\u5728\u5ba2\u6237\u7aef\u8fdb\u7a0b\u5185\u5d4c\u4e86\u7f13\u5b58\u7ba1\u7406\u6a21\u5757\uff0c\u4e0d\u9700\u8981\u4f9d\u8d56\u5206\u5e03\u5f0f\u670d\u52a1\uff0c\u76f4\u63a5\u5229\u7528\u672c\u5730\u7684\u78c1\u76d8\u6216\u8005\u5185\u5b58\u5bf9\u6570\u636e\u8fdb\u884c\u7f13\u5b58\u52a0\u901f\u3002 \u4f7f\u7528\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58 \u00a7 \u672c\u6587\u4ee5 JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u4e3a\u4f8b\u3002 Hadoop\u5ba2\u6237\u7aef\u4f7f\u7528\u672c\u5730\u7f13\u5b58 \u00a7 \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1 OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06 OSS \u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u914d\u7f6e\u672c\u5730\u7f13\u5b58\u52a8\u6001\u5e93\u8def\u5f84 \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u529f\u80fd\u901a\u8fc7\u63d2\u4ef6\u5f62\u5f0f\u63d0\u4f9b\u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728jindosdk-4.6.12/plugins/\u8def\u5f84\u4e0b\uff0c\u5c06\u8be5\u8def\u5f84\u7684\u7edd\u5bf9\u8def\u5f84\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.jdo.plugin.dir</name> <value>${JINDOSDK_HOME}/plugins</value> </property> </configuration> \u542f\u7528\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58 \u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\uff0c\u542f\u7528\u5e76\u914d\u7f6e\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u7684\u5fc5\u8981\u914d\u7f6e\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u603b\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>false</value> </property> <property> <!-- \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.local.enable</name> <value>false</value> </property> <property> <!-- \u672c\u5730\u7f13\u5b58\u78c1\u76d8\u8def\u5f84 --> <name>fs.jindofsx.data.cache.local.data-dirs</name> <value>/mnt/disk1/localcache,/mnt/disk2/localcache</value> </property> <property> <!-- \u672c\u5730\u7f13\u5b58\u78c1\u76d8\u5927\u5c0f --> <name>fs.jindofsx.data.cache.local.data-dirs.capacities</name> <value>50G,50G</value> </property> <property> <!-- \u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0a\u6c34\u4f4d\u6bd4\u4f8b --> <name>fs.jindofsx.data.cache.local.watermark.high.ratio</name> <value>0.8</value> </property> <property> <!-- \u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0b\u6c34\u4f4d\u6bd4\u4f8b --> <name>fs.jindofsx.data.cache.local.watermark.low.ratio</name> <value>0.4</value> </property> </configuration> Fuse\u5ba2\u6237\u7aef\u4f7f\u7528\u672c\u5730\u7f13\u5b58 \u00a7 JindoFuse \u8be6\u7ec6\u4f7f\u7528\u6587\u6863\u53c2\u8003 JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f OSS \u3002 \u5bf9\u4e8e\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u65e0\u9700\u90e8\u7f72\u542f\u52a8 JindoFSx \u7f13\u5b58\u52a0\u901f\u670d\u52a1\uff0c\u4ec5\u9700\u5ba2\u6237\u7aef\u914d\u7f6e\u672c\u5730\u7f13\u5b58\u76f8\u5173\u53c2\u6570\uff0c\u542f\u52a8fuse\u5373\u53ef\uff0c\u5177\u4f53\u5982\u4e0b\u3002 * \u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf \u914d\u7f6e\u6587\u4ef6 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u914d\u7f6e\u963f\u91cc\u4e91 OSS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou-internal.aliyuncs.com\u3002 fs.oss.endpoint= <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1\u7684 AccessKey ID \u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7 AccessKey \u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable = true # \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u529f\u80fd\u901a\u8fc7\u63d2\u4ef6\u5f62\u5f0f\u63d0\u4f9b\u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728jindosdk-4.6.12/plugins/\u8def\u5f84\u4e0b fs.jdo.plugin.dir=${JINDOSDK_HOME}/plugins # \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58 fs.jindofsx.data.cache.local.enable=true # \u672c\u5730\u7f13\u5b58\u78c1\u76d8\u8def\u5f84 fs.jindofsx.data.cache.local.data-dirs=/mnt/disk1/localcache/,/mnt/disk2/localcache/ # \u672c\u5730\u7f13\u5b58\u78c1\u76d8\u5927\u5c0f fs.jindofsx.data.cache.local.data-dirs.capacities=80G,80G # \u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0a\u6c34\u4f4d\u6bd4\u4f8b fs.jindofsx.data.cache.local.watermark.high.ratio=0.8 # \u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0b\u6c34\u4f4d\u6bd4\u4f8b fs.jindofsx.data.cache.local.watermark.low.ratio=0.4 \u6ce8\u610f\u4e8b\u9879 \u00a7 \u5f53\u524d\u7248\u672c\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u4e3a\u8fdb\u7a0b\u72ec\u5360\u78c1\u76d8\u8def\u5f84\uff0c\u540c\u4e00\u4e2a\u8def\u5f84\u53ea\u80fd\u7531\u4e00\u4e2a\u8fdb\u7a0b\u5360\u7528\uff0c\u4e14\u4e0d\u540c\u8fdb\u7a0b\u95f4\u4e0d\u5171\u4eab\u7f13\u5b58\u6570\u636e\uff1b \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u9002\u7528\u4e8e\u4e00\u4e9b\u5e38\u9a7b\u8fdb\u7a0b\u7684\u670d\u52a1\u4e2d\u4f7f\u7528\uff0c\u8fdb\u7a0b\u72ec\u7acb\u4eab\u6709\u7f13\u5b58\u3002","title":"\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofsx/jindofsx_local_cache/#_1","text":"\u6570\u636e\u7f13\u5b58\u52a0\u901f\u4f5c\u4e3aJindoFSx\u7684\u6838\u5fc3\u529f\u80fd\u4e4b\u4e00\uff0c4.4.0\u7248\u672c\u5f15\u5165\u4e86\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\uff08Local cache\uff09\u7684\u7f13\u5b58\u4f7f\u7528\u6a21\u5f0f\uff0c\u63d0\u4f9b\u4e00\u79cd\u7eaf\u5ba2\u6237\u7aef\u7684\u7f13\u5b58\u52a0\u901f\u80fd\u529b\u3002","title":"\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofsx/jindofsx_local_cache/#_2","text":"JindoFSx\u5ba2\u6237\u7aef\u63d0\u4f9b\u4e86\u4e24\u79cd\u6570\u636e\u7f13\u5b58\u4f7f\u7528\u6a21\u5f0f\uff0c\u5206\u5e03\u5f0f\u7f13\u5b58\uff08Dist cache\uff09\u548c\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\uff08Local cache\uff09\u3002\u5206\u5e03\u5f0f\u7f13\u5b58\u4f5c\u4e3aJindoFSx\u7684\u9ed8\u8ba4\u4f7f\u7528\u6a21\u5f0f\uff0c\u901a\u8fc7\u8bbf\u95eeJindoFSx\u5206\u5e03\u5f0f\u7f13\u5b58\u670d\u52a1\u5b9e\u73b0\u5bf9\u7f13\u5b58\u6570\u636e\u7684\u8bfb\u5199\uff1b 4.4.0\u989d\u5916\u5f15\u5165\u4e86\u672c\u5730\u7f13\u5b58\u7684\u4f7f\u7528\u6a21\u5f0f\uff0c\u5728\u5ba2\u6237\u7aef\u8fdb\u7a0b\u5185\u5d4c\u4e86\u7f13\u5b58\u7ba1\u7406\u6a21\u5757\uff0c\u4e0d\u9700\u8981\u4f9d\u8d56\u5206\u5e03\u5f0f\u670d\u52a1\uff0c\u76f4\u63a5\u5229\u7528\u672c\u5730\u7684\u78c1\u76d8\u6216\u8005\u5185\u5b58\u5bf9\u6570\u636e\u8fdb\u884c\u7f13\u5b58\u52a0\u901f\u3002","title":"\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u7b80\u4ecb"},{"location":"backup/jindofsx/jindofsx_local_cache/#_3","text":"\u672c\u6587\u4ee5 JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u4e3a\u4f8b\u3002","title":"\u4f7f\u7528\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58"},{"location":"backup/jindofsx/jindofsx_local_cache/#hadoop","text":"\u914d\u7f6e OSS \u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1 OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06 OSS \u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u914d\u7f6e\u672c\u5730\u7f13\u5b58\u52a8\u6001\u5e93\u8def\u5f84 \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u529f\u80fd\u901a\u8fc7\u63d2\u4ef6\u5f62\u5f0f\u63d0\u4f9b\u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728jindosdk-4.6.12/plugins/\u8def\u5f84\u4e0b\uff0c\u5c06\u8be5\u8def\u5f84\u7684\u7edd\u5bf9\u8def\u5f84\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.jdo.plugin.dir</name> <value>${JINDOSDK_HOME}/plugins</value> </property> </configuration> \u542f\u7528\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58 \u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\uff0c\u542f\u7528\u5e76\u914d\u7f6e\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u7684\u5fc5\u8981\u914d\u7f6e\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u603b\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>false</value> </property> <property> <!-- \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.local.enable</name> <value>false</value> </property> <property> <!-- \u672c\u5730\u7f13\u5b58\u78c1\u76d8\u8def\u5f84 --> <name>fs.jindofsx.data.cache.local.data-dirs</name> <value>/mnt/disk1/localcache,/mnt/disk2/localcache</value> </property> <property> <!-- \u672c\u5730\u7f13\u5b58\u78c1\u76d8\u5927\u5c0f --> <name>fs.jindofsx.data.cache.local.data-dirs.capacities</name> <value>50G,50G</value> </property> <property> <!-- \u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0a\u6c34\u4f4d\u6bd4\u4f8b --> <name>fs.jindofsx.data.cache.local.watermark.high.ratio</name> <value>0.8</value> </property> <property> <!-- \u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0b\u6c34\u4f4d\u6bd4\u4f8b --> <name>fs.jindofsx.data.cache.local.watermark.low.ratio</name> <value>0.4</value> </property> </configuration>","title":"Hadoop\u5ba2\u6237\u7aef\u4f7f\u7528\u672c\u5730\u7f13\u5b58"},{"location":"backup/jindofsx/jindofsx_local_cache/#fuse","text":"JindoFuse \u8be6\u7ec6\u4f7f\u7528\u6587\u6863\u53c2\u8003 JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f OSS \u3002 \u5bf9\u4e8e\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u65e0\u9700\u90e8\u7f72\u542f\u52a8 JindoFSx \u7f13\u5b58\u52a0\u901f\u670d\u52a1\uff0c\u4ec5\u9700\u5ba2\u6237\u7aef\u914d\u7f6e\u672c\u5730\u7f13\u5b58\u76f8\u5173\u53c2\u6570\uff0c\u542f\u52a8fuse\u5373\u53ef\uff0c\u5177\u4f53\u5982\u4e0b\u3002 * \u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf \u914d\u7f6e\u6587\u4ef6 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u914d\u7f6e\u963f\u91cc\u4e91 OSS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou-internal.aliyuncs.com\u3002 fs.oss.endpoint= <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1\u7684 AccessKey ID \u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7 AccessKey \u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable = true # \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u529f\u80fd\u901a\u8fc7\u63d2\u4ef6\u5f62\u5f0f\u63d0\u4f9b\u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728jindosdk-4.6.12/plugins/\u8def\u5f84\u4e0b fs.jdo.plugin.dir=${JINDOSDK_HOME}/plugins # \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58 fs.jindofsx.data.cache.local.enable=true # \u672c\u5730\u7f13\u5b58\u78c1\u76d8\u8def\u5f84 fs.jindofsx.data.cache.local.data-dirs=/mnt/disk1/localcache/,/mnt/disk2/localcache/ # \u672c\u5730\u7f13\u5b58\u78c1\u76d8\u5927\u5c0f fs.jindofsx.data.cache.local.data-dirs.capacities=80G,80G # \u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0a\u6c34\u4f4d\u6bd4\u4f8b fs.jindofsx.data.cache.local.watermark.high.ratio=0.8 # \u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0b\u6c34\u4f4d\u6bd4\u4f8b fs.jindofsx.data.cache.local.watermark.low.ratio=0.4","title":"Fuse\u5ba2\u6237\u7aef\u4f7f\u7528\u672c\u5730\u7f13\u5b58"},{"location":"backup/jindofsx/jindofsx_local_cache/#_4","text":"\u5f53\u524d\u7248\u672c\u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u4e3a\u8fdb\u7a0b\u72ec\u5360\u78c1\u76d8\u8def\u5f84\uff0c\u540c\u4e00\u4e2a\u8def\u5f84\u53ea\u80fd\u7531\u4e00\u4e2a\u8fdb\u7a0b\u5360\u7528\uff0c\u4e14\u4e0d\u540c\u8fdb\u7a0b\u95f4\u4e0d\u5171\u4eab\u7f13\u5b58\u6570\u636e\uff1b \u5ba2\u6237\u7aef\u672c\u5730\u7f13\u5b58\u9002\u7528\u4e8e\u4e00\u4e9b\u5e38\u9a7b\u8fdb\u7a0b\u7684\u670d\u52a1\u4e2d\u4f7f\u7528\uff0c\u8fdb\u7a0b\u72ec\u7acb\u4eab\u6709\u7f13\u5b58\u3002","title":"\u6ce8\u610f\u4e8b\u9879"},{"location":"backup/jindofsx/jindofsx_metrics/","text":"\u57fa\u4e8e Prometheus + Grafana \u642d\u5efa JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u53ef\u89c6\u5316\u6307\u6807\u89c2\u6d4b\u5e73\u53f0 \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 JindoFSx >= 4.1.0 Prometheus >= 2.10 Grafana >= 7.3.7 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u3002 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK\u3002 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u4ee5\u7b80\u5355\u65b9\u5f0f\u67e5\u770b metrics \u6307\u6807 \u00a7 \u5728\u5df2\u7ecf\u6210\u529f\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7684\u524d\u63d0\u4e0b\uff0c\u627e\u5230\u670d\u52a1 namesapce \u6240\u5728\u8282\u70b9\u7684 ip(localhost) \u548c\u670d\u52a1\u542f\u52a8\u7aef\u53e3 port(8101)\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 curl/wget \u7b49\u547d\u4ee4\u6765\u83b7\u53d6\u5bf9\u5e94\u7684 metrics \u8f93\u51fa\u7ed3\u679c\u3002\u5982 curl localhost:8101/jindo_metrics/prometheus \u6307\u6807\u8be6\u7ec6\u540d\u79f0\u548c\u542b\u4e49\u8bf7\u53c2\u8003\u6587\u7ae0\u672b\u5c3e 1\u3001Prometheus \u5b89\u88c5 \u00a7 \u9996\u5148\u9700\u8981\u5728\u73af\u5883\u91cc\u5b89\u88c5 Prometheus \u548c Grafana \u670d\u52a1\uff0c\u4e0b\u9762\u4ee5 centos7 \u73af\u5883\u4e3a\u4f8b * Prometheus \u76f8\u5173\u4ecb\u7ecd * Grafana \u76f8\u5173\u4ecb\u7ecd 1\u3001\u5b89\u88c5 Prometheus \u670d\u52a1 #\u4e0b\u8f7d\u5b89\u88c5\u5305 wget https://github.com/prometheus/prometheus/releases/download/v2.30.1/prometheus-2.30.1.linux-amd64.tar.gz #\u89e3\u538b\u5b89\u88c5\u5305 tar -zxvf prometheus-2.30.1.linux-amd64.tar.gz #\u91cd\u547d\u540d mv prometheus-2.30.1.linux-amd64 prometheus-server 2\u3001\u4fee\u6539\u89e3\u538b\u5b89\u88c5\u5305\u5185\u7684 prometheus.yml \u6587\u4ef6\uff0c\u66ff\u6362\u4e3a global: scrape_interval: 15s # By default, scrape targets every 15 seconds. # Attach these labels to any time series or alerts when communicating with # external systems (federation, remote storage, Alertmanager). external_labels: monitor: 'codelab-monitor' # A scrape configuration containing exactly one endpoint to scrape: # Here it's Prometheus itself. scrape_configs: # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config. - job_name: 'namespace-prometheus' scrape_interval: 5s metrics_path: /jindo_metrics/prometheus static_configs: - targets: ['<ns Ip>:<ns Port>'] # your ns Ip and ns port eg:localhost:8101 \u7b80\u5355\u4f7f\u7528\u60a8\u53ea\u9700\u586b\u5199 targets: ['<ns Ip>:<ns Port>'] \u5373\u53ef\uff1a \u5982 targets: ['localhost:8101'] \uff0c\u4fdd\u8bc1\u7f51\u7edc\u548c\u7aef\u53e3\u53ef\u8bbf\u95ee 3\u3001\u542f\u52a8 prometheus \u670d\u52a1 #\u5b89\u88c5\u76ee\u5f55\u4e0b cd prometheus-server #\u540e\u53f0\u542f\u52a8server nohup ./prometheus > myout.file 2>&1 & prometheus \u9ed8\u8ba4\u542f\u52a8\u57289090\u7aef\u53e3 $ netstat -ntlp | grep prometheus tcp 0 0 0.0.0.0:9090 0.0.0.0:* LISTEN 17448/./prometheus \u4ece\u6d4f\u89c8\u5668\u6253\u5f00 prometheus UI\u754c\u9762 \u5230\u6b64\uff0cprometheus \u670d\u52a1\u5b89\u88c5\u6210\u529f\uff0c\u63a5\u4e0b\u6765\u6211\u4eec\u901a\u8fc7 Grafana \u670d\u52a1\u6765\u5bf9\u63a5 Prometheus \u6570\u636e\u6e90\u8fdb\u884c metrics \u7684\u5c55\u793a\u5de5\u4f5c 2\u3001Grafana \u5b89\u88c5 \u00a7 1\u3001\u4e0b\u8f7d\u5b89\u88c5 Grafana \u670d\u52a1 # \u4e0b\u8f7d grafana 7.3.7 wget https://dl.grafana.com/enterprise/release/grafana-enterprise-7.3.7.linux-amd64.tar.gz # \u89e3\u538b tar -zxvf grafana-enterprise-7.3.7.linux-amd64.tar.gz 2\u3001\u542f\u52a8 grafana \u670d\u52a1 \u6700\u5c0f\u5316\u542f\u52a8 grafana \u670d\u52a1\uff0c\u5176\u4ed6\u9009\u9879\u53ef\u53c2\u8003 grafana \u5b98\u65b9\u6587\u6863 cd grafana-7.3.7/bin # ls -l \u603b\u7528\u91cf 85416 -rwxr-xr-x 1 root root 25691632 1\u6708 14 2021 grafana-cli -rw-r--r-- 1 root root 33 1\u6708 14 2021 grafana-cli.md5 -rwxr-xr-x 1 root root 61760192 1\u6708 14 2021 grafana-server -rw-r--r-- 1 root root 33 1\u6708 14 2021 grafana-server.md5 \u540e\u53f0\u542f\u52a8 grafana \u670d\u52a1 nohup ./grafana-server > myout.file 2>&1 & \u6253\u5f00 grafana UI \u754c\u9762\uff0c\u9ed8\u8ba4 3000 \u7aef\u53e3 $ netstat -ntlp | grep grafana tcp 0 0 0.0.0.0:3000 0.0.0.0:* LISTEN 27223/./grafan-ser \u5982\u6253\u5f00 localhost:3000 \uff0c\u521d\u59cb\u5316\u7528\u6237\u540d/\u5bc6\u7801\uff1aadmin/admin\uff0c\u70b9\u51fb login \u767b\u9646 \u6dfb\u52a0 prometheus \u6570\u636e\u6e90 \u586b\u5199 Prometheus \u7684 HTTP URL\uff0c\u8fdb\u884c Save&Test\uff0c\u6d4b\u8bd5\u901a\u8fc7\u5373\u4e3a\u6dfb\u52a0\u6210\u529f \u6dfb\u52a0 JindoFS cache \u6a21\u5f0f\u7684 JSON \u5c55\u793a\u6a21\u7248\uff0c \u70b9\u51fb\u8fd9\u91cc\u4e0b\u8f7d JSON \u6587\u4ef6 \uff0c\u4e0b\u8f7d\u5230\u672c\u5730\u540e\uff0c\u70b9\u51fb Upload JSON file \u70b9\u51fb import \u540e\u5373\u53ef\u770b\u5230 cache \u6a21\u5f0f\u7684 UI Prometheus \u6307\u6807\u9884\u89c8(\u66f4\u65b0\u4e2d) \u00a7 \u4e0b\u9762\u5bf9 JindoRuntime \u5f53\u524d\u6536\u96c6\u7684\u6307\u6807\u8fdb\u884c\u9884\u89c8\u548c\u63cf\u8ff0\uff0c\u60a8\u53ef\u4ee5\u57fa\u4e8e\u5df2\u6536\u96c6\u6307\u6807\u5728 Grafana \u4e2d\u8fdb\u884c\u7ec4\u5408\u548c\u56fe\u8868\u5c55\u793a\uff0c\u5982\u60a8\u9700\u8981\u989d\u5916\u7684\u57fa\u672c metrics \u5c55\u793a\uff0c\u8bf7\u5f00 issue \u8054\u7cfb\u6211\u4eec # HELP jindofsx_ns_backend_read_bytes_time_total_window # TYPE jindofsx_ns_backend_read_bytes_time_total_window gauge jindofsx_ns_backend_read_bytes_time_total_window 20367553 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_read_bytes_total # TYPE jindofsx_ns_backend_read_bytes_total gauge jindofsx_ns_backend_read_bytes_total 166703347021 #\u603b\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_bytes_total_window # TYPE jindofsx_ns_backend_read_bytes_total_window gauge jindofsx_ns_backend_read_bytes_total_window 1174671374 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_time_total # TYPE jindofsx_ns_backend_read_time_total gauge jindofsx_ns_backend_read_time_total 8717632094 #\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_readop_num_total # TYPE jindofsx_ns_backend_readop_num_total gauge jindofsx_ns_backend_readop_num_total 181951 #\u5e95\u5c42\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFS \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_local_read_bytes_time_total_window # TYPE jindofsx_ns_local_read_bytes_time_total_window gauge jindofsx_ns_local_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_bytes_total # TYPE jindofsx_ns_local_read_bytes_total gauge jindofsx_ns_local_read_bytes_total 0 #\u603b\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_local_read_bytes_total_window # TYPE jindofsx_ns_local_read_bytes_total_window gauge jindofsx_ns_local_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_time_total # TYPE jindofsx_ns_local_read_time_total gauge jindofsx_ns_local_read_time_total 0 #\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_readop_num_total # TYPE jindofsx_ns_local_readop_num_total gauge jindofsx_ns_local_readop_num_total 0 #\u77ed\u8def\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFS \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_remote_read_bytes_time_total_window # TYPE jindofsx_ns_remote_read_bytes_time_total_window gauge jindofsx_ns_remote_read_bytes_time_total_window 73714663 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_read_bytes_total # TYPE jindofsx_ns_remote_read_bytes_total gauge jindofsx_ns_remote_read_bytes_total 112108045498 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_bytes_total_window # TYPE jindofsx_ns_remote_read_bytes_total_window gauge jindofsx_ns_remote_read_bytes_total_window 13380884043 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_time_total # TYPE jindofsx_ns_remote_read_time_total gauge jindofsx_ns_remote_read_time_total 765272359 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u65f6\u95f4\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_readop_num_total # TYPE jindofsx_ns_remote_readop_num_total gauge jindofsx_ns_remote_readop_num_total 107314 #\u8fdc\u7aef\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFS \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_slicelet_read_bytes_time_total_window # TYPE jindofsx_ns_slicelet_read_bytes_time_total_window gauge jindofsx_ns_slicelet_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_read_bytes_total # TYPE jindofsx_ns_slicelet_read_bytes_total gauge jindofsx_ns_slicelet_read_bytes_total 0 #\u603b\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_bytes_total_window # TYPE jindofsx_ns_slicelet_read_bytes_total_window gauge jindofsx_ns_slicelet_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_time_total # TYPE jindofsx_ns_slicelet_read_time_total gauge jindofsx_ns_slicelet_read_time_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_readop_num_total # TYPE jindofsx_ns_slicelet_readop_num_total gauge jindofsx_ns_slicelet_readop_num_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFS \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_total_disk_cap # TYPE jindofsx_ns_total_disk_cap gauge jindofsx_ns_total_disk_cap 840739848192 #\u78c1\u76d8\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_mem_cap # TYPE jindofsx_ns_total_mem_cap gauge jindofsx_ns_total_mem_cap 0 #\u5185\u5b58\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_stsnodes_num # TYPE jindofsx_ns_total_stsnodes_num gauge jindofsx_ns_total_stsnodes_num 3 #alive worker \u8282\u70b9\u7684\u6570\u91cf # HELP jindofsx_ns_total_used_disk_cap # TYPE jindofsx_ns_total_used_disk_cap gauge jindofsx_ns_total_used_disk_cap 153423446016 #\u5df2\u7528\u78c1\u76d8\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_used_mem_cap # TYPE jindofsx_ns_total_used_mem_cap gauge jindofsx_ns_total_used_mem_cap 0 #\u5df2\u7528\u5185\u5b58\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte","title":"Jindofsx metrics"},{"location":"backup/jindofsx/jindofsx_metrics/#prometheus-grafana-jindofsx","text":"","title":"\u57fa\u4e8e Prometheus + Grafana \u642d\u5efa JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u53ef\u89c6\u5316\u6307\u6807\u89c2\u6d4b\u5e73\u53f0"},{"location":"backup/jindofsx/jindofsx_metrics/#_1","text":"JindoFSx >= 4.1.0 Prometheus >= 2.10 Grafana >= 7.3.7","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/jindofsx/jindofsx_metrics/#_2","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u3002 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK\u3002 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindofsx_metrics/#metrics","text":"\u5728\u5df2\u7ecf\u6210\u529f\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7684\u524d\u63d0\u4e0b\uff0c\u627e\u5230\u670d\u52a1 namesapce \u6240\u5728\u8282\u70b9\u7684 ip(localhost) \u548c\u670d\u52a1\u542f\u52a8\u7aef\u53e3 port(8101)\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 curl/wget \u7b49\u547d\u4ee4\u6765\u83b7\u53d6\u5bf9\u5e94\u7684 metrics \u8f93\u51fa\u7ed3\u679c\u3002\u5982 curl localhost:8101/jindo_metrics/prometheus \u6307\u6807\u8be6\u7ec6\u540d\u79f0\u548c\u542b\u4e49\u8bf7\u53c2\u8003\u6587\u7ae0\u672b\u5c3e","title":"\u4ee5\u7b80\u5355\u65b9\u5f0f\u67e5\u770b metrics \u6307\u6807"},{"location":"backup/jindofsx/jindofsx_metrics/#1prometheus","text":"\u9996\u5148\u9700\u8981\u5728\u73af\u5883\u91cc\u5b89\u88c5 Prometheus \u548c Grafana \u670d\u52a1\uff0c\u4e0b\u9762\u4ee5 centos7 \u73af\u5883\u4e3a\u4f8b * Prometheus \u76f8\u5173\u4ecb\u7ecd * Grafana \u76f8\u5173\u4ecb\u7ecd 1\u3001\u5b89\u88c5 Prometheus \u670d\u52a1 #\u4e0b\u8f7d\u5b89\u88c5\u5305 wget https://github.com/prometheus/prometheus/releases/download/v2.30.1/prometheus-2.30.1.linux-amd64.tar.gz #\u89e3\u538b\u5b89\u88c5\u5305 tar -zxvf prometheus-2.30.1.linux-amd64.tar.gz #\u91cd\u547d\u540d mv prometheus-2.30.1.linux-amd64 prometheus-server 2\u3001\u4fee\u6539\u89e3\u538b\u5b89\u88c5\u5305\u5185\u7684 prometheus.yml \u6587\u4ef6\uff0c\u66ff\u6362\u4e3a global: scrape_interval: 15s # By default, scrape targets every 15 seconds. # Attach these labels to any time series or alerts when communicating with # external systems (federation, remote storage, Alertmanager). external_labels: monitor: 'codelab-monitor' # A scrape configuration containing exactly one endpoint to scrape: # Here it's Prometheus itself. scrape_configs: # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config. - job_name: 'namespace-prometheus' scrape_interval: 5s metrics_path: /jindo_metrics/prometheus static_configs: - targets: ['<ns Ip>:<ns Port>'] # your ns Ip and ns port eg:localhost:8101 \u7b80\u5355\u4f7f\u7528\u60a8\u53ea\u9700\u586b\u5199 targets: ['<ns Ip>:<ns Port>'] \u5373\u53ef\uff1a \u5982 targets: ['localhost:8101'] \uff0c\u4fdd\u8bc1\u7f51\u7edc\u548c\u7aef\u53e3\u53ef\u8bbf\u95ee 3\u3001\u542f\u52a8 prometheus \u670d\u52a1 #\u5b89\u88c5\u76ee\u5f55\u4e0b cd prometheus-server #\u540e\u53f0\u542f\u52a8server nohup ./prometheus > myout.file 2>&1 & prometheus \u9ed8\u8ba4\u542f\u52a8\u57289090\u7aef\u53e3 $ netstat -ntlp | grep prometheus tcp 0 0 0.0.0.0:9090 0.0.0.0:* LISTEN 17448/./prometheus \u4ece\u6d4f\u89c8\u5668\u6253\u5f00 prometheus UI\u754c\u9762 \u5230\u6b64\uff0cprometheus \u670d\u52a1\u5b89\u88c5\u6210\u529f\uff0c\u63a5\u4e0b\u6765\u6211\u4eec\u901a\u8fc7 Grafana \u670d\u52a1\u6765\u5bf9\u63a5 Prometheus \u6570\u636e\u6e90\u8fdb\u884c metrics \u7684\u5c55\u793a\u5de5\u4f5c","title":"1\u3001Prometheus \u5b89\u88c5"},{"location":"backup/jindofsx/jindofsx_metrics/#2grafana","text":"1\u3001\u4e0b\u8f7d\u5b89\u88c5 Grafana \u670d\u52a1 # \u4e0b\u8f7d grafana 7.3.7 wget https://dl.grafana.com/enterprise/release/grafana-enterprise-7.3.7.linux-amd64.tar.gz # \u89e3\u538b tar -zxvf grafana-enterprise-7.3.7.linux-amd64.tar.gz 2\u3001\u542f\u52a8 grafana \u670d\u52a1 \u6700\u5c0f\u5316\u542f\u52a8 grafana \u670d\u52a1\uff0c\u5176\u4ed6\u9009\u9879\u53ef\u53c2\u8003 grafana \u5b98\u65b9\u6587\u6863 cd grafana-7.3.7/bin # ls -l \u603b\u7528\u91cf 85416 -rwxr-xr-x 1 root root 25691632 1\u6708 14 2021 grafana-cli -rw-r--r-- 1 root root 33 1\u6708 14 2021 grafana-cli.md5 -rwxr-xr-x 1 root root 61760192 1\u6708 14 2021 grafana-server -rw-r--r-- 1 root root 33 1\u6708 14 2021 grafana-server.md5 \u540e\u53f0\u542f\u52a8 grafana \u670d\u52a1 nohup ./grafana-server > myout.file 2>&1 & \u6253\u5f00 grafana UI \u754c\u9762\uff0c\u9ed8\u8ba4 3000 \u7aef\u53e3 $ netstat -ntlp | grep grafana tcp 0 0 0.0.0.0:3000 0.0.0.0:* LISTEN 27223/./grafan-ser \u5982\u6253\u5f00 localhost:3000 \uff0c\u521d\u59cb\u5316\u7528\u6237\u540d/\u5bc6\u7801\uff1aadmin/admin\uff0c\u70b9\u51fb login \u767b\u9646 \u6dfb\u52a0 prometheus \u6570\u636e\u6e90 \u586b\u5199 Prometheus \u7684 HTTP URL\uff0c\u8fdb\u884c Save&Test\uff0c\u6d4b\u8bd5\u901a\u8fc7\u5373\u4e3a\u6dfb\u52a0\u6210\u529f \u6dfb\u52a0 JindoFS cache \u6a21\u5f0f\u7684 JSON \u5c55\u793a\u6a21\u7248\uff0c \u70b9\u51fb\u8fd9\u91cc\u4e0b\u8f7d JSON \u6587\u4ef6 \uff0c\u4e0b\u8f7d\u5230\u672c\u5730\u540e\uff0c\u70b9\u51fb Upload JSON file \u70b9\u51fb import \u540e\u5373\u53ef\u770b\u5230 cache \u6a21\u5f0f\u7684 UI","title":"2\u3001Grafana \u5b89\u88c5"},{"location":"backup/jindofsx/jindofsx_metrics/#prometheus","text":"\u4e0b\u9762\u5bf9 JindoRuntime \u5f53\u524d\u6536\u96c6\u7684\u6307\u6807\u8fdb\u884c\u9884\u89c8\u548c\u63cf\u8ff0\uff0c\u60a8\u53ef\u4ee5\u57fa\u4e8e\u5df2\u6536\u96c6\u6307\u6807\u5728 Grafana \u4e2d\u8fdb\u884c\u7ec4\u5408\u548c\u56fe\u8868\u5c55\u793a\uff0c\u5982\u60a8\u9700\u8981\u989d\u5916\u7684\u57fa\u672c metrics \u5c55\u793a\uff0c\u8bf7\u5f00 issue \u8054\u7cfb\u6211\u4eec # HELP jindofsx_ns_backend_read_bytes_time_total_window # TYPE jindofsx_ns_backend_read_bytes_time_total_window gauge jindofsx_ns_backend_read_bytes_time_total_window 20367553 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_read_bytes_total # TYPE jindofsx_ns_backend_read_bytes_total gauge jindofsx_ns_backend_read_bytes_total 166703347021 #\u603b\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_bytes_total_window # TYPE jindofsx_ns_backend_read_bytes_total_window gauge jindofsx_ns_backend_read_bytes_total_window 1174671374 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_time_total # TYPE jindofsx_ns_backend_read_time_total gauge jindofsx_ns_backend_read_time_total 8717632094 #\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_readop_num_total # TYPE jindofsx_ns_backend_readop_num_total gauge jindofsx_ns_backend_readop_num_total 181951 #\u5e95\u5c42\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFS \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_local_read_bytes_time_total_window # TYPE jindofsx_ns_local_read_bytes_time_total_window gauge jindofsx_ns_local_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_bytes_total # TYPE jindofsx_ns_local_read_bytes_total gauge jindofsx_ns_local_read_bytes_total 0 #\u603b\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_local_read_bytes_total_window # TYPE jindofsx_ns_local_read_bytes_total_window gauge jindofsx_ns_local_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_time_total # TYPE jindofsx_ns_local_read_time_total gauge jindofsx_ns_local_read_time_total 0 #\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_readop_num_total # TYPE jindofsx_ns_local_readop_num_total gauge jindofsx_ns_local_readop_num_total 0 #\u77ed\u8def\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFS \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_remote_read_bytes_time_total_window # TYPE jindofsx_ns_remote_read_bytes_time_total_window gauge jindofsx_ns_remote_read_bytes_time_total_window 73714663 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_read_bytes_total # TYPE jindofsx_ns_remote_read_bytes_total gauge jindofsx_ns_remote_read_bytes_total 112108045498 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_bytes_total_window # TYPE jindofsx_ns_remote_read_bytes_total_window gauge jindofsx_ns_remote_read_bytes_total_window 13380884043 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_time_total # TYPE jindofsx_ns_remote_read_time_total gauge jindofsx_ns_remote_read_time_total 765272359 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u65f6\u95f4\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_readop_num_total # TYPE jindofsx_ns_remote_readop_num_total gauge jindofsx_ns_remote_readop_num_total 107314 #\u8fdc\u7aef\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFS \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_slicelet_read_bytes_time_total_window # TYPE jindofsx_ns_slicelet_read_bytes_time_total_window gauge jindofsx_ns_slicelet_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_read_bytes_total # TYPE jindofsx_ns_slicelet_read_bytes_total gauge jindofsx_ns_slicelet_read_bytes_total 0 #\u603b\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_bytes_total_window # TYPE jindofsx_ns_slicelet_read_bytes_total_window gauge jindofsx_ns_slicelet_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_time_total # TYPE jindofsx_ns_slicelet_read_time_total gauge jindofsx_ns_slicelet_read_time_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_readop_num_total # TYPE jindofsx_ns_slicelet_readop_num_total gauge jindofsx_ns_slicelet_readop_num_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFS \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_total_disk_cap # TYPE jindofsx_ns_total_disk_cap gauge jindofsx_ns_total_disk_cap 840739848192 #\u78c1\u76d8\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_mem_cap # TYPE jindofsx_ns_total_mem_cap gauge jindofsx_ns_total_mem_cap 0 #\u5185\u5b58\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_stsnodes_num # TYPE jindofsx_ns_total_stsnodes_num gauge jindofsx_ns_total_stsnodes_num 3 #alive worker \u8282\u70b9\u7684\u6570\u91cf # HELP jindofsx_ns_total_used_disk_cap # TYPE jindofsx_ns_total_used_disk_cap gauge jindofsx_ns_total_used_disk_cap 153423446016 #\u5df2\u7528\u78c1\u76d8\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_used_mem_cap # TYPE jindofsx_ns_total_used_mem_cap gauge jindofsx_ns_total_used_mem_cap 0 #\u5df2\u7528\u5185\u5b58\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte","title":"Prometheus \u6307\u6807\u9884\u89c8(\u66f4\u65b0\u4e2d)"},{"location":"backup/jindofsx/jindofsx_multi_mount/","text":"\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u6df7\u5408\u6302\u8f7d \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u4e86\u5bf9\u591a\u79cd\u6570\u636e\u6e90\u7684\u7f13\u5b58\u52a0\u901f\u529f\u80fd\uff0c\u800c\u4e14\u53ef\u4ee5\u5c06\u4e0d\u540c\u6570\u636e\u6e90\u7edf\u4e00\u7ba1\u7406\uff0c\u653e\u5230\u540c\u4e00\u4e2a\u547d\u540d\u7a7a\u95f4\u4e0b\u7edf\u4e00\u8bbf\u95ee\u3002 \u524d\u63d0 \u00a7 1\u3001\u5df2\u7ecf\u90e8\u7f72 jindofsx \u670d\u52a1\uff0c\u4e14\u5728\u670d\u52a1\u7aef\u914d\u7f6e\u597d\u5404\u79cd\u6570\u636e\u6e90\u7684\u8bbf\u95ee\u5bc6\u94a5\u3002 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u5feb\u901f\u5165\u95e8 2\u3001\u5728 core-site \u91cc\u6dfb\u52a0 jindo \u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u53c2\u6570 <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> \u6302\u8f7d OSS \u6570\u636e\u6e90 \u00a7 1\u3001\u5df2\u7ecf\u5c06 OSS \u76f8\u5173 ak/endpoint \u4fe1\u606f\u914d\u7f6e\u5230 core-site.xml \u91cc \u53ef\u53c2\u8003 JindoFSx \u6302\u8f7d OSS \u6570\u636e\u6e90 2\u3001\u6267\u884c\u6302\u8f7d\u547d\u4ee4 jindo admin -mount /oss oss://<bucket>/<dir> /oss\uff1a\u6302\u8f7d\u5230\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u4e0b\u7684\u8bbf\u95ee\u8def\u5f84 \u540c\u6837\u53ef\u4ee5\u628a\u591a\u4e2a OSS bucket \u6302\u8f7d\u5230\u4e0d\u540c\u7684\u8def\u5f84\u4e0b\u9762\u3002 \u6302\u8f7d OSS-HDFS \u670d\u52a1\u6570\u636e\u6e90 \u00a7 1\u3001\u5df2\u7ecf\u5c06 OSS-HDFS \u8bbf\u95ee\u4fe1\u606f\u914d\u7f6e\u5230 core-site.xml \u91cc \u53ef\u53c2\u8003 JindoFSx \u6302\u8f7d OSS-HDFS \u6570\u636e\u6e90 2\u3001\u6267\u884c\u6302\u8f7d\u547d\u4ee4 jindo admin -mount /oss-hdfs oss://<bucket>/<dir> /oss-hdfs\uff1a\u6302\u8f7d\u5230\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u4e0b\u7684\u8bbf\u95ee\u8def\u5f84 \u540c\u6837\u53ef\u4ee5\u628a\u591a\u4e2a OSS-HDFS bucket \u6302\u8f7d\u5230\u4e0d\u540c\u7684\u8def\u5f84\u4e0b\u9762\u3002 \u4f7f\u7528\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u8bbf\u95ee \u00a7 $ hadoop fs -ls jindo:/// drwxrwxr-x - root root 0 1970-01-01 00:00 jindo:///oss drwxrwxr-x - root root 0 1970-01-01 00:00 jindo:///hdfs $ hadoop fs -ls jindo:///oss/ drwxrwxr-x - root root 0 1970-01-01 00:00 jindo:///oss/test_a_dir -rwxrwxr-x 0 root root 19564111 2022-04-11 10:25 jindo:///oss/tmp.txt $ hadoop fs -ls jindo:///hdfs/ drwxrwxr-x - root root 0 1970-01-01 00:00 jindo:///hdfs/test_a_dir -rwxrwxr-x 0 root root 19564111 2022-04-11 10:25 jindo:///hdfs/tmp.txt \u6211\u4eec\u8fd8\u652f\u6301\u5176\u4ed6\u5176\u4ed6\u6570\u636e\u6e90\u7684\u6302\u8f7d\uff0c\u6bd4\u5982 S3\u3001HDFS\u3001NAS\u7b49\uff0c\u53ef\u53c2\u8003\u4ee5\u4e0b\u6587\u6863 JindoFSx \u6302\u8f7d S3 \u6570\u636e\u6e90 JindoFSx \u6302\u8f7d HDFS \u6570\u636e\u6e90 JindoFSx \u6302\u8f7d NAS \u6570\u636e\u6e90","title":"\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u6df7\u5408\u6302\u8f7d"},{"location":"backup/jindofsx/jindofsx_multi_mount/#_1","text":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u4e86\u5bf9\u591a\u79cd\u6570\u636e\u6e90\u7684\u7f13\u5b58\u52a0\u901f\u529f\u80fd\uff0c\u800c\u4e14\u53ef\u4ee5\u5c06\u4e0d\u540c\u6570\u636e\u6e90\u7edf\u4e00\u7ba1\u7406\uff0c\u653e\u5230\u540c\u4e00\u4e2a\u547d\u540d\u7a7a\u95f4\u4e0b\u7edf\u4e00\u8bbf\u95ee\u3002","title":"\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u6df7\u5408\u6302\u8f7d"},{"location":"backup/jindofsx/jindofsx_multi_mount/#_2","text":"1\u3001\u5df2\u7ecf\u90e8\u7f72 jindofsx \u670d\u52a1\uff0c\u4e14\u5728\u670d\u52a1\u7aef\u914d\u7f6e\u597d\u5404\u79cd\u6570\u636e\u6e90\u7684\u8bbf\u95ee\u5bc6\u94a5\u3002 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u5feb\u901f\u5165\u95e8 2\u3001\u5728 core-site \u91cc\u6dfb\u52a0 jindo \u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u53c2\u6570 <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property>","title":"\u524d\u63d0"},{"location":"backup/jindofsx/jindofsx_multi_mount/#oss","text":"1\u3001\u5df2\u7ecf\u5c06 OSS \u76f8\u5173 ak/endpoint \u4fe1\u606f\u914d\u7f6e\u5230 core-site.xml \u91cc \u53ef\u53c2\u8003 JindoFSx \u6302\u8f7d OSS \u6570\u636e\u6e90 2\u3001\u6267\u884c\u6302\u8f7d\u547d\u4ee4 jindo admin -mount /oss oss://<bucket>/<dir> /oss\uff1a\u6302\u8f7d\u5230\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u4e0b\u7684\u8bbf\u95ee\u8def\u5f84 \u540c\u6837\u53ef\u4ee5\u628a\u591a\u4e2a OSS bucket \u6302\u8f7d\u5230\u4e0d\u540c\u7684\u8def\u5f84\u4e0b\u9762\u3002","title":"\u6302\u8f7d OSS \u6570\u636e\u6e90"},{"location":"backup/jindofsx/jindofsx_multi_mount/#oss-hdfs","text":"1\u3001\u5df2\u7ecf\u5c06 OSS-HDFS \u8bbf\u95ee\u4fe1\u606f\u914d\u7f6e\u5230 core-site.xml \u91cc \u53ef\u53c2\u8003 JindoFSx \u6302\u8f7d OSS-HDFS \u6570\u636e\u6e90 2\u3001\u6267\u884c\u6302\u8f7d\u547d\u4ee4 jindo admin -mount /oss-hdfs oss://<bucket>/<dir> /oss-hdfs\uff1a\u6302\u8f7d\u5230\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u4e0b\u7684\u8bbf\u95ee\u8def\u5f84 \u540c\u6837\u53ef\u4ee5\u628a\u591a\u4e2a OSS-HDFS bucket \u6302\u8f7d\u5230\u4e0d\u540c\u7684\u8def\u5f84\u4e0b\u9762\u3002","title":"\u6302\u8f7d OSS-HDFS \u670d\u52a1\u6570\u636e\u6e90"},{"location":"backup/jindofsx/jindofsx_multi_mount/#_3","text":"$ hadoop fs -ls jindo:/// drwxrwxr-x - root root 0 1970-01-01 00:00 jindo:///oss drwxrwxr-x - root root 0 1970-01-01 00:00 jindo:///hdfs $ hadoop fs -ls jindo:///oss/ drwxrwxr-x - root root 0 1970-01-01 00:00 jindo:///oss/test_a_dir -rwxrwxr-x 0 root root 19564111 2022-04-11 10:25 jindo:///oss/tmp.txt $ hadoop fs -ls jindo:///hdfs/ drwxrwxr-x - root root 0 1970-01-01 00:00 jindo:///hdfs/test_a_dir -rwxrwxr-x 0 root root 19564111 2022-04-11 10:25 jindo:///hdfs/tmp.txt \u6211\u4eec\u8fd8\u652f\u6301\u5176\u4ed6\u5176\u4ed6\u6570\u636e\u6e90\u7684\u6302\u8f7d\uff0c\u6bd4\u5982 S3\u3001HDFS\u3001NAS\u7b49\uff0c\u53ef\u53c2\u8003\u4ee5\u4e0b\u6587\u6863 JindoFSx \u6302\u8f7d S3 \u6570\u636e\u6e90 JindoFSx \u6302\u8f7d HDFS \u6570\u636e\u6e90 JindoFSx \u6302\u8f7d NAS \u6570\u636e\u6e90","title":"\u4f7f\u7528\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u8bbf\u95ee"},{"location":"backup/jindofsx/jindofsx_p2p/","text":"P2P \u5206\u5e03\u5f0f\u4e0b\u8f7d\u7f13\u5b58\u4f7f\u7528 \u00a7 JindoFSx \u5ba2\u6237\u7aef P2P \u53ef\u4ee5\u88ab\u89c6\u4f5c\u4e00\u79cd\u672c\u5730\u7f13\u5b58\uff08LocalCache\uff09\u3002\u4e0e\u539f\u6709\u7684 LocalCache \u76f8\u6bd4\uff0cP2P \u7f13\u5b58\u4e2d\u7684\u672c\u5730\u6570\u636e\u5757\u4f1a\u4f18\u5148\u4ece\u5176\u4ed6\u6301\u6709\u8be5\u6570\u636e\u7684\u5ba2\u6237\u7aef\u62c9\u53d6\uff0c\u53ea\u6709\u65e0\u6cd5\u5411\u5176\u4ed6\u5ba2\u6237\u7aef\u8bf7\u6c42\u65f6\uff0c\u624d\u4f1a\u4ece STS \u6216\u8fdc\u7aef\u8bfb\u53d6\u3002\u4ece\u6982\u5ff5\u4e0a\u6765\u8bb2\uff0c\u53ef\u4ee5\u5c06 P2P \u7f13\u5b58\u89c6\u4f5c \u201c\u5ba2\u6237\u7aef\u4e4b\u95f4\u53ef\u4ee5\u4ea4\u4e92\u4f20\u8f93\u6570\u636e\u201d \u7684 \u201c\u672c\u5730\u7f13\u5b58\u201d\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u670d\u52a1\u7aef\u914d\u7f6e \u00a7 \u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 [jindofsx-common] jindofsx.p2p.tracker.thread.number = 1 # TrackerService \u7684\u5904\u7406\u7ebf\u7a0b\u6570\uff0c\u82e5\u8981\u5f00\u542f P2P \u529f\u80fd\uff0c\u5219\u5fc5\u987b\u8bbe\u7f6e\u5927\u4e8e 1 \u7684\u503c\u3002\u5982\u679c\u5c0f\u4e8e 1\uff0c\u5c31\u4e0d\u4f1a\u521b\u5efa TrackerService\uff0c\u4e5f\u5c31\u4e0d\u4f1a\u5f00\u542f P2P \u529f\u80fd\u3002 jindofsx.p2p.file.prefix = oss://bucket1/data-dir1/,oss://bucket2/data-dir2/ # \u4f7f\u7528 P2P \u4e0b\u8f7d\u7684\u524d\u7f00\u5217\u8868\uff0c\u7528\u534a\u89d2\u9017\u53f7\u9694\u5f00\uff0c\u6587\u4ef6\u8def\u5f84\u53ea\u6709\u5339\u914d\u5230\u5176\u4e2d\u4efb\u4e00\u4e2a\u524d\u7f00\uff0c\u624d\u4f1a\u4ee5 P2P \u65b9\u5f0f\u4e0b\u8f7d\u3002\u5728\u5e94\u7528\u5c42\u4f7f\u7528\u7edf\u4e00\u6302\u8f7d\u8def\u5f84\u8fdb\u884c\u4e0b\u8f7d\u65f6\uff0c\u8fd9\u91cc\u4ecd\u5e94\u914d\u7f6e\u4e3a\u771f\u5b9e\u7684\u5bf9\u8c61\u8def\u5f84\u3002 \u5e76\u5c06\u914d\u7f6e\u6587\u4ef6\u66f4\u65b0\u5230\u6240\u6709\u670d\u52a1\u8282\u70b9\uff08Namespace Service \u4e0e Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97 P2P \u914d\u7f6e\u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u5ba2\u6237\u7aef\u914d\u7f6e \u00a7 \u5728 $JINDOSDK_HOME/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6\uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [jindosdk] # P2P \u4e0b\u8f7d\u6700\u591a\u5360\u7528\u7684\u7f13\u5b58\u5927\u5c0f\uff0c\u5355\u4f4d\u4e3a\u5b57\u8282\uff0c\u9ed8\u8ba4\u4e3a 5GB\uff0c\u6700\u5c0f\u503c\u4e3a 1GB fs.jindofsx.p2p.cache.capacity.limit = 5 * 1024 * 1024 * 1024 # \u4e0b\u8f7d\u5355\u4e2a\u6587\u4ef6\u4f7f\u7528\u7684\u5e76\u53d1\u6570\u3002 fs.jindofsx.p2p.download.parallelism.per.file = 5 # P2P \u4e0b\u8f7d\u4f7f\u7528\u7684\u7ebf\u7a0b\u6c60\u603b\u5927\u5c0f\u3002 fs.jindofsx.p2p.download.thread.pool.size = 5 \u5e76\u5c06\u914d\u7f6e\u66f4\u65b0\u5230\u6240\u9700\u8282\u70b9\u3002 \u5b8c\u6210\u4ee5\u4e0a\u914d\u7f6e\u540e\uff0c\u4f5c\u4e1a\u81ea\u52a8\u4f7f\u7528 P2P \u7684\u65b9\u5f0f\u8bfb\u53d6\u3002","title":"P2P \u5206\u5e03\u5f0f\u4e0b\u8f7d\u7f13\u5b58\u4f7f\u7528"},{"location":"backup/jindofsx/jindofsx_p2p/#p2p","text":"JindoFSx \u5ba2\u6237\u7aef P2P \u53ef\u4ee5\u88ab\u89c6\u4f5c\u4e00\u79cd\u672c\u5730\u7f13\u5b58\uff08LocalCache\uff09\u3002\u4e0e\u539f\u6709\u7684 LocalCache \u76f8\u6bd4\uff0cP2P \u7f13\u5b58\u4e2d\u7684\u672c\u5730\u6570\u636e\u5757\u4f1a\u4f18\u5148\u4ece\u5176\u4ed6\u6301\u6709\u8be5\u6570\u636e\u7684\u5ba2\u6237\u7aef\u62c9\u53d6\uff0c\u53ea\u6709\u65e0\u6cd5\u5411\u5176\u4ed6\u5ba2\u6237\u7aef\u8bf7\u6c42\u65f6\uff0c\u624d\u4f1a\u4ece STS \u6216\u8fdc\u7aef\u8bfb\u53d6\u3002\u4ece\u6982\u5ff5\u4e0a\u6765\u8bb2\uff0c\u53ef\u4ee5\u5c06 P2P \u7f13\u5b58\u89c6\u4f5c \u201c\u5ba2\u6237\u7aef\u4e4b\u95f4\u53ef\u4ee5\u4ea4\u4e92\u4f20\u8f93\u6570\u636e\u201d \u7684 \u201c\u672c\u5730\u7f13\u5b58\u201d\u3002","title":"P2P \u5206\u5e03\u5f0f\u4e0b\u8f7d\u7f13\u5b58\u4f7f\u7528"},{"location":"backup/jindofsx/jindofsx_p2p/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindofsx_p2p/#_2","text":"\u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 [jindofsx-common] jindofsx.p2p.tracker.thread.number = 1 # TrackerService \u7684\u5904\u7406\u7ebf\u7a0b\u6570\uff0c\u82e5\u8981\u5f00\u542f P2P \u529f\u80fd\uff0c\u5219\u5fc5\u987b\u8bbe\u7f6e\u5927\u4e8e 1 \u7684\u503c\u3002\u5982\u679c\u5c0f\u4e8e 1\uff0c\u5c31\u4e0d\u4f1a\u521b\u5efa TrackerService\uff0c\u4e5f\u5c31\u4e0d\u4f1a\u5f00\u542f P2P \u529f\u80fd\u3002 jindofsx.p2p.file.prefix = oss://bucket1/data-dir1/,oss://bucket2/data-dir2/ # \u4f7f\u7528 P2P \u4e0b\u8f7d\u7684\u524d\u7f00\u5217\u8868\uff0c\u7528\u534a\u89d2\u9017\u53f7\u9694\u5f00\uff0c\u6587\u4ef6\u8def\u5f84\u53ea\u6709\u5339\u914d\u5230\u5176\u4e2d\u4efb\u4e00\u4e2a\u524d\u7f00\uff0c\u624d\u4f1a\u4ee5 P2P \u65b9\u5f0f\u4e0b\u8f7d\u3002\u5728\u5e94\u7528\u5c42\u4f7f\u7528\u7edf\u4e00\u6302\u8f7d\u8def\u5f84\u8fdb\u884c\u4e0b\u8f7d\u65f6\uff0c\u8fd9\u91cc\u4ecd\u5e94\u914d\u7f6e\u4e3a\u771f\u5b9e\u7684\u5bf9\u8c61\u8def\u5f84\u3002 \u5e76\u5c06\u914d\u7f6e\u6587\u4ef6\u66f4\u65b0\u5230\u6240\u6709\u670d\u52a1\u8282\u70b9\uff08Namespace Service \u4e0e Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97 P2P \u914d\u7f6e\u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u670d\u52a1\u7aef\u914d\u7f6e"},{"location":"backup/jindofsx/jindofsx_p2p/#_3","text":"\u5728 $JINDOSDK_HOME/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6\uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [jindosdk] # P2P \u4e0b\u8f7d\u6700\u591a\u5360\u7528\u7684\u7f13\u5b58\u5927\u5c0f\uff0c\u5355\u4f4d\u4e3a\u5b57\u8282\uff0c\u9ed8\u8ba4\u4e3a 5GB\uff0c\u6700\u5c0f\u503c\u4e3a 1GB fs.jindofsx.p2p.cache.capacity.limit = 5 * 1024 * 1024 * 1024 # \u4e0b\u8f7d\u5355\u4e2a\u6587\u4ef6\u4f7f\u7528\u7684\u5e76\u53d1\u6570\u3002 fs.jindofsx.p2p.download.parallelism.per.file = 5 # P2P \u4e0b\u8f7d\u4f7f\u7528\u7684\u7ebf\u7a0b\u6c60\u603b\u5927\u5c0f\u3002 fs.jindofsx.p2p.download.thread.pool.size = 5 \u5e76\u5c06\u914d\u7f6e\u66f4\u65b0\u5230\u6240\u9700\u8282\u70b9\u3002 \u5b8c\u6210\u4ee5\u4e0a\u914d\u7f6e\u540e\uff0c\u4f5c\u4e1a\u81ea\u52a8\u4f7f\u7528 P2P \u7684\u65b9\u5f0f\u8bfb\u53d6\u3002","title":"\u5ba2\u6237\u7aef\u914d\u7f6e"},{"location":"backup/jindofsx/jindofsx_quickstart/","text":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u5feb\u901f\u5165\u95e8 \u00a7 JindoFSx \u662f JindoData \u63d0\u4f9b\u7684\u4e91\u6570\u636e\u8bbf\u95ee\u4e2d\u5fc3\uff0c\u4e3a\u7528\u6237\u7ba1\u7406\u4e91\u5e73\u53f0\u5bf9\u4e8e\u4e91\u5b58\u50a8\u7684\u4f7f\u7528\uff0c\u9488\u5bf9\u5404\u79cd\u8bbf\u95ee\u573a\u666f\uff0c\u63d0\u4f9b\u4e00\u7ad9\u5f0f\u7684\u5b58\u50a8\u4f18\u5316\u670d\u52a1\u3002JindoFSx \u53ef\u4ee5\u4e3a\u7528\u6237\u540c\u65f6\u7ba1\u7406\u591a\u4e2a\u540e\u7aef\u5b58\u50a8\u7cfb\u7edf\u3002\u540c\u65f6\uff0cJindoFSx \u4f1a\u4e3a\u4e0a\u5c42\u5e94\u7528\u6784\u5efa\u57fa\u4e8e\u591a\u79cd\u5b58\u50a8\u4ecb\u8d28\u7684\u6570\u636e\u7f13\u5b58\uff0c\u5e76\u89c4\u5212\u9ad8\u6548\u7684\u6570\u636e\u8bbf\u95ee\u7b56\u7565\uff0c\u4fdd\u969c\u7528\u6237\u4f5c\u4e1a\u9ad8\u901f\u3001\u7a33\u5b9a\u8fd0\u884c\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u3002 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK\u3002 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u6587\u4ef6\u4ee5\u5bf9\u8c61\u7684\u5f62\u5f0f\u5b58\u50a8\u5728 OSS \u4e0a\u3002 \u672c\u6587\u4ee5 JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u4e3a\u4f8b\u3002\u5229\u7528\u96c6\u7fa4\u672c\u8eab\u7684\u5b58\u50a8\u8d44\u6e90\u7f13\u5b58 OSS \u6587\u4ef6\uff0c\u4ece\u800c\u52a0\u901f\u4f5c\u4e1a\u8bbf\u95ee OSS\u3002 \u670d\u52a1\u7aef\u914d\u7f6e\u963f\u91cc\u4e91 OSS AccessKey \u00a7 JindoFSx \u82e5\u8981\u63d0\u4f9b\u7f13\u5b58\u52a0\u901f\u670d\u52a1\uff0c\u9700\u914d\u7f6e\u5177\u6709\u8bbf\u95ee OSS \u6743\u9650\u7684 AccessKey\u3002 \u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u3002 [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = oss-cn-xxx-internal.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002 \u5c06\u914d\u7f6e\u6587\u4ef6\u90e8\u7f72\u5230\u6240\u9700\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1 OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06 OSS \u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.jindofsx.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.jindofsx.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.jindofsx.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u5b8c\u6210\u4ee5\u4e0a\u914d\u7f6e\u540e\uff0c\u4f5c\u4e1a\u8bbf\u95ee OSS \u5373\u53ef\u5229\u7528\u4e0a\u7f13\u5b58\uff0cJindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u4e86\u900f\u660e\u7f13\u5b58\u7684\u4f7f\u7528\u65b9\u5f0f\uff0c\u4f5c\u4e1a\u8bbf\u95ee OSS \u7684\u65b9\u5f0f\u65e0\u9700\u505a\u4efb\u4f55\u4fee\u6539\u3002 \u4f5c\u4e1a\u8bfb\u53d6 OSS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002 \u6ce8\u610f\uff1a\u6b64\u914d\u7f6e\u4e3a\u5ba2\u6237\u7aef\u914d\u7f6e\uff0c\u4e0d\u9700\u8981\u91cd\u542f JindoFSx \u670d\u52a1\u3002 \u4f7f\u7528 Prometheus + Grafana \u53ef\u89c6\u5316\u6307\u6807\u89c2\u6d4b\u5e73\u53f0 \u00a7 \u5728\u5b8c\u6210 JindoFSx \u7f13\u5b58\u96c6\u7fa4\u7684\u542f\u52a8\u540e\uff0cJindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u57fa\u4e8e Prometheus + Grafana \u7684\u53ef\u89c6\u5316\u6307\u6807\u89c2\u6d4b\u5e73\u53f0\uff0c\u4f60\u53ef\u4ee5\u53c2\u8003\u5982\u4e0b\u6587\u6863\u8fdb\u884c\u5b89\u88c5\u548c\u4f7f\u7528\u8be5\u529f\u80fd \u57fa\u4e8e Prometheus + Grafana \u7684\u53ef\u89c6\u5316\u6307\u6807\u89c2\u6d4b\u5e73\u53f0 \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u5feb\u901f\u5165\u95e8"},{"location":"backup/jindofsx/jindofsx_quickstart/#jindofsx","text":"JindoFSx \u662f JindoData \u63d0\u4f9b\u7684\u4e91\u6570\u636e\u8bbf\u95ee\u4e2d\u5fc3\uff0c\u4e3a\u7528\u6237\u7ba1\u7406\u4e91\u5e73\u53f0\u5bf9\u4e8e\u4e91\u5b58\u50a8\u7684\u4f7f\u7528\uff0c\u9488\u5bf9\u5404\u79cd\u8bbf\u95ee\u573a\u666f\uff0c\u63d0\u4f9b\u4e00\u7ad9\u5f0f\u7684\u5b58\u50a8\u4f18\u5316\u670d\u52a1\u3002JindoFSx \u53ef\u4ee5\u4e3a\u7528\u6237\u540c\u65f6\u7ba1\u7406\u591a\u4e2a\u540e\u7aef\u5b58\u50a8\u7cfb\u7edf\u3002\u540c\u65f6\uff0cJindoFSx \u4f1a\u4e3a\u4e0a\u5c42\u5e94\u7528\u6784\u5efa\u57fa\u4e8e\u591a\u79cd\u5b58\u50a8\u4ecb\u8d28\u7684\u6570\u636e\u7f13\u5b58\uff0c\u5e76\u89c4\u5212\u9ad8\u6548\u7684\u6570\u636e\u8bbf\u95ee\u7b56\u7565\uff0c\u4fdd\u969c\u7528\u6237\u4f5c\u4e1a\u9ad8\u901f\u3001\u7a33\u5b9a\u8fd0\u884c\u3002","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u5feb\u901f\u5165\u95e8"},{"location":"backup/jindofsx/jindofsx_quickstart/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u3002 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK\u3002 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u6587\u4ef6\u4ee5\u5bf9\u8c61\u7684\u5f62\u5f0f\u5b58\u50a8\u5728 OSS \u4e0a\u3002 \u672c\u6587\u4ee5 JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u4e3a\u4f8b\u3002\u5229\u7528\u96c6\u7fa4\u672c\u8eab\u7684\u5b58\u50a8\u8d44\u6e90\u7f13\u5b58 OSS \u6587\u4ef6\uff0c\u4ece\u800c\u52a0\u901f\u4f5c\u4e1a\u8bbf\u95ee OSS\u3002","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindofsx_quickstart/#oss-accesskey","text":"JindoFSx \u82e5\u8981\u63d0\u4f9b\u7f13\u5b58\u52a0\u901f\u670d\u52a1\uff0c\u9700\u914d\u7f6e\u5177\u6709\u8bbf\u95ee OSS \u6743\u9650\u7684 AccessKey\u3002 \u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u3002 [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = oss-cn-xxx-internal.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002 \u5c06\u914d\u7f6e\u6587\u4ef6\u90e8\u7f72\u5230\u6240\u9700\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002","title":"\u670d\u52a1\u7aef\u914d\u7f6e\u963f\u91cc\u4e91 OSS AccessKey"},{"location":"backup/jindofsx/jindofsx_quickstart/#jindofsx_1","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/jindofsx_quickstart/#jindosdk","text":"\u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1 OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06 OSS \u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.jindofsx.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.jindofsx.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.jindofsx.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u5b8c\u6210\u4ee5\u4e0a\u914d\u7f6e\u540e\uff0c\u4f5c\u4e1a\u8bbf\u95ee OSS \u5373\u53ef\u5229\u7528\u4e0a\u7f13\u5b58\uff0cJindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u4e86\u900f\u660e\u7f13\u5b58\u7684\u4f7f\u7528\u65b9\u5f0f\uff0c\u4f5c\u4e1a\u8bbf\u95ee OSS \u7684\u65b9\u5f0f\u65e0\u9700\u505a\u4efb\u4f55\u4fee\u6539\u3002 \u4f5c\u4e1a\u8bfb\u53d6 OSS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002 \u6ce8\u610f\uff1a\u6b64\u914d\u7f6e\u4e3a\u5ba2\u6237\u7aef\u914d\u7f6e\uff0c\u4e0d\u9700\u8981\u91cd\u542f JindoFSx \u670d\u52a1\u3002","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/jindofsx_quickstart/#prometheus-grafana","text":"\u5728\u5b8c\u6210 JindoFSx \u7f13\u5b58\u96c6\u7fa4\u7684\u542f\u52a8\u540e\uff0cJindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u57fa\u4e8e Prometheus + Grafana \u7684\u53ef\u89c6\u5316\u6307\u6807\u89c2\u6d4b\u5e73\u53f0\uff0c\u4f60\u53ef\u4ee5\u53c2\u8003\u5982\u4e0b\u6587\u6863\u8fdb\u884c\u5b89\u88c5\u548c\u4f7f\u7528\u8be5\u529f\u80fd \u57fa\u4e8e Prometheus + Grafana \u7684\u53ef\u89c6\u5316\u6307\u6807\u89c2\u6d4b\u5e73\u53f0","title":"\u4f7f\u7528 Prometheus + Grafana \u53ef\u89c6\u5316\u6307\u6807\u89c2\u6d4b\u5e73\u53f0"},{"location":"backup/jindofsx/jindofsx_quickstart/#_2","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/jindoshell_jindofsx_howto/","text":"JindoShell CLI \u652f\u6301 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4f7f\u7528\u8bf4\u660e \u00a7 JindoShell CLI \u652f\u6301\u64cd\u4f5c JindoFSx \u6570\u636e\u7f13\u5b58\u3001\u5143\u6570\u636e\u7f13\u5b58\u548c\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u7b49\u547d\u4ee4\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u6570\u636e\u7f13\u5b58\u547d\u4ee4 \u00a7 \u6570\u636e\u7f13\u5b58\u547d\u4ee4\u53ef\u4ee5\u5907\u4efd\u5bf9\u5e94\u8def\u5f84\u7684\u6570\u636e\u81f3\u672c\u96c6\u7fa4\u7684\u78c1\u76d8\uff0c\u4ee5\u4fbf\u4e8e\u540e\u7eed\u53ef\u4ee5\u8bfb\u53d6\u672c\u5730\u6570\u636e\uff0c\u65e0\u9700\u8bfb\u53d6OSS\u7b49\u540e\u7aef\u4e0a\u7684\u6570\u636e\u3002 jindo fs -load -data <options> <path> options \u5404\u79cd\u53ef\u9009\u53c2\u6570 path \u6570\u636e\u7f13\u5b58\u8def\u5f84 \u5176\u4e2d options \u53ef\u9009\u53c2\u6570\u6709: \u53c2\u6570 \u8bf4\u660e -s \u8868\u793a\u7f13\u5b58\u8fc7\u7a0b\u540c\u6b65\u6267\u884c\uff0c\u6253\u5370\u8fdb\u5ea6\u548c\u6267\u884c\u4fe1\u606f\uff0c\u63a8\u8350\u5f00\u542f -replica \u7f13\u5b58\u526f\u672c\u6570\u91cf\uff0c\u9ed8\u8ba4\u7f13\u5b581\u4e2a\u526f\u672c -R \u9012\u5f52\u7f13\u5b58\u6587\u4ef6\uff0c\u5f53 path \u662f\u6587\u4ef6\u5939\u662f\u5f00\u542f -cachelist \u63a5\u53d7\u672c\u5730\u6587\u4ef6\uff0c\u6587\u4ef6\u5185\u5bb9\u4e3acache\u5217\u8868 \u63a8\u8350\u4f7f\u7528\u547d\u4ee4\u7ec4\u5408 jindo fs -load -data -s -R <path> \u5143\u6570\u636e\u7f13\u5b58\u547d\u4ee4 \u00a7 \u5143\u6570\u636e\u7f13\u5b58\u547d\u4ee4\u53ef\u4ee5\u5907\u4efd\u8fdc\u7aef\u6587\u4ef6\u7684\u5143\u6570\u636e\u4fe1\u606f\uff0c\u4ece\u800c\u540e\u7eed\u65e0\u9700\u4eceOSS\u7b49\u540e\u7aef\u8bfb\u53d6\u6587\u4ef6\u5143\u6570\u636e\u4fe1\u606f\u3002 jindo fs -load -meta <options> <path> options \u5404\u79cd\u53ef\u9009\u53c2\u6570 path \u6570\u636e\u7f13\u5b58\u8def\u5f84 \u5176\u4e2d options \u53ef\u9009\u53c2\u6570\u6709: \u53c2\u6570 \u8bf4\u660e -s \u8868\u793a\u5143\u7f13\u5b58\u8fc7\u7a0b\u540c\u6b65\u6267\u884c\uff0c\u6253\u5370\u8fdb\u5ea6\u548c\u6267\u884c\u4fe1\u606f\uff0c\u63a8\u8350\u5f00\u542f -R \u9012\u5f52\u7f13\u5b58\u6587\u4ef6\uff0c\u5f53 path \u662f\u6587\u4ef6\u5939\u662f\u5f00\u542f \u63a8\u8350\u4f7f\u7528\u547d\u4ee4\u7ec4\u5408 jindo fs -load -meta -s -R <path> \u6570\u636e\u7f13\u5b58\u548c\u5143\u6570\u636e\u7f13\u5b58\u53ef\u4ee5\u7ec4\u5408\u4f7f\u7528\uff0c\u5f53\u9700\u8981\u540c\u65f6\u8fdb\u884c\u4e8c\u8005\u7f13\u5b58\u65f6\uff0c\u53ef\u642d\u914d\u4e0a\u9762\u53c2\u6570\u4f7f\u7528\u3002 \u63a8\u8350\u4f7f\u7528\u547d\u4ee4\u7ec4\u5408 jindo fs -load -meta -data -s -R <path> \u901a\u8fc7\u901a\u914d\u7b26\u5339\u914d\u6765\u6307\u5b9a\u7f13\u5b58\u6587\u4ef6 \u00a7 \u901a\u8fc7\u6307\u5b9a\u901a\u914d\u7b26\u6765\u9009\u62e9\u5728\u5177\u4f53\u7684 path \u4e0b\u9700\u8981\u7f13\u5b58\u7684\u6587\u4ef6\uff0c\u7c7b\u4f3c\u767d\u540d\u5355\u529f\u80fd jindo fs -load -meta -data -s -R -filter <wildcard> -dryrun <path> \u53c2\u6570 \u8bf4\u660e -filter \u8868\u793a\u5177\u4f53\u7684\u901a\u914d\u7b26\uff0c\u7528\u6765\u5bf9path\u4e0b\u7684\u6240\u6709\u6587\u4ef6\u8fdb\u884c\u5339\u914d\u3002\u6bd4\u5982 .log,\u5982\u679c\u6709\u591a\u4e2a\u901a\u914d\u7b26\u7528,\u9694\u5f00,\u6bd4\u5982 .log,*.dat -dryrun \u8868\u793a\u5355\u7eaf\u8f93\u51fa\u7b26\u5408\u5339\u914d\u7684\u7ed3\u679c\uff0c\u4e0d\u771f\u6b63\u8fdb\u884c\u7f13\u5b58\uff0c\u7528\u4e8e\u8fdb\u884c\u5224\u65ad\u901a\u914d\u7b26\u662f\u5426\u6b63\u786e -thread \u8868\u793a\u5ba2\u6237\u7aef\u5bf9\u8fc7\u6ee4\u51fa\u6765\u7684\u6587\u4ef6\u8fdb\u884c\u7f13\u5b58\u7684\u7ebf\u7a0b\u6570\uff0c\u9ed8\u8ba4\u662f10\u4e2a \u4f7f\u7528\u793a\u4f8b jindo fs -load -meta -data -s -R -filter *.log oss://bucket/dir \u6e05\u7406\u7f13\u5b58\u547d\u4ee4 \u00a7 uncache\u547d\u4ee4\u53ef\u4ee5\u5220\u9664\u672c\u5730\u96c6\u7fa4\u4e2d\u7684\u672c\u5730\u5907\u4efd\uff0c\u53ea\u5b58\u50a8\u6570\u636e\u5728OSS\u6807\u51c6\u5b58\u50a8\u4e0a\uff0c\u4ee5\u4fbf\u4e8e\u540e\u7eed\u8bfb\u53d6OSS\u4e0a\u7684\u6570\u636e\u3002 jindo fs -uncache <path> \u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u547d\u4ee4 \u00a7 \u6dfb\u52a0\u4e00\u4e2a\u6302\u8f7d\u70b9\u3002 jindo admin -mount <path> <realpath> \u79fb\u9664\u4e00\u4e2a\u6302\u8f7d\u70b9\u3002 jindo admin -unmount <path> \u5176\u4ed6\u547d\u4ee4 \u00a7 \u8f93\u51fa\u5f53\u524dJindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7684\u4e00\u4e9b\u4fe1\u606f\uff0c\u6bd4\u5982\u7f13\u5b58\u5927\u5c0f\uff0c\u7f13\u5b58\u5bb9\u91cf\u7b49\u3002 jindo fs -report \u8f93\u51fa\u683c\u5f0f\u5982\u4e0b\uff1a Namespace Address: 127.0.0.1:8101 Rpc Port: 8101 Started: Mon Jan 10 15:23:51 2022 Version: 4.1.0 Live Nodes: 2 Decommission Nodes: 0 Total Disk Capacity: 438.17GB Used Disk Capacity: 5120.00MB Total MEM Capacity: 4096.00MB Used MEM Capacity: 0B","title":"JindoShell CLI \u652f\u6301 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofsx/jindoshell_jindofsx_howto/#jindoshell-cli-jindofsx","text":"JindoShell CLI \u652f\u6301\u64cd\u4f5c JindoFSx \u6570\u636e\u7f13\u5b58\u3001\u5143\u6570\u636e\u7f13\u5b58\u548c\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u7b49\u547d\u4ee4\u3002","title":"JindoShell CLI \u652f\u6301 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/jindofsx/jindoshell_jindofsx_howto/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindoshell_jindofsx_howto/#_2","text":"\u6570\u636e\u7f13\u5b58\u547d\u4ee4\u53ef\u4ee5\u5907\u4efd\u5bf9\u5e94\u8def\u5f84\u7684\u6570\u636e\u81f3\u672c\u96c6\u7fa4\u7684\u78c1\u76d8\uff0c\u4ee5\u4fbf\u4e8e\u540e\u7eed\u53ef\u4ee5\u8bfb\u53d6\u672c\u5730\u6570\u636e\uff0c\u65e0\u9700\u8bfb\u53d6OSS\u7b49\u540e\u7aef\u4e0a\u7684\u6570\u636e\u3002 jindo fs -load -data <options> <path> options \u5404\u79cd\u53ef\u9009\u53c2\u6570 path \u6570\u636e\u7f13\u5b58\u8def\u5f84 \u5176\u4e2d options \u53ef\u9009\u53c2\u6570\u6709: \u53c2\u6570 \u8bf4\u660e -s \u8868\u793a\u7f13\u5b58\u8fc7\u7a0b\u540c\u6b65\u6267\u884c\uff0c\u6253\u5370\u8fdb\u5ea6\u548c\u6267\u884c\u4fe1\u606f\uff0c\u63a8\u8350\u5f00\u542f -replica \u7f13\u5b58\u526f\u672c\u6570\u91cf\uff0c\u9ed8\u8ba4\u7f13\u5b581\u4e2a\u526f\u672c -R \u9012\u5f52\u7f13\u5b58\u6587\u4ef6\uff0c\u5f53 path \u662f\u6587\u4ef6\u5939\u662f\u5f00\u542f -cachelist \u63a5\u53d7\u672c\u5730\u6587\u4ef6\uff0c\u6587\u4ef6\u5185\u5bb9\u4e3acache\u5217\u8868 \u63a8\u8350\u4f7f\u7528\u547d\u4ee4\u7ec4\u5408 jindo fs -load -data -s -R <path>","title":"\u6570\u636e\u7f13\u5b58\u547d\u4ee4"},{"location":"backup/jindofsx/jindoshell_jindofsx_howto/#_3","text":"\u5143\u6570\u636e\u7f13\u5b58\u547d\u4ee4\u53ef\u4ee5\u5907\u4efd\u8fdc\u7aef\u6587\u4ef6\u7684\u5143\u6570\u636e\u4fe1\u606f\uff0c\u4ece\u800c\u540e\u7eed\u65e0\u9700\u4eceOSS\u7b49\u540e\u7aef\u8bfb\u53d6\u6587\u4ef6\u5143\u6570\u636e\u4fe1\u606f\u3002 jindo fs -load -meta <options> <path> options \u5404\u79cd\u53ef\u9009\u53c2\u6570 path \u6570\u636e\u7f13\u5b58\u8def\u5f84 \u5176\u4e2d options \u53ef\u9009\u53c2\u6570\u6709: \u53c2\u6570 \u8bf4\u660e -s \u8868\u793a\u5143\u7f13\u5b58\u8fc7\u7a0b\u540c\u6b65\u6267\u884c\uff0c\u6253\u5370\u8fdb\u5ea6\u548c\u6267\u884c\u4fe1\u606f\uff0c\u63a8\u8350\u5f00\u542f -R \u9012\u5f52\u7f13\u5b58\u6587\u4ef6\uff0c\u5f53 path \u662f\u6587\u4ef6\u5939\u662f\u5f00\u542f \u63a8\u8350\u4f7f\u7528\u547d\u4ee4\u7ec4\u5408 jindo fs -load -meta -s -R <path> \u6570\u636e\u7f13\u5b58\u548c\u5143\u6570\u636e\u7f13\u5b58\u53ef\u4ee5\u7ec4\u5408\u4f7f\u7528\uff0c\u5f53\u9700\u8981\u540c\u65f6\u8fdb\u884c\u4e8c\u8005\u7f13\u5b58\u65f6\uff0c\u53ef\u642d\u914d\u4e0a\u9762\u53c2\u6570\u4f7f\u7528\u3002 \u63a8\u8350\u4f7f\u7528\u547d\u4ee4\u7ec4\u5408 jindo fs -load -meta -data -s -R <path>","title":"\u5143\u6570\u636e\u7f13\u5b58\u547d\u4ee4"},{"location":"backup/jindofsx/jindoshell_jindofsx_howto/#_4","text":"\u901a\u8fc7\u6307\u5b9a\u901a\u914d\u7b26\u6765\u9009\u62e9\u5728\u5177\u4f53\u7684 path \u4e0b\u9700\u8981\u7f13\u5b58\u7684\u6587\u4ef6\uff0c\u7c7b\u4f3c\u767d\u540d\u5355\u529f\u80fd jindo fs -load -meta -data -s -R -filter <wildcard> -dryrun <path> \u53c2\u6570 \u8bf4\u660e -filter \u8868\u793a\u5177\u4f53\u7684\u901a\u914d\u7b26\uff0c\u7528\u6765\u5bf9path\u4e0b\u7684\u6240\u6709\u6587\u4ef6\u8fdb\u884c\u5339\u914d\u3002\u6bd4\u5982 .log,\u5982\u679c\u6709\u591a\u4e2a\u901a\u914d\u7b26\u7528,\u9694\u5f00,\u6bd4\u5982 .log,*.dat -dryrun \u8868\u793a\u5355\u7eaf\u8f93\u51fa\u7b26\u5408\u5339\u914d\u7684\u7ed3\u679c\uff0c\u4e0d\u771f\u6b63\u8fdb\u884c\u7f13\u5b58\uff0c\u7528\u4e8e\u8fdb\u884c\u5224\u65ad\u901a\u914d\u7b26\u662f\u5426\u6b63\u786e -thread \u8868\u793a\u5ba2\u6237\u7aef\u5bf9\u8fc7\u6ee4\u51fa\u6765\u7684\u6587\u4ef6\u8fdb\u884c\u7f13\u5b58\u7684\u7ebf\u7a0b\u6570\uff0c\u9ed8\u8ba4\u662f10\u4e2a \u4f7f\u7528\u793a\u4f8b jindo fs -load -meta -data -s -R -filter *.log oss://bucket/dir","title":"\u901a\u8fc7\u901a\u914d\u7b26\u5339\u914d\u6765\u6307\u5b9a\u7f13\u5b58\u6587\u4ef6"},{"location":"backup/jindofsx/jindoshell_jindofsx_howto/#_5","text":"uncache\u547d\u4ee4\u53ef\u4ee5\u5220\u9664\u672c\u5730\u96c6\u7fa4\u4e2d\u7684\u672c\u5730\u5907\u4efd\uff0c\u53ea\u5b58\u50a8\u6570\u636e\u5728OSS\u6807\u51c6\u5b58\u50a8\u4e0a\uff0c\u4ee5\u4fbf\u4e8e\u540e\u7eed\u8bfb\u53d6OSS\u4e0a\u7684\u6570\u636e\u3002 jindo fs -uncache <path>","title":"\u6e05\u7406\u7f13\u5b58\u547d\u4ee4"},{"location":"backup/jindofsx/jindoshell_jindofsx_howto/#_6","text":"\u6dfb\u52a0\u4e00\u4e2a\u6302\u8f7d\u70b9\u3002 jindo admin -mount <path> <realpath> \u79fb\u9664\u4e00\u4e2a\u6302\u8f7d\u70b9\u3002 jindo admin -unmount <path>","title":"\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u547d\u4ee4"},{"location":"backup/jindofsx/jindoshell_jindofsx_howto/#_7","text":"\u8f93\u51fa\u5f53\u524dJindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7684\u4e00\u4e9b\u4fe1\u606f\uff0c\u6bd4\u5982\u7f13\u5b58\u5927\u5c0f\uff0c\u7f13\u5b58\u5bb9\u91cf\u7b49\u3002 jindo fs -report \u8f93\u51fa\u683c\u5f0f\u5982\u4e0b\uff1a Namespace Address: 127.0.0.1:8101 Rpc Port: 8101 Started: Mon Jan 10 15:23:51 2022 Version: 4.1.0 Live Nodes: 2 Decommission Nodes: 0 Total Disk Capacity: 438.17GB Used Disk Capacity: 5120.00MB Total MEM Capacity: 4096.00MB Used MEM Capacity: 0B","title":"\u5176\u4ed6\u547d\u4ee4"},{"location":"backup/jindofsx/outline/","text":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7528\u6237\u6587\u6863 \u00a7 \u5feb\u901f\u5165\u95e8 \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u5feb\u901f\u5165\u95e8 \u96c6\u7fa4\u90e8\u7f72 \u00a7 \u5feb\u901f\u90e8\u7f72\u4e00\u4e2a\u7b80\u5355\u7684 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u90e8\u7f72\u9ad8\u53ef\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u57fa\u4e8e Prometheus + Grafana \u642d\u5efa JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u53ef\u89c6\u5316\u6307\u6807\u89c2\u6d4b\u5e73\u53f0 \u57fa\u672c\u529f\u80fd \u00a7 \u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u6df7\u5408\u6302\u8f7d JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u963f\u91cc\u4e91 OSS \u7f13\u5b58\u52a0\u901f \u00a7 \u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f \u963f\u91cc\u4e91 OSS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7f13\u5b58\u52a0\u901f \u00a7 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u900f\u660e\u7f13\u5b58\u52a0\u901f \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f Apache HDFS \u7f13\u5b58\u52a0\u901f \u00a7 Apache HDFS \u900f\u660e\u7f13\u5b58\u52a0\u901f Apache HDFS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f \u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8 NAS \u7f13\u5b58\u52a0\u901f \u00a7 \u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8 NAS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f \u591a\u4e91\u5b58\u50a8\u652f\u6301 [NEW] \u00a7 S3\u3001COS\u3001OBS \u591a\u4e91\u5b58\u50a8\u652f\u6301 JindoFSx \u5ba2\u6237\u7aef\u4fa7 P2P \u52a0\u901f \u00a7 P2P \u5206\u5e03\u5f0f\u4e0b\u8f7d\u7f13\u5b58\u4f7f\u7528\u8bf4\u660e JindoShell CLI \u652f\u6301 \u00a7 JindoShell CLI \u652f\u6301 JindoFSx \u4f7f\u7528\u8bf4\u660e JindoFSx \u652f\u6301 Kerberos + Ranger \u9274\u6743 \u00a7 \u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u6027\u80fd\u6d4b\u8bd5 \u00a7 \u5927\u6570\u636e\u751f\u6001 \u00a7 Hadoop \u7ec4\u4ef6 \u00a7 Hadoop \u8bbf\u95ee\u963f\u91cc\u4e91 OSS + JindoFSx \u900f\u660e\u52a0\u901f Hadoop \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09+ JindoFSx \u900f\u660e\u52a0\u901f Hadoop \u8bbf\u95ee JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e Spark \u7ec4\u4ef6 \u00a7 Spark \u5904\u7406\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Spark \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Spark \u5904\u7406 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e Hive \u7ec4\u4ef6 \u00a7 Hive \u5904\u7406\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Hive \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Hive \u5904\u7406 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e Presto \u7ec4\u4ef6 \u00a7 Presto \u67e5\u8be2\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Presto \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Presto \u67e5\u8be2 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e Impala \u7ec4\u4ef6 \u00a7 Impala \u67e5\u8be2\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Impala \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Impala \u67e5\u8be2 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e HBase \u7ec4\u4ef6 \u00a7 HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 + JindoFSx \u900f\u660e\u52a0\u901f HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 + JindoFSx \u900f\u660e\u52a0\u901f HBase \u4f7f\u7528 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e AI \u751f\u6001 \u00a7 JindoFuse \u4f7f\u7528 \u00a7 JindoFuse \u8bfb\u5199 JindoFSx \u7f13\u5b58(oss://\u8def\u5f84\u6302\u8f7d) JindoFuse \u8bfb\u5199 JindoFSx \u7f13\u5b58(jindo://\u8def\u5f84\u6302\u8f7d) Fluid + JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e91\u539f\u751f \u00a7 Fluid \u7ed3\u5408 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4ecb\u7ecd\u548c\u4f7f\u7528 \u6700\u4f73\u5b9e\u8df5 \u00a7 \u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e \u00a7 \u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e \u5386\u53f2\u7248\u672c \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.6.0 \u7528\u6237\u6587\u6863 \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.5.1 \u7528\u6237\u6587\u6863 \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.5.0 \u7528\u6237\u6587\u6863 \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.4.0 \u7528\u6237\u6587\u6863 \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.3.0 \u7528\u6237\u6587\u6863 \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.2.0 \u7528\u6237\u6587\u6863 \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.1.0 \u7528\u6237\u6587\u6863 \u00a7","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7528\u6237\u6587\u6863"},{"location":"backup/jindofsx/outline/#jindofsx","text":"","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7528\u6237\u6587\u6863"},{"location":"backup/jindofsx/outline/#_1","text":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u5feb\u901f\u5165\u95e8","title":"\u5feb\u901f\u5165\u95e8"},{"location":"backup/jindofsx/outline/#_2","text":"\u5feb\u901f\u90e8\u7f72\u4e00\u4e2a\u7b80\u5355\u7684 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u90e8\u7f72\u9ad8\u53ef\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u57fa\u4e8e Prometheus + Grafana \u642d\u5efa JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u53ef\u89c6\u5316\u6307\u6807\u89c2\u6d4b\u5e73\u53f0","title":"\u96c6\u7fa4\u90e8\u7f72"},{"location":"backup/jindofsx/outline/#_3","text":"\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u6df7\u5408\u6302\u8f7d JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e","title":"\u57fa\u672c\u529f\u80fd"},{"location":"backup/jindofsx/outline/#oss","text":"\u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f \u963f\u91cc\u4e91 OSS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f","title":"\u963f\u91cc\u4e91 OSS \u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/outline/#oss-hdfs-jindofs","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u900f\u660e\u7f13\u5b58\u52a0\u901f \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/outline/#apache-hdfs","text":"Apache HDFS \u900f\u660e\u7f13\u5b58\u52a0\u901f Apache HDFS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f","title":"Apache HDFS \u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/outline/#nas","text":"\u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8 NAS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f","title":"\u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8 NAS \u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/outline/#new","text":"S3\u3001COS\u3001OBS \u591a\u4e91\u5b58\u50a8\u652f\u6301","title":"\u591a\u4e91\u5b58\u50a8\u652f\u6301 [NEW]"},{"location":"backup/jindofsx/outline/#jindofsx-p2p","text":"P2P \u5206\u5e03\u5f0f\u4e0b\u8f7d\u7f13\u5b58\u4f7f\u7528\u8bf4\u660e","title":"JindoFSx \u5ba2\u6237\u7aef\u4fa7 P2P \u52a0\u901f"},{"location":"backup/jindofsx/outline/#jindoshell-cli","text":"JindoShell CLI \u652f\u6301 JindoFSx \u4f7f\u7528\u8bf4\u660e","title":"JindoShell CLI \u652f\u6301"},{"location":"backup/jindofsx/outline/#jindofsx-kerberos-ranger","text":"\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848","title":"JindoFSx \u652f\u6301 Kerberos + Ranger \u9274\u6743"},{"location":"backup/jindofsx/outline/#_4","text":"","title":"\u6027\u80fd\u6d4b\u8bd5"},{"location":"backup/jindofsx/outline/#_5","text":"","title":"\u5927\u6570\u636e\u751f\u6001"},{"location":"backup/jindofsx/outline/#hadoop","text":"Hadoop \u8bbf\u95ee\u963f\u91cc\u4e91 OSS + JindoFSx \u900f\u660e\u52a0\u901f Hadoop \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09+ JindoFSx \u900f\u660e\u52a0\u901f Hadoop \u8bbf\u95ee JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e","title":"Hadoop \u7ec4\u4ef6"},{"location":"backup/jindofsx/outline/#spark","text":"Spark \u5904\u7406\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Spark \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Spark \u5904\u7406 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e","title":"Spark \u7ec4\u4ef6"},{"location":"backup/jindofsx/outline/#hive","text":"Hive \u5904\u7406\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Hive \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Hive \u5904\u7406 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e","title":"Hive \u7ec4\u4ef6"},{"location":"backup/jindofsx/outline/#presto","text":"Presto \u67e5\u8be2\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Presto \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Presto \u67e5\u8be2 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e","title":"Presto \u7ec4\u4ef6"},{"location":"backup/jindofsx/outline/#impala","text":"Impala \u67e5\u8be2\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Impala \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f Impala \u67e5\u8be2 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e","title":"Impala \u7ec4\u4ef6"},{"location":"backup/jindofsx/outline/#hbase","text":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 + JindoFSx \u900f\u660e\u52a0\u901f HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 + JindoFSx \u900f\u660e\u52a0\u901f HBase \u4f7f\u7528 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e","title":"HBase \u7ec4\u4ef6"},{"location":"backup/jindofsx/outline/#ai","text":"","title":"AI \u751f\u6001"},{"location":"backup/jindofsx/outline/#jindofuse","text":"JindoFuse \u8bfb\u5199 JindoFSx \u7f13\u5b58(oss://\u8def\u5f84\u6302\u8f7d) JindoFuse \u8bfb\u5199 JindoFSx \u7f13\u5b58(jindo://\u8def\u5f84\u6302\u8f7d)","title":"JindoFuse \u4f7f\u7528"},{"location":"backup/jindofsx/outline/#fluid-jindofsx","text":"Fluid \u7ed3\u5408 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4ecb\u7ecd\u548c\u4f7f\u7528","title":"Fluid + JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e91\u539f\u751f"},{"location":"backup/jindofsx/outline/#_6","text":"","title":"\u6700\u4f73\u5b9e\u8df5"},{"location":"backup/jindofsx/outline/#jindotable-hive","text":"\u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e","title":"\u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e"},{"location":"backup/jindofsx/outline/#_7","text":"","title":"\u5386\u53f2\u7248\u672c"},{"location":"backup/jindofsx/outline/#jindofsx-460","text":"","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.6.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofsx/outline/#jindofsx-451","text":"","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.5.1 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofsx/outline/#jindofsx-450","text":"","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.5.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofsx/outline/#jindofsx-440","text":"","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.4.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofsx/outline/#jindofsx-430","text":"","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.3.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofsx/outline/#jindofsx-420","text":"","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.2.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofsx/outline/#jindofsx-410","text":"","title":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 4.1.0 \u7528\u6237\u6587\u6863"},{"location":"backup/jindofsx/configuration/jindosdk_configuration_list/","text":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e \u00a7 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- xengine \u5f00\u5173 --> <name>fs.xengine</name> <value>jindofsx</value> </property> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>false</value> </property> <property> <!-- \u5143\u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.meta.cache.enable</name> <value>false</value> </property> <property> <!-- \u5c0f\u6587\u4ef6\u7f13\u5b58\u4f18\u5316\u5f00\u5173 --> <name>fs.jindofsx.slice.cache.enable</name> <value>false</value> </property> <property> <!-- \u5185\u5b58\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.ram.cache.enable</name> <value>false</value> </property> <property> <!-- \u662f\u5426\u76f4\u63a5\u8fde\u63a5\u672c\u5730 storageservice --> <name>fs.jindofsx.storage.connect.enable</name> <value>true</value> </property> <property> <!-- \u77ed\u8def\u8bfb\u5f00\u5173\uff0c\u5728\u7f13\u5b58\u7ed3\u70b9\u4e0e\u5ba2\u6237\u7aef\u5728\u540c\u4e00\u53f0\u673a\u5668\u4e0a\u7684\u65f6\u5019\u53ef\u4ee5\u76f4\u63a5\u8bfb\u672c\u5730\u6587\u4ef6\uff0c\u8981\u6c42\u6253\u5f00\u672c\u5730 storageservice \u8fde\u63a5 --> <name>fs.jindofsx.short.circuit.enable</name> <value>true</value> </property> </configuration> RPC \u4f18\u5316\u76f8\u5173\u53c2\u6570 \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- brpc\u7ebf\u7a0b\u6c60\u5927\u5c0f --> <name>fs.jindofsx.rpc.backup.threads</name> <value>100</value> </property> <property> <!-- \u5e76\u53d1\u7ebf\u7a0b\u6570\u91cf --> <name>fs.jindofsx.rpc.thread.concurrency</name> <value>10</value> </property> <property> <!-- \u6d88\u606f\u4f53\u5927\u5c0f --> <name>fs.jindofsx.rpc.body.size</name> <value>256 * 1024 * 1024</value> </property> <property> <!-- \u8d85\u65f6\u65f6\u95f4\u5927\u5c0f\uff08\u6beb\u79d2\uff09--> <name>fs.jindofsx.rpc.timeout</name> <value>12 * 60 * 60 * 1000</value> </property> <property> <!-- \u6d88\u606f\u961f\u5217\u5927\u5c0f --> <name>fs.jindofsx.rpc.max.unwritten.size</name> <value>256 * 1024 * 1024</value> </property> </configuration> \u7f13\u5b58 Metrics \u76f8\u5173\u53c2\u6570 \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u5ba2\u6237\u7aefmetrics\u5f00\u5173 --> <name>fs.jindofsx.client.metrics.enable</name> <value>false</value> </property> </configuration> \u3010\u6ce8\uff1a4.3.0 \u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u3011","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/jindofsx/configuration/jindosdk_configuration_list/#jindosdk","text":"","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/jindofsx/configuration/jindosdk_configuration_list/#_1","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- xengine \u5f00\u5173 --> <name>fs.xengine</name> <value>jindofsx</value> </property> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>false</value> </property> <property> <!-- \u5143\u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.meta.cache.enable</name> <value>false</value> </property> <property> <!-- \u5c0f\u6587\u4ef6\u7f13\u5b58\u4f18\u5316\u5f00\u5173 --> <name>fs.jindofsx.slice.cache.enable</name> <value>false</value> </property> <property> <!-- \u5185\u5b58\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.ram.cache.enable</name> <value>false</value> </property> <property> <!-- \u662f\u5426\u76f4\u63a5\u8fde\u63a5\u672c\u5730 storageservice --> <name>fs.jindofsx.storage.connect.enable</name> <value>true</value> </property> <property> <!-- \u77ed\u8def\u8bfb\u5f00\u5173\uff0c\u5728\u7f13\u5b58\u7ed3\u70b9\u4e0e\u5ba2\u6237\u7aef\u5728\u540c\u4e00\u53f0\u673a\u5668\u4e0a\u7684\u65f6\u5019\u53ef\u4ee5\u76f4\u63a5\u8bfb\u672c\u5730\u6587\u4ef6\uff0c\u8981\u6c42\u6253\u5f00\u672c\u5730 storageservice \u8fde\u63a5 --> <name>fs.jindofsx.short.circuit.enable</name> <value>true</value> </property> </configuration>","title":"\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570"},{"location":"backup/jindofsx/configuration/jindosdk_configuration_list/#rpc","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- brpc\u7ebf\u7a0b\u6c60\u5927\u5c0f --> <name>fs.jindofsx.rpc.backup.threads</name> <value>100</value> </property> <property> <!-- \u5e76\u53d1\u7ebf\u7a0b\u6570\u91cf --> <name>fs.jindofsx.rpc.thread.concurrency</name> <value>10</value> </property> <property> <!-- \u6d88\u606f\u4f53\u5927\u5c0f --> <name>fs.jindofsx.rpc.body.size</name> <value>256 * 1024 * 1024</value> </property> <property> <!-- \u8d85\u65f6\u65f6\u95f4\u5927\u5c0f\uff08\u6beb\u79d2\uff09--> <name>fs.jindofsx.rpc.timeout</name> <value>12 * 60 * 60 * 1000</value> </property> <property> <!-- \u6d88\u606f\u961f\u5217\u5927\u5c0f --> <name>fs.jindofsx.rpc.max.unwritten.size</name> <value>256 * 1024 * 1024</value> </property> </configuration>","title":"RPC \u4f18\u5316\u76f8\u5173\u53c2\u6570"},{"location":"backup/jindofsx/configuration/jindosdk_configuration_list/#metrics","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u5ba2\u6237\u7aefmetrics\u5f00\u5173 --> <name>fs.jindofsx.client.metrics.enable</name> <value>false</value> </property> </configuration> \u3010\u6ce8\uff1a4.3.0 \u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u3011","title":"\u7f13\u5b58 Metrics \u76f8\u5173\u53c2\u6570"},{"location":"backup/jindofsx/configuration/jindosdk_configuration_list_ini/","text":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e \u00a7 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 jindosdk.cfg \u4e2d\u3002 # \u9009\u62e9 jindofsx \u4f5c\u4e3a xengine \u5b9e\u73b0 fs.xengine=jindofsx # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable=false # \u5143\u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.meta.cache.enable=false # \u5c0f\u6587\u4ef6\u7f13\u5b58\u4f18\u5316\u5f00\u5173 fs.jindofsx.slice.cache.enable=false # \u5185\u5b58\u7f13\u5b58\u5f00\u5173 fs.jindofsx.ram.cache.enable=false # \u662f\u5426\u76f4\u63a5\u8fde\u63a5\u672c\u5730 storageservice fs.jindofsx.storage.connect.enable=true # \u77ed\u8def\u8bfb\u5f00\u5173\uff0c\u5728\u7f13\u5b58\u7ed3\u70b9\u4e0e\u5ba2\u6237\u7aef\u5728\u540c\u4e00\u53f0\u673a\u5668\u4e0a\u7684\u65f6\u5019\u53ef\u4ee5\u76f4\u63a5\u8bfb\u672c\u5730\u6587\u4ef6\uff0c\u8981\u6c42\u6253\u5f00\u672c\u5730 storageservice \u8fde\u63a5 fs.jindofsx.short.circuit.enable=true RPC \u4f18\u5316\u76f8\u5173\u53c2\u6570 \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 jindosdk.cfg \u4e2d\u3002 # brpc\u7ebf\u7a0b\u6c60\u5927\u5c0f fs.jindofsx.rpc.backup.threads=100 # \u5e76\u53d1\u7ebf\u7a0b\u6570\u91cf fs.jindofsx.rpc.thread.concurrency=10 # \u6d88\u606f\u4f53\u5927\u5c0f fs.jindofsx.rpc.body.size=268435456 # \u8d85\u65f6\u65f6\u95f4\u5927\u5c0f\uff08\u6beb\u79d2\uff09 fs.jindofsx.rpc.timeout=43200000 # \u6d88\u606f\u961f\u5217\u5927\u5c0f fs.jindofsx.rpc.max.unwritten.size=268435456 \u7f13\u5b58 Metrics \u76f8\u5173\u53c2\u6570 \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 jindosdk.cfg \u4e2d\u3002 # \u5ba2\u6237\u7aefmetrics\u5f00\u5173 fs.jindofsx.client.metrics.enable=false \u3010\u6ce8\uff1a4.3.0 \u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u3011","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/jindofsx/configuration/jindosdk_configuration_list_ini/#jindosdk","text":"","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/jindofsx/configuration/jindosdk_configuration_list_ini/#_1","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 jindosdk.cfg \u4e2d\u3002 # \u9009\u62e9 jindofsx \u4f5c\u4e3a xengine \u5b9e\u73b0 fs.xengine=jindofsx # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable=false # \u5143\u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.meta.cache.enable=false # \u5c0f\u6587\u4ef6\u7f13\u5b58\u4f18\u5316\u5f00\u5173 fs.jindofsx.slice.cache.enable=false # \u5185\u5b58\u7f13\u5b58\u5f00\u5173 fs.jindofsx.ram.cache.enable=false # \u662f\u5426\u76f4\u63a5\u8fde\u63a5\u672c\u5730 storageservice fs.jindofsx.storage.connect.enable=true # \u77ed\u8def\u8bfb\u5f00\u5173\uff0c\u5728\u7f13\u5b58\u7ed3\u70b9\u4e0e\u5ba2\u6237\u7aef\u5728\u540c\u4e00\u53f0\u673a\u5668\u4e0a\u7684\u65f6\u5019\u53ef\u4ee5\u76f4\u63a5\u8bfb\u672c\u5730\u6587\u4ef6\uff0c\u8981\u6c42\u6253\u5f00\u672c\u5730 storageservice \u8fde\u63a5 fs.jindofsx.short.circuit.enable=true","title":"\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570"},{"location":"backup/jindofsx/configuration/jindosdk_configuration_list_ini/#rpc","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 jindosdk.cfg \u4e2d\u3002 # brpc\u7ebf\u7a0b\u6c60\u5927\u5c0f fs.jindofsx.rpc.backup.threads=100 # \u5e76\u53d1\u7ebf\u7a0b\u6570\u91cf fs.jindofsx.rpc.thread.concurrency=10 # \u6d88\u606f\u4f53\u5927\u5c0f fs.jindofsx.rpc.body.size=268435456 # \u8d85\u65f6\u65f6\u95f4\u5927\u5c0f\uff08\u6beb\u79d2\uff09 fs.jindofsx.rpc.timeout=43200000 # \u6d88\u606f\u961f\u5217\u5927\u5c0f fs.jindofsx.rpc.max.unwritten.size=268435456","title":"RPC \u4f18\u5316\u76f8\u5173\u53c2\u6570"},{"location":"backup/jindofsx/configuration/jindosdk_configuration_list_ini/#metrics","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 jindosdk.cfg \u4e2d\u3002 # \u5ba2\u6237\u7aefmetrics\u5f00\u5173 fs.jindofsx.client.metrics.enable=false \u3010\u6ce8\uff1a4.3.0 \u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u3011","title":"\u7f13\u5b58 Metrics \u76f8\u5173\u53c2\u6570"},{"location":"backup/jindofsx/deploy/deploy_jindofsx/","text":"\u5feb\u901f\u90e8\u7f72\u4e00\u4e2a\u7b80\u5355\u7684 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u00a7 1. \u4e0b\u8f7d JindoFSx \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 Release \u5305 jindofsx-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u89e3\u538b \u00a7 tar -xzvf jindofsx-x.x.x.tar.gz \u5e76\u5c06\u5b89\u88c5\u76ee\u5f55\u90e8\u7f72\u5230\u6240\u6709\u6240\u9700\u8282\u70b9\u4e0a\u3002 3. \u914d\u7f6e jindofsx.cfg \u00a7 \u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002\u5e76\u5c06\u914d\u7f6e\u6587\u4ef6\u90e8\u7f72\u5230\u6240\u9700\u8282\u70b9\u4e0a\uff08NamespaceService \u548c StorageService \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] logger.jnilogger=false # jni log \u5f00\u5173 metric.report.on = true # metrics \u5f00\u5173 logger.dir = /tmp/jindofsx-log # \u65e5\u5fd7\u76ee\u5f55 [jindofsx-storage]# Storage service \u914d\u7f6e storage.rpc.port=6101 # Storage service \u7aef\u53e3 storage.data-dirs = /mnt/disk1/jindofsx,/mnt/disk2/jindofsx,/mnt/disk3/jindofsx,/mnt/disk4/jindofsx # \u7f13\u5b58\u78c1\u76d8\u76ee\u5f55 storage.data-dirs.capacities = 527371075584,527371075584,527371075584,527371075584 # \u7f13\u5b58\u78c1\u76d8\u5927\u5c0f storage.namespace.rpc.address = emr-header-1:8101 # Namespace service \u5730\u5740\u53ca\u7aef\u53e3 storage.watermark.high.ratio=0.4 # \u7f13\u5b58\u4e0a\u6c34\u4f4d storage.watermark.low.ratio=0.2 # \u7f13\u5b58\u4e0b\u6c34\u4f4d [jindofsx-namespace]# Namespace service \u914d\u7f6e namespace.rpc.port = 8101 # Namespace service \u76d1\u542c\u7aef\u53e3 namespace.meta-dir = /tmp/jindofsx/server # \u5143\u6570\u636e\u5b58\u653e\u76ee\u5f55 \u5fc5\u8981\u914d\u7f6e\u8bf4\u660e\uff1a 1. [jindofsx-storage] \u4e2d\u7684\u53c2\u6570 storage.namespace.rpc.address \u4fee\u6539\u4e3a JindoFSx \u96c6\u7fa4\u7684 NamespaceService \u670d\u52a1\u7684\u6240\u5728\u8282\u70b9\u5730\u5740\uff08hostname \u6216\u8005 ip \u5747\u53ef\uff09\uff0c\u5176\u4e2d 8101 \u4e3a NamespaceService \u670d\u52a1 RPC \u7684\u7aef\u53e3\u53f7\uff0c \u9700\u8981\u4e0e [jindofsx-namespace] \u4e2d\u7684\u53c2\u6570 namespace.rpc.port \u4e2d\u7684\u7aef\u53e3\u53f7\u4e00\u81f4\uff1b 2. [jindofsx-storage] \u4e2d\u9700\u914d\u7f6e\u597d\u6570\u636e\u7f13\u5b58\u76f8\u5173\u53c2\u6570\uff0cstorage.data-dirs \u4e3a\u7f13\u5b58\u76ee\u5f55\uff0c\u53ef\u4ee5\u6307\u5b9a\u591a\u4e2a\uff0c\u4ee5\u9017\u53f7\u9694\u5f00\uff1bstorage.data-dirs.capacities \u4e3a\u6bcf\u5757\u78c1\u76d8\u5927\u5c0f\uff0c\u6570\u91cf\u5e94\u4e0e storage.data-dirs \u4e00\u81f4\uff1bstorage.watermark.high.ratio \u8868\u793a\u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0a\u6c34\u4f4d\u6bd4\u4f8b\uff080\u52301\u4e4b\u95f4\u7684\u5c0f\u6570\uff09\uff0c\u6bcf\u5757\u6570\u636e\u76d8\u7684\u7f13\u5b58\u6570\u636e\u76ee\u5f55\u5360\u7528\u7684\u78c1\u76d8\u7a7a\u95f4\u5230\u8fbe\u4e0a\u6c34\u4f4d\u5373\u4f1a\u89e6\u53d1\u6e05\u7406\uff0c\u5e76\u4e14 StorageService \u4f1a\u786e\u4fdd\u7f13\u5b58\u7a7a\u95f4\u5360\u7528\u4e0d\u8d85\u8fc7\u4e0a\u6c34\u4f4d\uff1bstorage.watermark.low.ratio \u8868\u793a\u4f7f\u7528\u91cf\u7684\u4e0b\u6c34\u4f4d\u6bd4\u4f8b\uff080\u52301\u4e4b\u95f4\u7684\u5c0f\u6570\uff0c\u9700\u5c0f\u4e8e\u4e0a\u6c34\u4f4d\uff09\uff0c\u89e6\u53d1\u6e05\u7406\u540e\u4f1a\u81ea\u52a8\u6e05\u7406\u51b7\u6570\u636e\uff0c\u5c06\u7f13\u5b58\u6570\u636e\u76ee\u5f55\u5360\u7528\u7a7a\u95f4\u6e05\u7406\u5230\u4e0b\u6c34\u4f4d\u3002 3. [jindofsx-common] \u4e2d\u9700\u914d\u7f6e\u65e5\u5fd7\u3001metrics\u3001credentials\u3001endpoint \u76f8\u5173\u4fe1\u606f\uff0c\u52a0\u901f\u5e95\u5c42\u5b58\u50a8\u6240\u9700\u8981\u7684 credentials \u548c endpoint \u914d\u7f6e\u65b9\u6cd5\u89c1\u76f8\u5e94 credential provider \u914d\u7f6e\u6587\u6863 \u3002 4. \u914d\u7f6e\u73af\u5883\u53d8\u91cf \u00a7 \u4ee5 jindofsx-4.6.12 \u5b89\u88c5\u5728 /usr/lib \u4e3a\u4f8b\uff1a export JINDOFSX_CONF_DIR=/usr/lib/jindofsx-4.6.12/conf/ \u5e76\u786e\u8ba4\u90e8\u7f72\u5230\u6240\u6709\u8282\u70b9\u3002 5. \u542f\u52a8 JindoFSx \u7f13\u5b58\u670d\u52a1 \u00a7 \u5728 master \u8282\u70b9\uff08\u542f\u52a8 JindoFSx NamespaceService \u7684\u8282\u70b9\uff09\u7f16\u8f91 sbin/nodes \u6587\u4ef6\uff0c\u914d\u7f6e\u4e0a\u6240\u6709\u9700\u8981\u542f\u52a8 StorageService \u7684\u8282\u70b9\u5217\u8868\uff0c\u4f8b\u5982\uff1a worker-1 worker-2 worker-3 \u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\uff08\u6ce8\u610f\uff1a\u7531\u4e8e\u811a\u672c\u901a\u8fc7 pssh \u547d\u4ee4\u6279\u91cf\u6267\u884c\u8fdc\u7a0b\u8282\u70b9\u7684\u670d\u52a1\u542f\u505c\uff0c\u9700\u8981\u4fdd\u8bc1 master \u8282\u70b9\u5230\u6240\u6709\u5176\u4ed6\u8282\u70b9 ssh \u7684\u514d\u5bc6\u8fde\u901a\uff09 cd jindofsx-x.x.x sh sbin/start-service.sh \u6267\u884c\u5b8c\u6210\u540e\uff0cmaster \u8282\u70b9\u5c06\u4f1a\u542f\u52a8 Namespace Service\uff0cnodes \u914d\u7f6e\u7684\u6240\u6709\u8282\u70b9\u5c06\u4f1a\u542f\u52a8 StorageService\u3002 \u53e6\u5916\uff0c\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u547d\u4ee4\u505c\u6b62\u6240\u6709\u7f13\u5b58\u670d\u52a1\uff1a cd jindofsx-x.x.x sh sbin/stop-service.sh 6. \u7f13\u5b58\u670d\u52a1\u72b6\u6001\u68c0\u67e5 \u00a7 \u7f13\u5b58\u670d\u52a1\u7684\u8fdb\u7a0b\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u547d\u4ee4\u67e5\u770b\uff0c\u65e5\u5fd7\u6587\u4ef6\u53ef\u5728 logger.dir \u914d\u7f6e\u7684\u76ee\u5f55\u4e0b\u67e5\u770b ps -aux | grep jindofsx-namespaceservice ps -aux | grep jindofsx-storageservice \u5982\u9700\u90e8\u7f72\u9ad8\u53ef\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf, \u8bf7\u53c2\u8003 \u90e8\u7f72\u9ad8\u53ef\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf","title":"\u5feb\u901f\u90e8\u7f72\u4e00\u4e2a\u7b80\u5355\u7684 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf"},{"location":"backup/jindofsx/deploy/deploy_jindofsx/#jindofsx","text":"","title":"\u5feb\u901f\u90e8\u7f72\u4e00\u4e2a\u7b80\u5355\u7684 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf"},{"location":"backup/jindofsx/deploy/deploy_jindofsx/#1-jindofsx","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 Release \u5305 jindofsx-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoFSx \u5305"},{"location":"backup/jindofsx/deploy/deploy_jindofsx/#2","text":"tar -xzvf jindofsx-x.x.x.tar.gz \u5e76\u5c06\u5b89\u88c5\u76ee\u5f55\u90e8\u7f72\u5230\u6240\u6709\u6240\u9700\u8282\u70b9\u4e0a\u3002","title":"2. \u89e3\u538b"},{"location":"backup/jindofsx/deploy/deploy_jindofsx/#3-jindofsxcfg","text":"\u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002\u5e76\u5c06\u914d\u7f6e\u6587\u4ef6\u90e8\u7f72\u5230\u6240\u9700\u8282\u70b9\u4e0a\uff08NamespaceService \u548c StorageService \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] logger.jnilogger=false # jni log \u5f00\u5173 metric.report.on = true # metrics \u5f00\u5173 logger.dir = /tmp/jindofsx-log # \u65e5\u5fd7\u76ee\u5f55 [jindofsx-storage]# Storage service \u914d\u7f6e storage.rpc.port=6101 # Storage service \u7aef\u53e3 storage.data-dirs = /mnt/disk1/jindofsx,/mnt/disk2/jindofsx,/mnt/disk3/jindofsx,/mnt/disk4/jindofsx # \u7f13\u5b58\u78c1\u76d8\u76ee\u5f55 storage.data-dirs.capacities = 527371075584,527371075584,527371075584,527371075584 # \u7f13\u5b58\u78c1\u76d8\u5927\u5c0f storage.namespace.rpc.address = emr-header-1:8101 # Namespace service \u5730\u5740\u53ca\u7aef\u53e3 storage.watermark.high.ratio=0.4 # \u7f13\u5b58\u4e0a\u6c34\u4f4d storage.watermark.low.ratio=0.2 # \u7f13\u5b58\u4e0b\u6c34\u4f4d [jindofsx-namespace]# Namespace service \u914d\u7f6e namespace.rpc.port = 8101 # Namespace service \u76d1\u542c\u7aef\u53e3 namespace.meta-dir = /tmp/jindofsx/server # \u5143\u6570\u636e\u5b58\u653e\u76ee\u5f55 \u5fc5\u8981\u914d\u7f6e\u8bf4\u660e\uff1a 1. [jindofsx-storage] \u4e2d\u7684\u53c2\u6570 storage.namespace.rpc.address \u4fee\u6539\u4e3a JindoFSx \u96c6\u7fa4\u7684 NamespaceService \u670d\u52a1\u7684\u6240\u5728\u8282\u70b9\u5730\u5740\uff08hostname \u6216\u8005 ip \u5747\u53ef\uff09\uff0c\u5176\u4e2d 8101 \u4e3a NamespaceService \u670d\u52a1 RPC \u7684\u7aef\u53e3\u53f7\uff0c \u9700\u8981\u4e0e [jindofsx-namespace] \u4e2d\u7684\u53c2\u6570 namespace.rpc.port \u4e2d\u7684\u7aef\u53e3\u53f7\u4e00\u81f4\uff1b 2. [jindofsx-storage] \u4e2d\u9700\u914d\u7f6e\u597d\u6570\u636e\u7f13\u5b58\u76f8\u5173\u53c2\u6570\uff0cstorage.data-dirs \u4e3a\u7f13\u5b58\u76ee\u5f55\uff0c\u53ef\u4ee5\u6307\u5b9a\u591a\u4e2a\uff0c\u4ee5\u9017\u53f7\u9694\u5f00\uff1bstorage.data-dirs.capacities \u4e3a\u6bcf\u5757\u78c1\u76d8\u5927\u5c0f\uff0c\u6570\u91cf\u5e94\u4e0e storage.data-dirs \u4e00\u81f4\uff1bstorage.watermark.high.ratio \u8868\u793a\u78c1\u76d8\u4f7f\u7528\u91cf\u7684\u4e0a\u6c34\u4f4d\u6bd4\u4f8b\uff080\u52301\u4e4b\u95f4\u7684\u5c0f\u6570\uff09\uff0c\u6bcf\u5757\u6570\u636e\u76d8\u7684\u7f13\u5b58\u6570\u636e\u76ee\u5f55\u5360\u7528\u7684\u78c1\u76d8\u7a7a\u95f4\u5230\u8fbe\u4e0a\u6c34\u4f4d\u5373\u4f1a\u89e6\u53d1\u6e05\u7406\uff0c\u5e76\u4e14 StorageService \u4f1a\u786e\u4fdd\u7f13\u5b58\u7a7a\u95f4\u5360\u7528\u4e0d\u8d85\u8fc7\u4e0a\u6c34\u4f4d\uff1bstorage.watermark.low.ratio \u8868\u793a\u4f7f\u7528\u91cf\u7684\u4e0b\u6c34\u4f4d\u6bd4\u4f8b\uff080\u52301\u4e4b\u95f4\u7684\u5c0f\u6570\uff0c\u9700\u5c0f\u4e8e\u4e0a\u6c34\u4f4d\uff09\uff0c\u89e6\u53d1\u6e05\u7406\u540e\u4f1a\u81ea\u52a8\u6e05\u7406\u51b7\u6570\u636e\uff0c\u5c06\u7f13\u5b58\u6570\u636e\u76ee\u5f55\u5360\u7528\u7a7a\u95f4\u6e05\u7406\u5230\u4e0b\u6c34\u4f4d\u3002 3. [jindofsx-common] \u4e2d\u9700\u914d\u7f6e\u65e5\u5fd7\u3001metrics\u3001credentials\u3001endpoint \u76f8\u5173\u4fe1\u606f\uff0c\u52a0\u901f\u5e95\u5c42\u5b58\u50a8\u6240\u9700\u8981\u7684 credentials \u548c endpoint \u914d\u7f6e\u65b9\u6cd5\u89c1\u76f8\u5e94 credential provider \u914d\u7f6e\u6587\u6863 \u3002","title":"3. \u914d\u7f6e jindofsx.cfg"},{"location":"backup/jindofsx/deploy/deploy_jindofsx/#4","text":"\u4ee5 jindofsx-4.6.12 \u5b89\u88c5\u5728 /usr/lib \u4e3a\u4f8b\uff1a export JINDOFSX_CONF_DIR=/usr/lib/jindofsx-4.6.12/conf/ \u5e76\u786e\u8ba4\u90e8\u7f72\u5230\u6240\u6709\u8282\u70b9\u3002","title":"4. \u914d\u7f6e\u73af\u5883\u53d8\u91cf"},{"location":"backup/jindofsx/deploy/deploy_jindofsx/#5-jindofsx","text":"\u5728 master \u8282\u70b9\uff08\u542f\u52a8 JindoFSx NamespaceService \u7684\u8282\u70b9\uff09\u7f16\u8f91 sbin/nodes \u6587\u4ef6\uff0c\u914d\u7f6e\u4e0a\u6240\u6709\u9700\u8981\u542f\u52a8 StorageService \u7684\u8282\u70b9\u5217\u8868\uff0c\u4f8b\u5982\uff1a worker-1 worker-2 worker-3 \u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\uff08\u6ce8\u610f\uff1a\u7531\u4e8e\u811a\u672c\u901a\u8fc7 pssh \u547d\u4ee4\u6279\u91cf\u6267\u884c\u8fdc\u7a0b\u8282\u70b9\u7684\u670d\u52a1\u542f\u505c\uff0c\u9700\u8981\u4fdd\u8bc1 master \u8282\u70b9\u5230\u6240\u6709\u5176\u4ed6\u8282\u70b9 ssh \u7684\u514d\u5bc6\u8fde\u901a\uff09 cd jindofsx-x.x.x sh sbin/start-service.sh \u6267\u884c\u5b8c\u6210\u540e\uff0cmaster \u8282\u70b9\u5c06\u4f1a\u542f\u52a8 Namespace Service\uff0cnodes \u914d\u7f6e\u7684\u6240\u6709\u8282\u70b9\u5c06\u4f1a\u542f\u52a8 StorageService\u3002 \u53e6\u5916\uff0c\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u547d\u4ee4\u505c\u6b62\u6240\u6709\u7f13\u5b58\u670d\u52a1\uff1a cd jindofsx-x.x.x sh sbin/stop-service.sh","title":"5. \u542f\u52a8 JindoFSx \u7f13\u5b58\u670d\u52a1"},{"location":"backup/jindofsx/deploy/deploy_jindofsx/#6","text":"\u7f13\u5b58\u670d\u52a1\u7684\u8fdb\u7a0b\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u547d\u4ee4\u67e5\u770b\uff0c\u65e5\u5fd7\u6587\u4ef6\u53ef\u5728 logger.dir \u914d\u7f6e\u7684\u76ee\u5f55\u4e0b\u67e5\u770b ps -aux | grep jindofsx-namespaceservice ps -aux | grep jindofsx-storageservice \u5982\u9700\u90e8\u7f72\u9ad8\u53ef\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf, \u8bf7\u53c2\u8003 \u90e8\u7f72\u9ad8\u53ef\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf","title":"6. \u7f13\u5b58\u670d\u52a1\u72b6\u6001\u68c0\u67e5"},{"location":"backup/jindofsx/deploy/deploy_jindofsx_nsonly/","text":"\u5feb\u901f\u90e8\u7f72\u4e00\u4e2a\u7b80\u5355\u7684 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf(\u4ec5 Namespace) \u00a7 \u5f53\u65e0\u9700\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u65f6\uff0c\u53ef\u4ee5\u642d\u5efa\u4e00\u4e2a\u4ec5\u5305\u542b Namespace \u670d\u52a1\u7684 JindoFSx \u96c6\u7fa4\uff0c\u800c\u4e0d\u9700\u8981\u90e8\u7f72 Storage \u8fdb\u7a0b\u3002 1. \u4e0b\u8f7d JindoFSx \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 Release \u5305 jindofsx-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u89e3\u538b \u00a7 tar -xzvf jindofsx-x.x.x.tar.gz \u5e76\u5c06\u5b89\u88c5\u76ee\u5f55\u90e8\u7f72\u5230 master \u8282\u70b9\u4e0a\u3002 3. \u914d\u7f6e jindofsx.cfg \u00a7 \u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002\u5e76\u5c06\u914d\u7f6e\u6587\u4ef6\u90e8\u7f72\u5230 NamespaceService \u6240\u5728\u8282\u70b9\u3002 [jindofsx-common] logger.jnilogger=false # jni log \u5f00\u5173 metric.report.on = true # metrics \u5f00\u5173 logger.dir = /tmp/jindofsx-log # \u65e5\u5fd7\u76ee\u5f55 [jindofsx-namespace]# Namespace service \u914d\u7f6e namespace.rpc.port = 8101 # Namespace service \u76d1\u542c\u7aef\u53e3 namespace.meta-dir = /tmp/jindofsx/server # \u5143\u6570\u636e\u5b58\u653e\u76ee\u5f55 \u5fc5\u8981\u914d\u7f6e\u8bf4\u660e\uff1a 1. [jindofsx-common] \u4e2d\u9700\u914d\u7f6e\u65e5\u5fd7\u3001metrics\u3001credentials\u3001endpoint \u76f8\u5173\u4fe1\u606f\uff0c\u52a0\u901f\u5e95\u5c42\u5b58\u50a8\u6240\u9700\u8981\u7684 credentials \u548c endpoint \u914d\u7f6e\u65b9\u6cd5\u89c1\u76f8\u5e94 credential provider \u914d\u7f6e\u6587\u6863 \u3002 4. \u914d\u7f6e\u73af\u5883\u53d8\u91cf \u00a7 \u4ee5 jindofsx-4.6.12 \u5b89\u88c5\u5728 /usr/lib \u4e3a\u4f8b\uff1a export JINDOFSX_CONF_DIR=/usr/lib/jindofsx-4.6.12/conf/ \u5e76\u786e\u8ba4\u90e8\u7f72\u5230 Namespace \u670d\u52a1\u6240\u5728\u8282\u70b9\u3002 5. \u542f\u52a8 JindoFSx \u7f13\u5b58\u670d\u52a1 \u00a7 \u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\uff1a cd jindofsx-x.x.x sh sbin/start-service.sh \u6267\u884c\u5b8c\u6210\u540e\uff0cmaster \u8282\u70b9\u5c06\u4f1a\u542f\u52a8 Namespace Service\u3002 \u53e6\u5916\uff0c\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u547d\u4ee4\u505c\u6b62\u6240\u6709\u7f13\u5b58\u670d\u52a1\uff1a cd jindofsx-x.x.x sh sbin/stop-service.sh \u5982\u9700\u90e8\u7f72\u4ec5 Namespace \u7684\u9ad8\u53ef\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf, \u8bf7\u53c2\u8003 \u90e8\u7f72\u9ad8\u53ef\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf 6. \u5ba2\u6237\u7aef\u914d\u7f6e \u00a7 \u914d\u7f6e\u5728\u6240\u6709\u8282\u70b9\u7684 Hadoop \u914d\u7f6e\u6587\u4ef6\u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Client\u7aef\u8bbf\u95ee\u7684Namespace\u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>hostname:8101</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u6ce8\uff1a fs.jindofsx.namespace.rpc.address \u4e2d\u7684 hostname \u4e3a Namespace \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\u5730\u5740\u3002 \u53ef\u4ee5\u7528\u5982\u4e0b\u547d\u4ee4\u9a8c\u8bc1 JindoFSx \u670d\u52a1\u53ef\u7528\uff1a jindo admin -reportMetrics","title":"\u5feb\u901f\u90e8\u7f72\u4e00\u4e2a\u7b80\u5355\u7684 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf(\u4ec5 Namespace)"},{"location":"backup/jindofsx/deploy/deploy_jindofsx_nsonly/#jindofsx-namespace","text":"\u5f53\u65e0\u9700\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u65f6\uff0c\u53ef\u4ee5\u642d\u5efa\u4e00\u4e2a\u4ec5\u5305\u542b Namespace \u670d\u52a1\u7684 JindoFSx \u96c6\u7fa4\uff0c\u800c\u4e0d\u9700\u8981\u90e8\u7f72 Storage \u8fdb\u7a0b\u3002","title":"\u5feb\u901f\u90e8\u7f72\u4e00\u4e2a\u7b80\u5355\u7684 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf(\u4ec5 Namespace)"},{"location":"backup/jindofsx/deploy/deploy_jindofsx_nsonly/#1-jindofsx","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 Release \u5305 jindofsx-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoFSx \u5305"},{"location":"backup/jindofsx/deploy/deploy_jindofsx_nsonly/#2","text":"tar -xzvf jindofsx-x.x.x.tar.gz \u5e76\u5c06\u5b89\u88c5\u76ee\u5f55\u90e8\u7f72\u5230 master \u8282\u70b9\u4e0a\u3002","title":"2. \u89e3\u538b"},{"location":"backup/jindofsx/deploy/deploy_jindofsx_nsonly/#3-jindofsxcfg","text":"\u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002\u5e76\u5c06\u914d\u7f6e\u6587\u4ef6\u90e8\u7f72\u5230 NamespaceService \u6240\u5728\u8282\u70b9\u3002 [jindofsx-common] logger.jnilogger=false # jni log \u5f00\u5173 metric.report.on = true # metrics \u5f00\u5173 logger.dir = /tmp/jindofsx-log # \u65e5\u5fd7\u76ee\u5f55 [jindofsx-namespace]# Namespace service \u914d\u7f6e namespace.rpc.port = 8101 # Namespace service \u76d1\u542c\u7aef\u53e3 namespace.meta-dir = /tmp/jindofsx/server # \u5143\u6570\u636e\u5b58\u653e\u76ee\u5f55 \u5fc5\u8981\u914d\u7f6e\u8bf4\u660e\uff1a 1. [jindofsx-common] \u4e2d\u9700\u914d\u7f6e\u65e5\u5fd7\u3001metrics\u3001credentials\u3001endpoint \u76f8\u5173\u4fe1\u606f\uff0c\u52a0\u901f\u5e95\u5c42\u5b58\u50a8\u6240\u9700\u8981\u7684 credentials \u548c endpoint \u914d\u7f6e\u65b9\u6cd5\u89c1\u76f8\u5e94 credential provider \u914d\u7f6e\u6587\u6863 \u3002","title":"3. \u914d\u7f6e jindofsx.cfg"},{"location":"backup/jindofsx/deploy/deploy_jindofsx_nsonly/#4","text":"\u4ee5 jindofsx-4.6.12 \u5b89\u88c5\u5728 /usr/lib \u4e3a\u4f8b\uff1a export JINDOFSX_CONF_DIR=/usr/lib/jindofsx-4.6.12/conf/ \u5e76\u786e\u8ba4\u90e8\u7f72\u5230 Namespace \u670d\u52a1\u6240\u5728\u8282\u70b9\u3002","title":"4. \u914d\u7f6e\u73af\u5883\u53d8\u91cf"},{"location":"backup/jindofsx/deploy/deploy_jindofsx_nsonly/#5-jindofsx","text":"\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\uff1a cd jindofsx-x.x.x sh sbin/start-service.sh \u6267\u884c\u5b8c\u6210\u540e\uff0cmaster \u8282\u70b9\u5c06\u4f1a\u542f\u52a8 Namespace Service\u3002 \u53e6\u5916\uff0c\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u547d\u4ee4\u505c\u6b62\u6240\u6709\u7f13\u5b58\u670d\u52a1\uff1a cd jindofsx-x.x.x sh sbin/stop-service.sh \u5982\u9700\u90e8\u7f72\u4ec5 Namespace \u7684\u9ad8\u53ef\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf, \u8bf7\u53c2\u8003 \u90e8\u7f72\u9ad8\u53ef\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf","title":"5. \u542f\u52a8 JindoFSx \u7f13\u5b58\u670d\u52a1"},{"location":"backup/jindofsx/deploy/deploy_jindofsx_nsonly/#6","text":"\u914d\u7f6e\u5728\u6240\u6709\u8282\u70b9\u7684 Hadoop \u914d\u7f6e\u6587\u4ef6\u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Client\u7aef\u8bbf\u95ee\u7684Namespace\u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>hostname:8101</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u6ce8\uff1a fs.jindofsx.namespace.rpc.address \u4e2d\u7684 hostname \u4e3a Namespace \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\u5730\u5740\u3002 \u53ef\u4ee5\u7528\u5982\u4e0b\u547d\u4ee4\u9a8c\u8bc1 JindoFSx \u670d\u52a1\u53ef\u7528\uff1a jindo admin -reportMetrics","title":"6. \u5ba2\u6237\u7aef\u914d\u7f6e"},{"location":"backup/jindofsx/deploy/deploy_jindosdk/","text":"\u90e8\u7f72 JindoSDK \u00a7 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 Release \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u89e3\u538b \u00a7 tar -xzvf jindosdk-x.x.x.tar.gz \u5e76\u5c06 JindoSDK \u5b89\u88c5\u5305\u90e8\u7f72\u5230\u6240\u6709\u8282\u70b9\u3002 3. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ \u914d\u7f6e core-site.xml \u914d\u7f6e\u5728\u6240\u6709\u8282\u70b9\u7684 Hadoop \u914d\u7f6e\u6587\u4ef6\u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Client\u7aef\u8bbf\u95ee\u7684Namespace\u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u5e76\u5c06\u73af\u5883\u53d8\u91cf\u914d\u7f6e\u66f4\u65b0\u5230\u6240\u6709\u8282\u70b9\u3002","title":"\u90e8\u7f72 JindoSDK"},{"location":"backup/jindofsx/deploy/deploy_jindosdk/#jindosdk","text":"","title":"\u90e8\u7f72 JindoSDK"},{"location":"backup/jindofsx/deploy/deploy_jindosdk/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 Release \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindofsx/deploy/deploy_jindosdk/#2","text":"tar -xzvf jindosdk-x.x.x.tar.gz \u5e76\u5c06 JindoSDK \u5b89\u88c5\u5305\u90e8\u7f72\u5230\u6240\u6709\u8282\u70b9\u3002","title":"2. \u89e3\u538b"},{"location":"backup/jindofsx/deploy/deploy_jindosdk/#3-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ \u914d\u7f6e core-site.xml \u914d\u7f6e\u5728\u6240\u6709\u8282\u70b9\u7684 Hadoop \u914d\u7f6e\u6587\u4ef6\u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Client\u7aef\u8bbf\u95ee\u7684Namespace\u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u5e76\u5c06\u73af\u5883\u53d8\u91cf\u914d\u7f6e\u66f4\u65b0\u5230\u6240\u6709\u8282\u70b9\u3002","title":"3. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofsx/deploy/deploy_raft_ns/","text":"\u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u00a7 \u670d\u52a1\u7aef\u914d\u7f6e\u9ad8\u53ef\u7528 JindoFSx Namespace \u00a7 JindoFSx Namespace \u9ad8\u53ef\u7528\u90e8\u7f72\u9700\u8981 3 \u4e2a\u8282\u70b9\u5206\u522b\u542f\u52a8 NamespaceService \u670d\u52a1\u7ec4\u6210 1 \u4e2a Raft \u5b9e\u4f8b\u3002 1. \u6682\u505c JindoFSx \u670d\u52a1 \u00a7 cd jindofsx-x.x.x sh sbin/stop-service.sh 2. \u914d\u7f6e\u672c\u5730 raft \u540e\u7aef \u00a7 \u53d8\u66f4\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg \u6587\u4ef6\u4e2d [jindofsx-namespace] section \u4e0b\u7684\u914d\u7f6e\u9879\u3002 [jindofsx-namespace]# Namespace \u914d\u7f6e namespace.backend.type = raft #\u8bbe\u7f6enamespace\u540e\u7aef\u5b58\u50a8\u7c7b\u578b\uff0c\u652f\u6301 rocksdb \u548c raft, \u9ed8\u8ba4\u4e3a rocksdb\u3002 namespace.backend.raft.initial-conf = master-1-1:8103:0,master-1-2:8103:0,master-1-3:8103:0 #\u90e8\u7f72raft\u5b9e\u4f8b\u76843\u4e2aNamespace\u5730\u5740 3. \u914d\u7f6e Namespace \u5730\u5740 \u00a7 \u56e0\u4e3a 3 \u4e2a Master \u8282\u70b9\u7ec4\u6210 1 \u4e2a Raft \u5b9e\u4f8b\uff0c\u9700\u66f4\u65b0 jindofsx-x.x.x/sbin/headers \u6587\u4ef6\uff0c\u6dfb\u52a0 3 \u4e2a Namespace \u6240\u5728\u8282\u70b9\u5217\u8868\uff0c\u4f8b\u5982\uff1a master-1-1 master-1-2 master-1-3 4. \u540c\u6b65 jindofsx-x.x.x \u6587\u4ef6\u5939 \u00a7 \u5b8c\u6210\u4e0a\u8ff0\u914d\u7f6e\u540e\uff0c\u5c06 jindofsx-x.x.x \u540c\u6b65\u81f3\u53e6\u4e24\u53f0 Namespace \u8282\u70b9\u7684\u540c\u4e00\u76ee\u5f55\u3002 5. \u542f\u52a8 JindoFSx \u670d\u52a1 \u00a7 cd jindofsx-x.x.x sh sbin/start-service.sh \u5ba2\u6237\u7aef\u914d\u7f6e \u00a7 \u914d\u7f6e\u5728\u6240\u6709\u8282\u70b9\u7684 Hadoop \u914d\u7f6e\u6587\u4ef6\u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Client\u7aef\u8bbf\u95eeraft\u5b9e\u4f8b\u76843\u4e2aNamespace\u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>master-1-1:8101,master-1-2:8101,master-1-3:8101</value> </property> </configuration> \u6ce8\uff1a fs.jindofsx.namespace.rpc.address \u4e2d\u7684 hostname \u4e3a Namespace \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\u5730\u5740\u5217\u8868\uff0c \u4e0e namespace.backend.raft.initial-conf \u4e2d\u7684 hostname \u4e00\u81f4\u3002","title":"\u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528"},{"location":"backup/jindofsx/deploy/deploy_raft_ns/#jindofsx-namespace","text":"","title":"\u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528"},{"location":"backup/jindofsx/deploy/deploy_raft_ns/#jindofsx-namespace_1","text":"JindoFSx Namespace \u9ad8\u53ef\u7528\u90e8\u7f72\u9700\u8981 3 \u4e2a\u8282\u70b9\u5206\u522b\u542f\u52a8 NamespaceService \u670d\u52a1\u7ec4\u6210 1 \u4e2a Raft \u5b9e\u4f8b\u3002","title":"\u670d\u52a1\u7aef\u914d\u7f6e\u9ad8\u53ef\u7528 JindoFSx Namespace"},{"location":"backup/jindofsx/deploy/deploy_raft_ns/#1-jindofsx","text":"cd jindofsx-x.x.x sh sbin/stop-service.sh","title":"1. \u6682\u505c JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/deploy/deploy_raft_ns/#2-raft","text":"\u53d8\u66f4\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg \u6587\u4ef6\u4e2d [jindofsx-namespace] section \u4e0b\u7684\u914d\u7f6e\u9879\u3002 [jindofsx-namespace]# Namespace \u914d\u7f6e namespace.backend.type = raft #\u8bbe\u7f6enamespace\u540e\u7aef\u5b58\u50a8\u7c7b\u578b\uff0c\u652f\u6301 rocksdb \u548c raft, \u9ed8\u8ba4\u4e3a rocksdb\u3002 namespace.backend.raft.initial-conf = master-1-1:8103:0,master-1-2:8103:0,master-1-3:8103:0 #\u90e8\u7f72raft\u5b9e\u4f8b\u76843\u4e2aNamespace\u5730\u5740","title":"2. \u914d\u7f6e\u672c\u5730 raft \u540e\u7aef"},{"location":"backup/jindofsx/deploy/deploy_raft_ns/#3-namespace","text":"\u56e0\u4e3a 3 \u4e2a Master \u8282\u70b9\u7ec4\u6210 1 \u4e2a Raft \u5b9e\u4f8b\uff0c\u9700\u66f4\u65b0 jindofsx-x.x.x/sbin/headers \u6587\u4ef6\uff0c\u6dfb\u52a0 3 \u4e2a Namespace \u6240\u5728\u8282\u70b9\u5217\u8868\uff0c\u4f8b\u5982\uff1a master-1-1 master-1-2 master-1-3","title":"3. \u914d\u7f6e Namespace \u5730\u5740"},{"location":"backup/jindofsx/deploy/deploy_raft_ns/#4-jindofsx-xxx","text":"\u5b8c\u6210\u4e0a\u8ff0\u914d\u7f6e\u540e\uff0c\u5c06 jindofsx-x.x.x \u540c\u6b65\u81f3\u53e6\u4e24\u53f0 Namespace \u8282\u70b9\u7684\u540c\u4e00\u76ee\u5f55\u3002","title":"4. \u540c\u6b65 jindofsx-x.x.x \u6587\u4ef6\u5939"},{"location":"backup/jindofsx/deploy/deploy_raft_ns/#5-jindofsx","text":"cd jindofsx-x.x.x sh sbin/start-service.sh","title":"5. \u542f\u52a8 JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/deploy/deploy_raft_ns/#_1","text":"\u914d\u7f6e\u5728\u6240\u6709\u8282\u70b9\u7684 Hadoop \u914d\u7f6e\u6587\u4ef6\u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Client\u7aef\u8bbf\u95eeraft\u5b9e\u4f8b\u76843\u4e2aNamespace\u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>master-1-1:8101,master-1-2:8101,master-1-3:8101</value> </property> </configuration> \u6ce8\uff1a fs.jindofsx.namespace.rpc.address \u4e2d\u7684 hostname \u4e3a Namespace \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\u5730\u5740\u5217\u8868\uff0c \u4e0e namespace.backend.raft.initial-conf \u4e2d\u7684 hostname \u4e00\u81f4\u3002","title":"\u5ba2\u6237\u7aef\u914d\u7f6e"},{"location":"backup/jindofsx/deploy/deploy_raft_ns_emr/","text":"\u5728 EMR \u96c6\u7fa4\u4e2d\u914d\u7f6e\u9ad8\u53ef\u7528 JindoFSx Namespace \u00a7 \u670d\u52a1\u7aef\u914d\u7f6e\u9ad8\u53ef\u7528 JindoFSx Namespace \u00a7 JindoFSx Namespace \u9ad8\u53ef\u7528\u90e8\u7f72\u9700\u8981 3 \u4e2a\u8282\u70b9\u5206\u522b\u542f\u52a8 NamespaceService \u670d\u52a1\u7ec4\u6210 1 \u4e2a Raft \u5b9e\u4f8b\u3002 2. \u914d\u7f6e\u672c\u5730 raft \u540e\u7aef \u00a7 2.1 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u00a7 \u53c2\u6570 \u503c \u8bf4\u660e namespace.backend.type raft \u8bbe\u7f6enamespace\u540e\u7aef\u5b58\u50a8\u7c7b\u578b\uff0c\u652f\u6301 rocksdb \u548c raft, \u9ed8\u8ba4\u4e3a rocksdb\u3002 namespace.backend.raft.initial-conf master-1-1:8103:0,master-1-2:8103:0,master-1-3:8103:0 \u90e8\u7f72raft\u5b9e\u4f8b\u76843\u4e2aNamespace\u5730\u5740\u3002 2.2 \u91cd\u542f JindoFSx Namespace \u670d\u52a1\u3002 \u00a7 \u5ba2\u6237\u7aef\u914d\u7f6e \u00a7 \u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \uff0c \u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 <configuration> <property> <!-- Client\u7aef\u8bbf\u95eeraft\u5b9e\u4f8b\u76843\u4e2aNamespace\u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>master-1-1:8101,master-1-2:8101,master-1-3:8101</value> </property> </configuration> \u6ce8\uff1a fs.jindofsx.namespace.rpc.address \u4e2d\u7684 hostname \u4e3a Namespace \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\u5730\u5740\u5217\u8868\uff0c \u4e0e namespace.backend.raft.initial-conf \u4e2d\u7684 hostname \u4e00\u81f4\u3002","title":"\u5728 EMR \u96c6\u7fa4\u4e2d\u914d\u7f6e\u9ad8\u53ef\u7528 JindoFSx Namespace"},{"location":"backup/jindofsx/deploy/deploy_raft_ns_emr/#emr-jindofsx-namespace","text":"","title":"\u5728 EMR \u96c6\u7fa4\u4e2d\u914d\u7f6e\u9ad8\u53ef\u7528 JindoFSx Namespace"},{"location":"backup/jindofsx/deploy/deploy_raft_ns_emr/#jindofsx-namespace","text":"JindoFSx Namespace \u9ad8\u53ef\u7528\u90e8\u7f72\u9700\u8981 3 \u4e2a\u8282\u70b9\u5206\u522b\u542f\u52a8 NamespaceService \u670d\u52a1\u7ec4\u6210 1 \u4e2a Raft \u5b9e\u4f8b\u3002","title":"\u670d\u52a1\u7aef\u914d\u7f6e\u9ad8\u53ef\u7528 JindoFSx Namespace"},{"location":"backup/jindofsx/deploy/deploy_raft_ns_emr/#2-raft","text":"","title":"2. \u914d\u7f6e\u672c\u5730 raft \u540e\u7aef"},{"location":"backup/jindofsx/deploy/deploy_raft_ns_emr/#21-jindodata-namespace","text":"\u53c2\u6570 \u503c \u8bf4\u660e namespace.backend.type raft \u8bbe\u7f6enamespace\u540e\u7aef\u5b58\u50a8\u7c7b\u578b\uff0c\u652f\u6301 rocksdb \u548c raft, \u9ed8\u8ba4\u4e3a rocksdb\u3002 namespace.backend.raft.initial-conf master-1-1:8103:0,master-1-2:8103:0,master-1-3:8103:0 \u90e8\u7f72raft\u5b9e\u4f8b\u76843\u4e2aNamespace\u5730\u5740\u3002","title":"2.1 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9\u914d\u7f6e &gt; namespace &gt; \u65b0\u589e\u914d\u7f6e\u9879\uff0c\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002"},{"location":"backup/jindofsx/deploy/deploy_raft_ns_emr/#22-jindofsx-namespace","text":"","title":"2.2 \u91cd\u542f JindoFSx Namespace \u670d\u52a1\u3002"},{"location":"backup/jindofsx/deploy/deploy_raft_ns_emr/#_1","text":"\u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \uff0c \u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 <configuration> <property> <!-- Client\u7aef\u8bbf\u95eeraft\u5b9e\u4f8b\u76843\u4e2aNamespace\u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>master-1-1:8101,master-1-2:8101,master-1-3:8101</value> </property> </configuration> \u6ce8\uff1a fs.jindofsx.namespace.rpc.address \u4e2d\u7684 hostname \u4e3a Namespace \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\u5730\u5740\u5217\u8868\uff0c \u4e0e namespace.backend.raft.initial-conf \u4e2d\u7684 hostname \u4e00\u81f4\u3002","title":"\u5ba2\u6237\u7aef\u914d\u7f6e"},{"location":"backup/jindofsx/hadoop/hadoop_with_dls/","text":"Hadoop \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09+ JindoFSx \u900f\u660e\u52a0\u901f \u00a7 JindoSDK \u4e3a JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7cfb\u7edf\u63d0\u4f9b\u4e86 Apache Hadoop \u652f\u6301\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1 OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e Access Key \u5c06 OSS-HDFS \u670d\u52a1 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 JindoFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u6570\u636e\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u00a7 \u7528 Hadoop Shell \u8bbf\u95ee JindoFS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<Bucket>.<Endpoint>/ ls \u64cd\u4f5c hadoop fs -ls oss://<Bucket>.<Endpoint>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<Bucket>.<Endpoint>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<Bucket>.<Endpoint>/<path> \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"Hadoop \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09+ JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hadoop/hadoop_with_dls/#hadoop-oss-hdfs-jindofs-jindofsx","text":"JindoSDK \u4e3a JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7cfb\u7edf\u63d0\u4f9b\u4e86 Apache Hadoop \u652f\u6301\u3002","title":"Hadoop \u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09+ JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hadoop/hadoop_with_dls/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/hadoop/hadoop_with_dls/#jindosdk","text":"\u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1 OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e Access Key \u5c06 OSS-HDFS \u670d\u52a1 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 JindoFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u6570\u636e\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/hadoop/hadoop_with_dls/#jindosdk-oss-hdfs","text":"\u7528 Hadoop Shell \u8bbf\u95ee JindoFS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<Bucket>.<Endpoint>/ ls \u64cd\u4f5c hadoop fs -ls oss://<Bucket>.<Endpoint>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<Bucket>.<Endpoint>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<Bucket>.<Endpoint>/<path>","title":"\u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS-HDFS \u670d\u52a1"},{"location":"backup/jindofsx/hadoop/hadoop_with_dls/#_2","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/hadoop/hadoop_with_jindo/","text":"Hadoop \u8bbf\u95ee JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e \u00a7 JindoSDK \u4e3a JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7cfb\u7edf\u63d0\u4f9b\u4e86 Apache Hadoop \u652f\u6301\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 AccessKey \u5c06 OSS \u6216 OSS-HDFS \u670d\u52a1 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint \u5c06 OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u6570\u636e\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55 \u00a7 \u6302\u8f7d\u547d\u4ee4 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/ \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1 \u00a7 \u7528 Hadoop Shell \u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> jindo://emr-header-1:8101/jindooss/ ls \u64cd\u4f5c hadoop fs -ls jindo://emr-header-1:8101/jindooss/ mkdir \u64cd\u4f5c hadoop fs -mkdir jindo://emr-header-1:8101/jindooss/<path> rm \u64cd\u4f5c hadoop fs -rm jindo://emr-header-1:8101/jindooss/<path> \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"Hadoop \u8bbf\u95ee JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/hadoop/hadoop_with_jindo/#hadoop-jindofsx","text":"JindoSDK \u4e3a JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7cfb\u7edf\u63d0\u4f9b\u4e86 Apache Hadoop \u652f\u6301\u3002","title":"Hadoop \u8bbf\u95ee JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/hadoop/hadoop_with_jindo/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/hadoop/hadoop_with_jindo/#jindosdk","text":"\u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 AccessKey \u5c06 OSS \u6216 OSS-HDFS \u670d\u52a1 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint \u5c06 OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u6570\u636e\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/hadoop/hadoop_with_jindo/#oss-oss-hdfs","text":"\u6302\u8f7d\u547d\u4ee4 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/","title":"\u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55"},{"location":"backup/jindofsx/hadoop/hadoop_with_jindo/#jindosdk-oss-oss-hdfs","text":"\u7528 Hadoop Shell \u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> jindo://emr-header-1:8101/jindooss/ ls \u64cd\u4f5c hadoop fs -ls jindo://emr-header-1:8101/jindooss/ mkdir \u64cd\u4f5c hadoop fs -mkdir jindo://emr-header-1:8101/jindooss/<path> rm \u64cd\u4f5c hadoop fs -rm jindo://emr-header-1:8101/jindooss/<path>","title":"\u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1"},{"location":"backup/jindofsx/hadoop/hadoop_with_jindo/#_2","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/hadoop/hadoop_with_oss/","text":"Hadoop \u8bbf\u95ee\u963f\u91cc\u4e91 OSS + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 JindoSDK \u4e3a JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u4e86 Apache Hadoop \u652f\u6301\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1 OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e Access Key \u5c06 OSS \u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u00a7 \u7528 Hadoop Shell \u8bbf\u95ee JindoFS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<Bucket>/ ls \u64cd\u4f5c hadoop fs -ls oss://<Bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<Bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<Bucket>/<path> \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"Hadoop \u8bbf\u95ee\u963f\u91cc\u4e91 OSS + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hadoop/hadoop_with_oss/#hadoop-oss-jindofsx","text":"JindoSDK \u4e3a JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u4e86 Apache Hadoop \u652f\u6301\u3002","title":"Hadoop \u8bbf\u95ee\u963f\u91cc\u4e91 OSS + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hadoop/hadoop_with_oss/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/hadoop/hadoop_with_oss/#jindosdk","text":"\u914d\u7f6e OSS \u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1 OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e Access Key \u5c06 OSS \u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/hadoop/hadoop_with_oss/#jindosdk-oss","text":"\u7528 Hadoop Shell \u8bbf\u95ee JindoFS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<Bucket>/ ls \u64cd\u4f5c hadoop fs -ls oss://<Bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<Bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<Bucket>/<path>","title":"\u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/jindofsx/hadoop/hadoop_with_oss/#_2","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_dls/","text":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002 OSS-HDFS \u670d\u52a1\u662f\u963f\u91cc\u4e91\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9 HDFS \u63a5\u53e3, JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u6216 OSS \u670d\u52a1\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\u5b9e\u73b0\u5b58\u7b97\u5206\u79bb\uff0c\u76f8\u5bf9\u4e8e\u672c\u5730HDFS\u5b58\u50a8\uff0c\u4f7f\u7528\u66f4\u52a0\u7075\u6d3b\uff0c\u51cf\u5c11\u8fd0\u7ef4\u6210\u672c\u3002 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u00a7 \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84 \u00a7 \u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u7684\u503c\u4e3a OSS \u5730\u5740\uff0c\u4fee\u6539\u53c2\u6570 hbase.wal.dir \u7684\u503c\u4e3a\u672c\u5730\u7684 HDFS \u5730\u5740\uff0c\u901a\u8fc7\u672c\u5730 HDFS \u96c6\u7fa4\u5b58\u50a8 WAL \u6587\u4ef6\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 OSS\u3002\u53c2\u6570\u503c\u4e3a oss://bucket.endpoint/hbase-root-dir hbase.wal.dir \u6307\u5b9a HBase \u7684 WAL \u5b58\u50a8\u76ee\u5f55\u5230\u672c\u5730 HDFS \u96c6\u7fa4\u3002 \u53c2\u6570\u503c\u4e3a\uff1a HA\u96c6\u7fa4\uff1a hdfs://${nameservice}/hbase \u975eHA\u96c6\u7fa4\uff1a hdfs://${namenode_rpc_address}:${namenode_rpc_port}/hbase","title":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_dls/#hbase-oss-hdfs-jindofs-jindofsx","text":"HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002 OSS-HDFS \u670d\u52a1\u662f\u963f\u91cc\u4e91\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9 HDFS \u63a5\u53e3, JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u6216 OSS \u670d\u52a1\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\u5b9e\u73b0\u5b58\u7b97\u5206\u79bb\uff0c\u76f8\u5bf9\u4e8e\u672c\u5730HDFS\u5b58\u50a8\uff0c\u4f7f\u7528\u66f4\u52a0\u7075\u6d3b\uff0c\u51cf\u5c11\u8fd0\u7ef4\u6210\u672c\u3002","title":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_dls/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_dls/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_dls/#3-oss-hdfs-access-key","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"3. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_dls/#4-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002","title":"4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_dls/#hbase","text":"\u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u7684\u503c\u4e3a OSS \u5730\u5740\uff0c\u4fee\u6539\u53c2\u6570 hbase.wal.dir \u7684\u503c\u4e3a\u672c\u5730\u7684 HDFS \u5730\u5740\uff0c\u901a\u8fc7\u672c\u5730 HDFS \u96c6\u7fa4\u5b58\u50a8 WAL \u6587\u4ef6\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 OSS\u3002\u53c2\u6570\u503c\u4e3a oss://bucket.endpoint/hbase-root-dir hbase.wal.dir \u6307\u5b9a HBase \u7684 WAL \u5b58\u50a8\u76ee\u5f55\u5230\u672c\u5730 HDFS \u96c6\u7fa4\u3002 \u53c2\u6570\u503c\u4e3a\uff1a HA\u96c6\u7fa4\uff1a hdfs://${nameservice}/hbase \u975eHA\u96c6\u7fa4\uff1a hdfs://${namenode_rpc_address}:${namenode_rpc_port}/hbase","title":"\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_jindo/","text":"HBase \u4f7f\u7528 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e \u00a7 HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002 OSS-HDFS \u670d\u52a1\u662f\u963f\u91cc\u4e91\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9 HDFS \u63a5\u53e3, JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\u5b9e\u73b0\u5b58\u7b97\u5206\u79bb\uff0c\u76f8\u5bf9\u4e8e\u672c\u5730HDFS\u5b58\u50a8\uff0c\u4f7f\u7528\u66f4\u52a0\u7075\u6d3b\uff0c\u51cf\u5c11\u8fd0\u7ef4\u6210\u672c\u3002 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Access Key \u5c06 OSS Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindodls oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindodls \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/ \u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84 \u00a7 \u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u7684\u503c\u4e3a FSX \u5730\u5740\uff0c\u4fee\u6539\u53c2\u6570 hbase.wal.dir \u7684\u503c\u4e3a\u672c\u5730\u7684 HDFS \u5730\u5740\uff0c\u901a\u8fc7\u672c\u5730 HDFS \u96c6\u7fa4\u5b58\u50a8 WAL \u6587\u4ef6\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 FSX\u3002\u53c2\u6570\u503c\u4e3a jindo://headerhost:8101/jindooss/hbase-root-dir hbase.wal.dir \u6307\u5b9a HBase \u7684 WAL \u5b58\u50a8\u76ee\u5f55\u5230 FSX \u6216\u672c\u5730 HDFS\u3002\u53c2\u6570\u503c\u4e3a\uff1a HA\u96c6\u7fa4\uff1a hdfs://${nameservice}/hbase \u975eHA\u96c6\u7fa4\uff1a hdfs://${namenode_rpc_address}:${namenode_rpc_port}/hbase","title":"HBase \u4f7f\u7528 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_jindo/#hbase-jindofsx","text":"HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002 OSS-HDFS \u670d\u52a1\u662f\u963f\u91cc\u4e91\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9 HDFS \u63a5\u53e3, JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\u5b9e\u73b0\u5b58\u7b97\u5206\u79bb\uff0c\u76f8\u5bf9\u4e8e\u672c\u5730HDFS\u5b58\u50a8\uff0c\u4f7f\u7528\u66f4\u52a0\u7075\u6d3b\uff0c\u51cf\u5c11\u8fd0\u7ef4\u6210\u672c\u3002","title":"HBase \u4f7f\u7528 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_jindo/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_jindo/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_jindo/#3-jindosdk","text":"\u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Access Key \u5c06 OSS Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindodls oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindodls \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_jindo/#hbase","text":"\u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u7684\u503c\u4e3a FSX \u5730\u5740\uff0c\u4fee\u6539\u53c2\u6570 hbase.wal.dir \u7684\u503c\u4e3a\u672c\u5730\u7684 HDFS \u5730\u5740\uff0c\u901a\u8fc7\u672c\u5730 HDFS \u96c6\u7fa4\u5b58\u50a8 WAL \u6587\u4ef6\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 FSX\u3002\u53c2\u6570\u503c\u4e3a jindo://headerhost:8101/jindooss/hbase-root-dir hbase.wal.dir \u6307\u5b9a HBase \u7684 WAL \u5b58\u50a8\u76ee\u5f55\u5230 FSX \u6216\u672c\u5730 HDFS\u3002\u53c2\u6570\u503c\u4e3a\uff1a HA\u96c6\u7fa4\uff1a hdfs://${nameservice}/hbase \u975eHA\u96c6\u7fa4\uff1a hdfs://${namenode_rpc_address}:${namenode_rpc_port}/hbase","title":"\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_oss/","text":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\uff0c\u4f46\u9700\u901a\u8fc7\u672c\u5730 HDFS \u96c6\u7fa4\u5b58\u50a8 WAL \u6587\u4ef6\u3002 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u5c06 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u3002 \u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84 \u00a7 \u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84\uff0c\u662f\u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u7684\u503c\u4e3a OSS \u5730\u5740\uff0c\u4fee\u6539\u53c2\u6570 hbase.wal.dir \u7684\u503c\u4e3a\u672c\u5730\u7684 HDFS \u5730\u5740\uff0c\u901a\u8fc7\u672c\u5730 HDFS \u96c6\u7fa4\u5b58\u50a8 WAL \u6587\u4ef6\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 OSS\u3002\u53c2\u6570\u503c\u4e3a oss://bucket/hbase-root-dir hbase.wal.dir \u6307\u5b9a HBase \u7684 WAL \u5b58\u50a8\u76ee\u5f55\u5230\u672c\u5730 HDFS \u96c6\u7fa4\u3002 \u53c2\u6570\u503c\u4e3a\uff1a HA\u96c6\u7fa4\uff1a hdfs://${nameservice}/hbase \u975eHA\u96c6\u7fa4\uff1a hdfs://${namenode_rpc_address}:${namenode_rpc_port}/hbase","title":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_oss/#hbase-oss-jindofsx","text":"HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\uff0c\u4f46\u9700\u901a\u8fc7\u672c\u5730 HDFS \u96c6\u7fa4\u5b58\u50a8 WAL \u6587\u4ef6\u3002","title":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_oss/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_oss/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_oss/#3-oss-access-key","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u5c06 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u3002","title":"3. \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"backup/jindofsx/hbase/jindosdk_on_hbase_oss/#hbase","text":"\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84\uff0c\u662f\u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u7684\u503c\u4e3a OSS \u5730\u5740\uff0c\u4fee\u6539\u53c2\u6570 hbase.wal.dir \u7684\u503c\u4e3a\u672c\u5730\u7684 HDFS \u5730\u5740\uff0c\u901a\u8fc7\u672c\u5730 HDFS \u96c6\u7fa4\u5b58\u50a8 WAL \u6587\u4ef6\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 OSS\u3002\u53c2\u6570\u503c\u4e3a oss://bucket/hbase-root-dir hbase.wal.dir \u6307\u5b9a HBase \u7684 WAL \u5b58\u50a8\u76ee\u5f55\u5230\u672c\u5730 HDFS \u96c6\u7fa4\u3002 \u53c2\u6570\u503c\u4e3a\uff1a HA\u96c6\u7fa4\uff1a hdfs://${nameservice}/hbase \u975eHA\u96c6\u7fa4\uff1a hdfs://${namenode_rpc_address}:${namenode_rpc_port}/hbase","title":"\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs/","text":"Apache HDFS \u900f\u660e\u7f13\u5b58\u52a0\u901f \u00a7 \u5f53\u8ba1\u7b97\u96c6\u7fa4\u4f7f\u7528 Apache HDFS \u5b58\u50a8\uff0c\u5e76\u4e14\u4e0e\u4f7f\u7528\u7684 HDFS \u5b58\u50a8\u96c6\u7fa4\u5206\u79bb\u65f6\uff0c\u8bbf\u95ee HDFS \u96c6\u7fa4\u7684\u7f51\u7edc\u5e26\u5bbd\u53ef\u80fd\u53d7\u5230\u9650\u5236\u3002Apache HDFS \u900f\u660e\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002\u8ba1\u7b97\u96c6\u7fa4\u6216\u670d\u52a1\u53ef\u7ee7\u7eed\u4f7f\u7528\u539f\u6765\u7684\u5730\u5740 (hdfs://) \u8bbf\u95ee\u539f HDFS \u96c6\u7fa4\uff0c\u540c\u65f6\u901a\u8fc7\u672c\u5730\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u670d\u52a1\u7aef\u914d\u7f6e \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 \u914d\u7f6e HDFS \u96c6\u7fa4\u7684\u7f13\u5b58\u52a0\u901f [jindofsx-common] jindofsx.hdfs.user = xxx #Storage Service \u8bbf\u95ee HDFS \u4f7f\u7528\u7684\u7528\u6237\u540d \u914d\u7f6e HA HDFS \u96c6\u7fa4\u7684\u7f13\u5b58\u52a0\u901f\uff08\u53ef\u9009\uff09 [jindofsx-common] jindofsx.hdfs.XXX.dfs.ha.namenodes = nn1,nn2 jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn1 = nn1-host1:nn1-rpc-port jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn2 = nn2-host1:nn2-rpc-port \u8bf4\u660e: XXX \u4e3a HDFS \u96c6\u7fa4\u4e2d\u914d\u7f6e\u7684 dfs.nameservices \u53c2\u6570\u503c\u3002 \u5e76\u5c06\u914d\u7f6e\u6587\u4ef6\u66f4\u65b0\u5230\u6240\u6709\u670d\u52a1\u6240\u5728\u8282\u70b9\u4e0a\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u63d0\u4f9b\u7684 HDFS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.JindoHdfsFileSystem</value> </property> <property> <name>fs.AbstractFileSystem.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.HDFS</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e HA Namenodes (\u53ef\u9009) \u82e5\u4e3a HA HDFS\u96c6\u7fa4\uff0c\u5219\u5c06\u4e0b\u9762\u53c2\u6570\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.jindofsx.hdfs.XXX.dfs.ha.namenodes</name> <value>nn1,nn2</value> </property> <property> <name>fs.jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn1</name> <value>nn1-host1:nn1-rpc-port</value> </property> <property> <name>fs.jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn2</name> <value>nn1-host1:nn1-rpc-port</value> </property> </configuration> \u8bf4\u660e: XXX \u4e3a HDFS \u96c6\u7fa4\u4e2d\u914d\u7f6e\u7684 dfs.nameservices \u53c2\u6570\u503c\u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u914d\u7f6e\u7f13\u5b58\u52a0\u901f \u00a7 \u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u8bbf\u95ee HDFS \u00a7 \u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u901a\u8fc7 hdfs:// \u524d\u7f00\u8bfb\u53d6 HDFS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 hdfs:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002","title":"Apache HDFS \u900f\u660e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs/#apache-hdfs","text":"\u5f53\u8ba1\u7b97\u96c6\u7fa4\u4f7f\u7528 Apache HDFS \u5b58\u50a8\uff0c\u5e76\u4e14\u4e0e\u4f7f\u7528\u7684 HDFS \u5b58\u50a8\u96c6\u7fa4\u5206\u79bb\u65f6\uff0c\u8bbf\u95ee HDFS \u96c6\u7fa4\u7684\u7f51\u7edc\u5e26\u5bbd\u53ef\u80fd\u53d7\u5230\u9650\u5236\u3002Apache HDFS \u900f\u660e\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002\u8ba1\u7b97\u96c6\u7fa4\u6216\u670d\u52a1\u53ef\u7ee7\u7eed\u4f7f\u7528\u539f\u6765\u7684\u5730\u5740 (hdfs://) \u8bbf\u95ee\u539f HDFS \u96c6\u7fa4\uff0c\u540c\u65f6\u901a\u8fc7\u672c\u5730\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002","title":"Apache HDFS \u900f\u660e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs/#_2","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 \u914d\u7f6e HDFS \u96c6\u7fa4\u7684\u7f13\u5b58\u52a0\u901f [jindofsx-common] jindofsx.hdfs.user = xxx #Storage Service \u8bbf\u95ee HDFS \u4f7f\u7528\u7684\u7528\u6237\u540d \u914d\u7f6e HA HDFS \u96c6\u7fa4\u7684\u7f13\u5b58\u52a0\u901f\uff08\u53ef\u9009\uff09 [jindofsx-common] jindofsx.hdfs.XXX.dfs.ha.namenodes = nn1,nn2 jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn1 = nn1-host1:nn1-rpc-port jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn2 = nn2-host1:nn2-rpc-port \u8bf4\u660e: XXX \u4e3a HDFS \u96c6\u7fa4\u4e2d\u914d\u7f6e\u7684 dfs.nameservices \u53c2\u6570\u503c\u3002 \u5e76\u5c06\u914d\u7f6e\u6587\u4ef6\u66f4\u65b0\u5230\u6240\u6709\u670d\u52a1\u6240\u5728\u8282\u70b9\u4e0a\u3002","title":"\u670d\u52a1\u7aef\u914d\u7f6e"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs/#jindofsx","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs/#jindosdk","text":"\u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u63d0\u4f9b\u7684 HDFS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.JindoHdfsFileSystem</value> </property> <property> <name>fs.AbstractFileSystem.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.HDFS</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e HA Namenodes (\u53ef\u9009) \u82e5\u4e3a HA HDFS\u96c6\u7fa4\uff0c\u5219\u5c06\u4e0b\u9762\u53c2\u6570\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.jindofsx.hdfs.XXX.dfs.ha.namenodes</name> <value>nn1,nn2</value> </property> <property> <name>fs.jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn1</name> <value>nn1-host1:nn1-rpc-port</value> </property> <property> <name>fs.jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn2</name> <value>nn1-host1:nn1-rpc-port</value> </property> </configuration> \u8bf4\u660e: XXX \u4e3a HDFS \u96c6\u7fa4\u4e2d\u914d\u7f6e\u7684 dfs.nameservices \u53c2\u6570\u503c\u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs/#_3","text":"\u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e","title":"\u914d\u7f6e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs/#hdfs","text":"\u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u901a\u8fc7 hdfs:// \u524d\u7f00\u8bfb\u53d6 HDFS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 hdfs:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002","title":"\u8bbf\u95ee HDFS"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs_jindo/","text":"Apache HDFS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f \u00a7 \u5f53\u8ba1\u7b97\u96c6\u7fa4\u4f7f\u7528 Apache HDFS \u5b58\u50a8\uff0c\u5e76\u4e14\u4e0e\u4f7f\u7528\u7684 HDFS \u5b58\u50a8\u96c6\u7fa4\u5206\u79bb\u65f6\uff0c\u8bbf\u95ee HDFS \u96c6\u7fa4\u7684\u7f51\u7edc\u5e26\u5bbd\u53ef\u80fd\u53d7\u5230\u9650\u5236\u3002Apache HDFS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002\u8ba1\u7b97\u96c6\u7fa4\u6216\u670d\u52a1\u4f7f\u7528\u7edf\u4e00\u6302\u8f7d\u7684\u65b9\u5f0f (jindo://) \u8bbf\u95ee\u539f HDFS \u96c6\u7fa4\uff0c\u540c\u65f6\u901a\u8fc7\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u670d\u52a1\u7aef\u914d\u7f6e \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 \u914d\u7f6e HDFS \u96c6\u7fa4\u7684\u7f13\u5b58\u52a0\u901f [jindofsx-common] jindofsx.hdfs.user = xxx #Storage Service \u8bbf\u95ee HDFS \u4f7f\u7528\u7684\u7528\u6237\u540d \u914d\u7f6e HA HDFS \u96c6\u7fa4\u7684\u7f13\u5b58\u52a0\u901f\uff08\u53ef\u9009\uff09 [jindofsx-common] jindofsx.hdfs.XXX.dfs.ha.namenodes = nn1,nn2 jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn1 = nn1-host1:nn1-rpc-port jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn2 = nn2-host1:nn2-rpc-port \u8bf4\u660e: XXX \u4e3a HDFS \u96c6\u7fa4\u4e2d\u914d\u7f6e\u7684 dfs.nameservices \u53c2\u6570\u503c\u3002 \u5e76\u5c06\u914d\u7f6e\u6587\u4ef6\u66f4\u65b0\u5230\u6240\u6709\u670d\u52a1\u6240\u5728\u8282\u70b9\u4e0a\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e HA Namenodes (\u53ef\u9009) \u82e5\u4e3a HA HDFS\u96c6\u7fa4\uff0c\u5219\u5c06\u4e0b\u9762\u53c2\u6570\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.jindofsx.hdfs.XXX.dfs.ha.namenodes</name> <value>nn1,nn2</value> </property> <property> <name>fs.jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn1</name> <value>nn1-host1:nn1-rpc-port</value> </property> <property> <name>fs.jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn2</name> <value>nn1-host1:nn1-rpc-port</value> </property> </configuration> \u8bf4\u660e: XXX \u4e3a HDFS \u96c6\u7fa4\u4e2d\u914d\u7f6e\u7684 dfs.nameservices \u53c2\u6570\u503c\u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u6302\u8f7d HDFS \u00a7 \u6302\u8f7d HDFS \u96c6\u7fa4\u76ee\u5f55 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindohdfs hdfs://emr-header-1:9000/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindohdfs \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f hdfs://emr-header-1:9000/ [root@emr-header-1 ~]# hdfs dfs -ls jindo://emr-header-1:8101/ Found 2 items ---------- 1 0 1970-01-01 08:00 jindo://emr-header-1:8101/jindohdfs \u5373\u8bbf\u95ee jindo://emr-header-1:8101/jindohdfs/ \u7b49\u4ef7\u4e8e\u8bbf\u95ee hdfs://emr-header-1:9000/ \u914d\u7f6e\u7f13\u5b58\u52a0\u901f \u00a7 \u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u8bbf\u95ee HDFS \u00a7 \u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u5728\u6570\u636e\u7f13\u5b58\u5f00\u5173\u6253\u5f00\u65f6\u901a\u8fc7 jindo:// \u524d\u7f00\u8bfb\u53d6 HDFS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 jindo:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002","title":"Apache HDFS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs_jindo/#apache-hdfs","text":"\u5f53\u8ba1\u7b97\u96c6\u7fa4\u4f7f\u7528 Apache HDFS \u5b58\u50a8\uff0c\u5e76\u4e14\u4e0e\u4f7f\u7528\u7684 HDFS \u5b58\u50a8\u96c6\u7fa4\u5206\u79bb\u65f6\uff0c\u8bbf\u95ee HDFS \u96c6\u7fa4\u7684\u7f51\u7edc\u5e26\u5bbd\u53ef\u80fd\u53d7\u5230\u9650\u5236\u3002Apache HDFS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002\u8ba1\u7b97\u96c6\u7fa4\u6216\u670d\u52a1\u4f7f\u7528\u7edf\u4e00\u6302\u8f7d\u7684\u65b9\u5f0f (jindo://) \u8bbf\u95ee\u539f HDFS \u96c6\u7fa4\uff0c\u540c\u65f6\u901a\u8fc7\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002","title":"Apache HDFS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs_jindo/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs_jindo/#_2","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 \u914d\u7f6e HDFS \u96c6\u7fa4\u7684\u7f13\u5b58\u52a0\u901f [jindofsx-common] jindofsx.hdfs.user = xxx #Storage Service \u8bbf\u95ee HDFS \u4f7f\u7528\u7684\u7528\u6237\u540d \u914d\u7f6e HA HDFS \u96c6\u7fa4\u7684\u7f13\u5b58\u52a0\u901f\uff08\u53ef\u9009\uff09 [jindofsx-common] jindofsx.hdfs.XXX.dfs.ha.namenodes = nn1,nn2 jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn1 = nn1-host1:nn1-rpc-port jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn2 = nn2-host1:nn2-rpc-port \u8bf4\u660e: XXX \u4e3a HDFS \u96c6\u7fa4\u4e2d\u914d\u7f6e\u7684 dfs.nameservices \u53c2\u6570\u503c\u3002 \u5e76\u5c06\u914d\u7f6e\u6587\u4ef6\u66f4\u65b0\u5230\u6240\u6709\u670d\u52a1\u6240\u5728\u8282\u70b9\u4e0a\u3002","title":"\u670d\u52a1\u7aef\u914d\u7f6e"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs_jindo/#jindofsx","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs_jindo/#jindosdk","text":"\u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e HA Namenodes (\u53ef\u9009) \u82e5\u4e3a HA HDFS\u96c6\u7fa4\uff0c\u5219\u5c06\u4e0b\u9762\u53c2\u6570\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.jindofsx.hdfs.XXX.dfs.ha.namenodes</name> <value>nn1,nn2</value> </property> <property> <name>fs.jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn1</name> <value>nn1-host1:nn1-rpc-port</value> </property> <property> <name>fs.jindofsx.hdfs.XXX.dfs.namenode.rpc-address.nn2</name> <value>nn1-host1:nn1-rpc-port</value> </property> </configuration> \u8bf4\u660e: XXX \u4e3a HDFS \u96c6\u7fa4\u4e2d\u914d\u7f6e\u7684 dfs.nameservices \u53c2\u6570\u503c\u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs_jindo/#hdfs","text":"\u6302\u8f7d HDFS \u96c6\u7fa4\u76ee\u5f55 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindohdfs hdfs://emr-header-1:9000/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindohdfs \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f hdfs://emr-header-1:9000/ [root@emr-header-1 ~]# hdfs dfs -ls jindo://emr-header-1:8101/ Found 2 items ---------- 1 0 1970-01-01 08:00 jindo://emr-header-1:8101/jindohdfs \u5373\u8bbf\u95ee jindo://emr-header-1:8101/jindohdfs/ \u7b49\u4ef7\u4e8e\u8bbf\u95ee hdfs://emr-header-1:9000/","title":"\u6302\u8f7d HDFS"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs_jindo/#_3","text":"\u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e","title":"\u914d\u7f6e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/hdfs/jindofsx_on_hdfs_jindo/#hdfs_1","text":"\u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u5728\u6570\u636e\u7f13\u5b58\u5f00\u5173\u6253\u5f00\u65f6\u901a\u8fc7 jindo:// \u524d\u7f00\u8bfb\u53d6 HDFS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 jindo:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002","title":"\u8bbf\u95ee HDFS"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/","text":"Hive \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002 \u5b89\u88c5\u914d\u7f6e \u00a7 1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/ 2. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b \u5c06 JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key \u5c06\u5df2\u5f00\u542f OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 3. \u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 4. Hive on MR \u00a7 \u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002 5. Hive on Tez \u00a7 \u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002 6. Hive on Spark \u00a7 \u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002 OSS-HDFS \u670d\u52a1\u7528\u4e8e\u8868\u7684\u5b58\u50a8 \u00a7 \u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS-HDFS \u670d\u52a1\u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'oss://bucket_name/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'oss://bucket_name/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS-HDFS \u670d\u52a1\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>oss://bucket_name/path/to/warehouse</value> </property> </configuration> \u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a \u00a7 ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'oss://bucket_name/path/to/us/part210301cn'; \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Hive \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#hive-oss-hdfs-jindofs-jindofsx","text":"Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002","title":"Hive \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#_1","text":"","title":"\u5b89\u88c5\u914d\u7f6e"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#1-hive-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/","title":"1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#2-jindosdk","text":"\u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b \u5c06 JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key \u5c06\u5df2\u5f00\u542f OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"2. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#3-hive","text":"","title":"3. \u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#4-hive-on-mr","text":"\u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002","title":"4. Hive on MR"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#5-hive-on-tez","text":"\u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002","title":"5. Hive on Tez"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#6-hive-on-spark","text":"\u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002","title":"6. Hive on Spark"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#oss-hdfs","text":"\u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS-HDFS \u670d\u52a1\u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'oss://bucket_name/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'oss://bucket_name/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS-HDFS \u670d\u52a1\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>oss://bucket_name/path/to/warehouse</value> </property> </configuration>","title":"OSS-HDFS \u670d\u52a1\u7528\u4e8e\u8868\u7684\u5b58\u50a8"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#oss","text":"ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'oss://bucket_name/path/to/us/part210301cn';","title":"\u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_dls/#_2","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/","text":"Hive \u5904\u7406 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e \u00a7 Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS \u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002 \u5b89\u88c5\u914d\u7f6e \u00a7 1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/ 2. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Access Key \u5c06 OSS Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 3. \u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 4. Hive on MR \u00a7 \u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002 5. Hive on Tez \u00a7 \u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002 6. Hive on Spark \u00a7 \u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002 \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55 \u00a7 \u6302\u8f7d\u547d\u4ee4 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/ OSS \u7528\u4e8e\u8868\u7684\u5b58\u50a8 \u00a7 \u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS \u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS \u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'jindo://headerhost:8101/jindooss/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'jindo://headerhost:8101/jindooss/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS \u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>jindo://headerhost:8101/jindooss/path/to/warehouse</value> </property> </configuration> \u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a \u00a7 ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'jindo://headerhost:8101/jindooss/path/to/us/part210301cn'; \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Hive \u5904\u7406 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#hive-jindofsx","text":"Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS \u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002","title":"Hive \u5904\u7406 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#_1","text":"","title":"\u5b89\u88c5\u914d\u7f6e"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#1-hive-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/","title":"1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#2-jindosdk","text":"\u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Access Key \u5c06 OSS Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"2. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#3-hive","text":"","title":"3. \u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#4-hive-on-mr","text":"\u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002","title":"4. Hive on MR"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#5-hive-on-tez","text":"\u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002","title":"5. Hive on Tez"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#6-hive-on-spark","text":"\u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002","title":"6. Hive on Spark"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#oss-oss-hdfs","text":"\u6302\u8f7d\u547d\u4ee4 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/","title":"\u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#oss","text":"\u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS \u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS \u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'jindo://headerhost:8101/jindooss/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'jindo://headerhost:8101/jindooss/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS \u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>jindo://headerhost:8101/jindooss/path/to/warehouse</value> </property> </configuration>","title":"OSS \u7528\u4e8e\u8868\u7684\u5b58\u50a8"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#oss_1","text":"ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'jindo://headerhost:8101/jindooss/path/to/us/part210301cn';","title":"\u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_jindo/#_2","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/","text":"Hive \u5904\u7406\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS \u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002 \u5b89\u88c5\u914d\u7f6e \u00a7 1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/ 2. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key \u5c06 OSS Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 3. \u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 4. Hive on MR \u00a7 \u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002 5. Hive on Tez \u00a7 \u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002 6. Hive on Spark \u00a7 \u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002 OSS \u7528\u4e8e\u8868\u7684\u5b58\u50a8 \u00a7 \u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS \u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS \u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'oss://bucketname/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'oss://bucketname/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS \u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>oss://bucketname/path/to/warehouse</value> </property> </configuration> \u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a \u00a7 ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'oss://bucketname/path/to/us/part210301cn'; \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Hive \u5904\u7406\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#hive-oss-jindofsx","text":"Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS \u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002","title":"Hive \u5904\u7406\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#_1","text":"","title":"\u5b89\u88c5\u914d\u7f6e"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#1-hive-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/","title":"1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#2-jindosdk","text":"\u914d\u7f6e OSS \u5b9e\u73b0\u7c7b \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key \u5c06 OSS Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"2. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#3-hive","text":"","title":"3. \u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#4-hive-on-mr","text":"\u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002","title":"4. Hive on MR"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#5-hive-on-tez","text":"\u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002","title":"5. Hive on Tez"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#6-hive-on-spark","text":"\u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002","title":"6. Hive on Spark"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#oss","text":"\u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS \u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS \u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'oss://bucketname/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'oss://bucketname/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS \u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>oss://bucketname/path/to/warehouse</value> </property> </configuration>","title":"OSS \u7528\u4e8e\u8868\u7684\u5b58\u50a8"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#oss_1","text":"ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'oss://bucketname/path/to/us/part210301cn';","title":"\u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a"},{"location":"backup/jindofsx/hive/jindosdk_on_hive_oss/#_2","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_dls/","text":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Impala \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Impala \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fddImpala\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728\u6240\u6709 Impala \u8282\u70b9\u5b89\u88c5 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06 sdk \u5305\u5b89\u88c5\u5230 Impala \u7684 classpath \u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $IMPALA_HOME/lib/ 3. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e IMPALA \u4f7f\u7528\u7684 core-site.xml \u914d\u7f6e JindoSDK \u8bbf\u95ee OSS \u00a7 \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 4. \u4f7f\u7528 Impala \u8bbf\u95ee OSS\u3002 \u00a7 \u521b\u5efa\u8868 sql CREATE EXTERNAL TABLE customer_demographics ( `cd_demo_sk` INT, `cd_gender` STRING, `cd_marital_status` STRING, `cd_education_status` STRING, `cd_purchase_estimate` INT, `cd_credit_rating` STRING, `cd_dep_count` INT, `cd_dep_employed_count` INT, `cd_dep_college_count` INT) STORED AS PARQUET LOCATION 'oss://bucket.endpoint/dir'; \u67e5\u8be2 OSS \u8868 sql select * from customer_demographics; 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_dls/#impala-jindosdk-oss-hdfs-jindofs-jindofsx","text":"JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Impala \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_dls/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_dls/#1-impala-hadoop","text":"\u786e\u4fddImpala\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Impala \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_dls/#2-impala-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06 sdk \u5305\u5b89\u88c5\u5230 Impala \u7684 classpath \u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $IMPALA_HOME/lib/","title":"2. \u5728\u6240\u6709 Impala \u8282\u70b9\u5b89\u88c5 JindoSDK"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_dls/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_dls/#impala-core-sitexml-jindosdk-oss","text":"\u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"\u914d\u7f6e IMPALA \u4f7f\u7528\u7684 core-site.xml \u914d\u7f6e JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_dls/#4-impala-oss","text":"\u521b\u5efa\u8868 sql CREATE EXTERNAL TABLE customer_demographics ( `cd_demo_sk` INT, `cd_gender` STRING, `cd_marital_status` STRING, `cd_education_status` STRING, `cd_purchase_estimate` INT, `cd_credit_rating` STRING, `cd_dep_count` INT, `cd_dep_employed_count` INT, `cd_dep_college_count` INT) STORED AS PARQUET LOCATION 'oss://bucket.endpoint/dir'; \u67e5\u8be2 OSS \u8868 sql select * from customer_demographics;","title":"4. \u4f7f\u7528 Impala \u8bbf\u95ee OSS\u3002"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_dls/#5","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_jindo/","text":"Impala \u67e5\u8be2 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e \u00a7 JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Impala \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Impala \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fddImpala\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728\u6240\u6709 Impala \u8282\u70b9\u5b89\u88c5 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06 sdk \u5305\u5b89\u88c5\u5230 Impala \u7684 classpath \u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $IMPALA_HOME/lib/ 3. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e IMPALA \u4f7f\u7528\u7684 core-site.xml \u914d\u7f6e JindoSDK \u8bbf\u95ee OSS \u00a7 \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/ 4. \u4f7f\u7528 Impala \u8bbf\u95ee OSS\u3002 \u00a7 \u521b\u5efa\u8868 sql CREATE EXTERNAL TABLE customer_demographics ( `cd_demo_sk` INT, `cd_gender` STRING, `cd_marital_status` STRING, `cd_education_status` STRING, `cd_purchase_estimate` INT, `cd_credit_rating` STRING, `cd_dep_count` INT, `cd_dep_employed_count` INT, `cd_dep_college_count` INT) STORED AS PARQUET LOCATION 'jindo://headerhost:8101/jindooss/dir'; \u67e5\u8be2 OSS \u8868 sql select * from customer_demographics; 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Impala \u67e5\u8be2 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_jindo/#impala-jindofsx","text":"JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Impala \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Impala \u67e5\u8be2 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_jindo/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_jindo/#1-impala-hadoop","text":"\u786e\u4fddImpala\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Impala \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_jindo/#2-impala-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06 sdk \u5305\u5b89\u88c5\u5230 Impala \u7684 classpath \u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $IMPALA_HOME/lib/","title":"2. \u5728\u6240\u6709 Impala \u8282\u70b9\u5b89\u88c5 JindoSDK"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_jindo/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_jindo/#impala-core-sitexml-jindosdk-oss","text":"\u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/","title":"\u914d\u7f6e IMPALA \u4f7f\u7528\u7684 core-site.xml \u914d\u7f6e JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_jindo/#4-impala-oss","text":"\u521b\u5efa\u8868 sql CREATE EXTERNAL TABLE customer_demographics ( `cd_demo_sk` INT, `cd_gender` STRING, `cd_marital_status` STRING, `cd_education_status` STRING, `cd_purchase_estimate` INT, `cd_credit_rating` STRING, `cd_dep_count` INT, `cd_dep_employed_count` INT, `cd_dep_college_count` INT) STORED AS PARQUET LOCATION 'jindo://headerhost:8101/jindooss/dir'; \u67e5\u8be2 OSS \u8868 sql select * from customer_demographics;","title":"4. \u4f7f\u7528 Impala \u8bbf\u95ee OSS\u3002"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_jindo/#5","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_oss/","text":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Impala \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Impala \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fddImpala\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728\u6240\u6709 Impala \u8282\u70b9\u5b89\u88c5 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06 sdk \u5305\u5b89\u88c5\u5230 Impala \u7684 classpath \u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $IMPALA_HOME/lib/ 3. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e IMPALA \u4f7f\u7528\u7684 core-site.xml \u914d\u7f6e JindoSDK \u8bbf\u95ee OSS \u00a7 \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key \u5c06 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 4. \u4f7f\u7528 Impala \u8bbf\u95ee OSS\u3002 \u00a7 \u521b\u5efa\u8868 sql CREATE EXTERNAL TABLE customer_demographics ( `cd_demo_sk` INT, `cd_gender` STRING, `cd_marital_status` STRING, `cd_education_status` STRING, `cd_purchase_estimate` INT, `cd_credit_rating` STRING, `cd_dep_count` INT, `cd_dep_employed_count` INT, `cd_dep_college_count` INT) STORED AS PARQUET LOCATION 'oss://bucket.endpoint/dir'; \u67e5\u8be2 OSS \u8868 sql select * from customer_demographics; 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_oss/#impala-jindosdk-oss-jindofsx","text":"JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Impala \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_oss/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_oss/#1-impala-hadoop","text":"\u786e\u4fddImpala\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Impala \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_oss/#2-impala-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06 sdk \u5305\u5b89\u88c5\u5230 Impala \u7684 classpath \u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $IMPALA_HOME/lib/","title":"2. \u5728\u6240\u6709 Impala \u8282\u70b9\u5b89\u88c5 JindoSDK"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_oss/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_oss/#impala-core-sitexml-jindosdk-oss","text":"\u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key \u5c06 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"\u914d\u7f6e IMPALA \u4f7f\u7528\u7684 core-site.xml \u914d\u7f6e JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_oss/#4-impala-oss","text":"\u521b\u5efa\u8868 sql CREATE EXTERNAL TABLE customer_demographics ( `cd_demo_sk` INT, `cd_gender` STRING, `cd_marital_status` STRING, `cd_education_status` STRING, `cd_purchase_estimate` INT, `cd_credit_rating` STRING, `cd_dep_count` INT, `cd_dep_employed_count` INT, `cd_dep_college_count` INT) STORED AS PARQUET LOCATION 'oss://bucket.endpoint/dir'; \u67e5\u8be2 OSS \u8868 sql select * from customer_demographics;","title":"4. \u4f7f\u7528 Impala \u8bbf\u95ee OSS\u3002"},{"location":"backup/jindofsx/impala/jindosdk_on_impala_oss/#5","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_aliyun_sls_example/","text":"\u4f7f\u7528\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u6536\u96c6\u670d\u52a1 \u00a7 JindoRuntime \u9ed8\u8ba4\u65e5\u5fd7\u6253\u5728\u5bb9\u5668 std \u8f93\u51fa\u91cc\u9762\uff0c\u53ef\u4ee5\u4f7f\u7528\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u6536\u96c6\u670d\u52a1\u5c06\u65e5\u5fd7\u6536\u96c6\u8d77\u6765\u3002 1\u3001ack \u96c6\u7fa4\u7ed1\u5b9a sls \u670d\u52a1 \u00a7 \u60a8\u53ef\u4ee5\u5728\u521b\u5efa\u96c6\u7fa4\u65f6\u7ed1\u5b9a sls \u670d\u52a1 \u4e5f\u53ef\u4ee5\u5728\u521b\u5efa\u96c6\u7fa4\u540e\u7ed1\u5b9a\u963f\u91cc\u4e91 sls \u670d\u52a1 \u9009\u62e9\u96c6\u7fa4\u4fe1\u606f > \u96c6\u7fa4\u8d44\u6e90 > \u65e5\u5fd7\u670d\u52a1 Project 2\u3001\u521b\u5efa JindoRuntime \u96c6\u7fa4 \u00a7 \u53ef\u53c2\u8003\u6587\u7ae0 \u52a0\u901fOSS\u4e0a\u6570\u636e \u521b\u5efa JindoRuntime \u96c6\u7fa4\u3002 3\u3001\u914d\u7f6e sls \u6536\u96c6 \u00a7 \u60a8\u53ef\u4ee5\u9009\u62e9\u5728 ack \u63a7\u5236\u53f0\u64cd\u4f5c\uff0c\u6216\u8005\u76f4\u63a5\u64cd\u4f5c\u96c6\u7fa4 K8S \u8d44\u6e90 3.1 ack \u63a7\u5236\u53f0\u64cd\u4f5c \u00a7 \u672c\u6587\u4ee5 JindoRuntime \u540d\u5b57\u4e3a test \u8fdb\u884c\u4e3e\u4f8b \u60a8\u53ef\u4ee5\u767b\u9646 ack \u63a7\u5236\u53f0 > \u5de5\u4f5c\u8d1f\u8f7d * \u6709\u72b6\u6001\uff08master/worker\uff09 \u5b88\u62a4\u8fdb\u7a0b\u96c6\uff08fuse\uff09 \u70b9\u51fb \u7f16\u8f91 \u6dfb\u52a0\u65e5\u5fd7\u6536\u96c6 \u5176\u4e2d\u65e5\u5fd7\u5e93\u540d\u79f0\u53ef\u81ea\u5b9a\u4e49\u586b\u5199\uff0c\u8be5\u540d\u79f0\u5373\u4e3a sls \u4e0a\u65e5\u5fd7\u5e93\u7684\u540d\u5b57\uff0c\u5bb9\u5668\u5185\u65e5\u5fd7\u8def\u5f84\u4e3a stdout \u70b9\u51fb\u53f3\u4fa7\u66f4\u65b0 3.2\u3001\u76f4\u63a5\u64cd\u4f5c K8S \u8d44\u6e90 \u00a7 \u4ee5\u6536\u96c6 fuse \u65e5\u5fd7\u4e3a\u4f8b\uff0c\u6267\u884c kubectl edit daemonset test-jindofs-fuse \u4fee\u6539 spec.env \u589e\u52a0 - name: aliyun_logs_jindo-fuse value: stdout name \u683c\u5f0f\u4e3a\uff1a aliyun_logs_ \uff0c\u524d\u7f00\u4e3a aliyun_logs_ value \u4e3a\u56fa\u5b9a\u503c stdout \u5982\u679c\u60f3\u4fee\u6539 master/worker\uff0c\u53ef\u6267\u884c kubectl edit statefulset test-jindofs-master kubectl edit statefulset test-jindofs-worker \u8fdb\u884c\u5bf9\u5e94\u7684\u7f16\u8f91 4\u3001\u767b\u9646\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u670d\u52a1\u67e5\u770b\u76f8\u5173\u65e5\u5fd7\u60c5\u51b5 \u00a7 \u4ee5 jindofuse \u65e5\u5fd7\u6536\u96c6\u4e3a\u4f8b\uff0c\u65e5\u5fd7\u5e93\u540d\u79f0\u4e3a jindo-fuse ,\u5728sls\u4e0a\u627e\u5230\u5bf9\u5e94\u7684\u65e5\u5fd7\u5e93","title":"\u4f7f\u7528\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u6536\u96c6\u670d\u52a1"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_aliyun_sls_example/#sls","text":"JindoRuntime \u9ed8\u8ba4\u65e5\u5fd7\u6253\u5728\u5bb9\u5668 std \u8f93\u51fa\u91cc\u9762\uff0c\u53ef\u4ee5\u4f7f\u7528\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u6536\u96c6\u670d\u52a1\u5c06\u65e5\u5fd7\u6536\u96c6\u8d77\u6765\u3002","title":"\u4f7f\u7528\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u6536\u96c6\u670d\u52a1"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_aliyun_sls_example/#1ack-sls","text":"\u60a8\u53ef\u4ee5\u5728\u521b\u5efa\u96c6\u7fa4\u65f6\u7ed1\u5b9a sls \u670d\u52a1 \u4e5f\u53ef\u4ee5\u5728\u521b\u5efa\u96c6\u7fa4\u540e\u7ed1\u5b9a\u963f\u91cc\u4e91 sls \u670d\u52a1 \u9009\u62e9\u96c6\u7fa4\u4fe1\u606f > \u96c6\u7fa4\u8d44\u6e90 > \u65e5\u5fd7\u670d\u52a1 Project","title":"1\u3001ack \u96c6\u7fa4\u7ed1\u5b9a sls \u670d\u52a1"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_aliyun_sls_example/#2-jindoruntime","text":"\u53ef\u53c2\u8003\u6587\u7ae0 \u52a0\u901fOSS\u4e0a\u6570\u636e \u521b\u5efa JindoRuntime \u96c6\u7fa4\u3002","title":"2\u3001\u521b\u5efa JindoRuntime \u96c6\u7fa4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_aliyun_sls_example/#3-sls","text":"\u60a8\u53ef\u4ee5\u9009\u62e9\u5728 ack \u63a7\u5236\u53f0\u64cd\u4f5c\uff0c\u6216\u8005\u76f4\u63a5\u64cd\u4f5c\u96c6\u7fa4 K8S \u8d44\u6e90","title":"3\u3001\u914d\u7f6e sls \u6536\u96c6"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_aliyun_sls_example/#31-ack","text":"\u672c\u6587\u4ee5 JindoRuntime \u540d\u5b57\u4e3a test \u8fdb\u884c\u4e3e\u4f8b \u60a8\u53ef\u4ee5\u767b\u9646 ack \u63a7\u5236\u53f0 > \u5de5\u4f5c\u8d1f\u8f7d * \u6709\u72b6\u6001\uff08master/worker\uff09 \u5b88\u62a4\u8fdb\u7a0b\u96c6\uff08fuse\uff09 \u70b9\u51fb \u7f16\u8f91 \u6dfb\u52a0\u65e5\u5fd7\u6536\u96c6 \u5176\u4e2d\u65e5\u5fd7\u5e93\u540d\u79f0\u53ef\u81ea\u5b9a\u4e49\u586b\u5199\uff0c\u8be5\u540d\u79f0\u5373\u4e3a sls \u4e0a\u65e5\u5fd7\u5e93\u7684\u540d\u5b57\uff0c\u5bb9\u5668\u5185\u65e5\u5fd7\u8def\u5f84\u4e3a stdout \u70b9\u51fb\u53f3\u4fa7\u66f4\u65b0","title":"3.1 ack \u63a7\u5236\u53f0\u64cd\u4f5c"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_aliyun_sls_example/#32-k8s","text":"\u4ee5\u6536\u96c6 fuse \u65e5\u5fd7\u4e3a\u4f8b\uff0c\u6267\u884c kubectl edit daemonset test-jindofs-fuse \u4fee\u6539 spec.env \u589e\u52a0 - name: aliyun_logs_jindo-fuse value: stdout name \u683c\u5f0f\u4e3a\uff1a aliyun_logs_ \uff0c\u524d\u7f00\u4e3a aliyun_logs_ value \u4e3a\u56fa\u5b9a\u503c stdout \u5982\u679c\u60f3\u4fee\u6539 master/worker\uff0c\u53ef\u6267\u884c kubectl edit statefulset test-jindofs-master kubectl edit statefulset test-jindofs-worker \u8fdb\u884c\u5bf9\u5e94\u7684\u7f16\u8f91","title":"3.2\u3001\u76f4\u63a5\u64cd\u4f5c K8S \u8d44\u6e90"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_aliyun_sls_example/#4-sls","text":"\u4ee5 jindofuse \u65e5\u5fd7\u6536\u96c6\u4e3a\u4f8b\uff0c\u65e5\u5fd7\u5e93\u540d\u79f0\u4e3a jindo-fuse ,\u5728sls\u4e0a\u627e\u5230\u5bf9\u5e94\u7684\u65e5\u5fd7\u5e93","title":"4\u3001\u767b\u9646\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u670d\u52a1\u67e5\u770b\u76f8\u5173\u65e5\u5fd7\u60c5\u51b5"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/","text":"\u6570\u636e\u9884\u52a0\u8f7d \u00a7 \u914d\u7f6e\u5f85\u521b\u5efa\u7684DataLoad\u5bf9\u8c61 \u521b\u5efaDataLoad\u5bf9\u8c61 \u67e5\u770b\u521b\u5efa\u7684DataLoad\u5bf9\u8c61\u72b6\u6001 \u7b49\u5f85\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5b8c\u6210 DataLoad\u8fdb\u9636\u914d\u7f6e \u6307\u5b9a\u4e00\u4e2a\u6216\u591a\u4e2a\u6570\u636e\u96c6\u5b50\u76ee\u5f55\u8fdb\u884c\u52a0\u8f7d \u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf \u539f\u5b50\u6027\u7f13\u5b58 \u5355\u72ec\u8fdb\u884c\u5143\u6570\u636e\u7f13\u5b58\uff0c\u4e0d\u505a\u6570\u636e\u7f13\u5b58 \u7f13\u5b58\u8fdb\u5ea6\u53caLOG\u67e5\u770b \u73af\u5883\u6e05\u7406 \u4e3a\u4e86\u4fdd\u8bc1\u5e94\u7528\u5728\u8bbf\u95ee\u6570\u636e\u65f6\u7684\u6027\u80fd\uff0c\u53ef\u4ee5\u901a\u8fc7 \u6570\u636e\u9884\u52a0\u8f7d \u63d0\u524d\u5c06\u8fdc\u7a0b\u5b58\u50a8\u7cfb\u7edf\u4e2d\u7684\u6570\u636e\u62c9\u53d6\u5230\u9760\u8fd1\u8ba1\u7b97\u7ed3\u70b9\u7684\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d\uff0c\u4f7f\u5f97\u6d88\u8d39\u8be5\u6570\u636e\u96c6\u7684\u5e94\u7528\u80fd\u591f\u5728\u9996\u6b21\u8fd0\u884c\u65f6\u5373\u53ef\u4eab\u53d7\u5230\u7f13\u5b58\u5e26\u6765\u7684\u52a0\u901f\u6548\u679c\u3002 \u4e3a\u6b64\uff0c\u6211\u4eec\u63d0\u4f9b\u4e86DataLoad CRD, \u8be5CRD\u8ba9\u4f60\u53ef\u901a\u8fc7\u7b80\u5355\u7684\u914d\u7f6e\u5c31\u80fd\u5b8c\u6210\u6574\u4e2a\u6570\u636e\u9884\u52a0\u8f7d\u8fc7\u7a0b\uff0c\u5e76\u5bf9\u6570\u636e\u9884\u52a0\u8f7d\u7684\u8bb8\u591a\u884c\u4e3a\u8fdb\u884c\u81ea\u5b9a\u4e49\u63a7\u5236\u3002 \u5047\u8bbe\u6211\u4eec\u5df2\u7ecf\u6709\u5982\u4e0b\u4e00\u4e2aname\u4e3a spark \u7684dataset\uff1a apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: spark spec: mounts: - mountPoint: oss://test-bucket/spark name: sparkV1 accessModes: - ReadWriteMany \u63a5\u4e0b\u6765\u5bf9\u8fd9\u4e2adataset\u8fdb\u884c\u9884\u70ed\uff0c\u8bf7\u6ce8\u610f\u4e0b\u6587\u4e2d\u7684 /sparkV1 \u4e3a mounts[].name \u7684\u503c\uff0c\u5982\u679c\u60a8\u7684 mounts[].path \u4e3a / , \u90a3\u4e48\u4e0b\u6587\u4e2d\u7684 /sparkV1 \u53ef\u4ee5\u76f4\u63a5\u6539\u4e3a / \u914d\u7f6e\u5f85\u521b\u5efa\u7684DataLoad\u5bf9\u8c61 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default target: - path: /sparkV1/ spec.dataset \u6307\u660e\u4e86\u9700\u8981\u8fdb\u884c\u6570\u636e\u9884\u52a0\u8f7d\u7684\u76ee\u6807\u6570\u636e\u96c6\uff0c\u5728\u8be5\u4f8b\u5b50\u4e2d\uff0c\u6211\u4eec\u7684\u6570\u636e\u9884\u52a0\u8f7d\u76ee\u6807\u4e3a default \u547d\u540d\u7a7a\u95f4\u4e0b\u540d\u4e3a spark \u7684\u6570\u636e\u96c6\uff0c\u5982\u679c\u8be5\u914d\u7f6e\u4e0e\u4f60\u6240\u5728\u7684\u5b9e\u9645\u73af\u5883\u4e0d\u7b26\uff0c\u8bf7\u6839\u636e\u4f60\u7684\u5b9e\u9645\u73af\u5883\u5bf9\u5176\u8fdb\u884c\u8c03\u6574\u3002 \u521b\u5efaDataLoad\u5bf9\u8c61 \u00a7 kubectl create -f dataload.yaml \u67e5\u770b\u521b\u5efa\u7684DataLoad\u5bf9\u8c61\u72b6\u6001 \u00a7 kubectl get dataload spark-dataload \u4e0a\u8ff0\u547d\u4ee4\u4f1a\u5f97\u5230\u7c7b\u4f3c\u4ee5\u4e0b\u7ed3\u679c\uff1a NAME DATASET PHASE AGE spark-dataload spark Loading 2m13s \u4f60\u4e5f\u53ef\u4ee5\u901a\u8fc7 kubectl describe \u83b7\u53d6\u6709\u5173\u8be5DataLoad\u7684\u66f4\u591a\u8be6\u7ec6\u4fe1\u606f\uff1a kubectl describe dataload spark-dataload \u5f97\u5230\u4ee5\u4e0b\u7ed3\u679c\uff1a Name: spark-dataload Namespace: default Labels: <none> Annotations: <none> API Version: data.fluid.io/v1alpha1 Kind: DataLoad ... Spec: Dataset: Name: spark Namespace: default Status: Conditions: Phase: Loading Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal DataLoadJobStarted 80s DataLoad The DataLoad job spark-dataload-loader-job started \u4e0a\u8ff0\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u6839\u636e\u4f60\u6240\u5728\u7684\u7f51\u7edc\u73af\u5883\u4e0d\u540c\uff0c\u53ef\u80fd\u4f1a\u8017\u8d39\u6570\u5206\u949f \u7b49\u5f85\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5b8c\u6210 \u00a7 kubectl get dataload spark-dataload \u4f60\u4f1a\u770b\u5230\u8be5DataLoad\u7684 Phase \u72b6\u6001\u5df2\u7ecf\u4ece Loading \u53d8\u4e3a Complete \uff0c\u8fd9\u8868\u660e\u6574\u4e2a\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5df2\u7ecf\u5b8c\u6210 NAME DATASET PHASE AGE spark-dataload spark Complete 5m17s \u6b64\u65f6\u518d\u6b21\u67e5\u770bDataset\u5bf9\u8c61\u7684\u7f13\u5b58\u72b6\u6001\uff1a kubectl get dataset spark \u53ef\u53d1\u73b0\uff0c\u8fdc\u7a0b\u5b58\u50a8\u7cfb\u7edf\u4e2d\u7684\u5168\u90e8\u6570\u636e\u5747\u5df2\u6210\u529f\u7f13\u5b58\u5230\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE spark 1.92GiB 1.92GiB 4.00GiB 100.0% Bound 7m41s DataLoad\u8fdb\u9636\u914d\u7f6e \u00a7 \u6307\u5b9a\u4e00\u4e2a\u6216\u591a\u4e2a\u6570\u636e\u96c6\u5b50\u76ee\u5f55\u8fdb\u884c\u52a0\u8f7d \u00a7 \u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6\u53ef\u4ee5\u52a0\u8f7d\u6307\u5b9a\u7684\u5b50\u76ee\u5f55(\u6216\u6587\u4ef6)\uff0c\u800c\u4e0d\u662f\u6574\u4e2a\u6570\u636e\u96c6\uff0c\u4f8b\u5982\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 - path: /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \u4e0a\u8ff0DataLoad\u4ec5\u4f1a\u52a0\u8f7d /sparkV1/spark-2.4.7 \u76ee\u5f55\u4e0b\u7684\u5168\u90e8\u6587\u4ef6\uff0c\u4ee5\u53ca /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \u6587\u4ef6 \u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf \u00a7 \u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6,\u4f60\u4e5f\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u63a7\u5236\u52a0\u8f7d\u7684\u6570\u636e\u526f\u672c\u6570\u91cf\uff0c\u4f8b\u5982\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 replicas: 1 - path: /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz replicas: 2 \u4e0a\u8ff0DataLoad\u5728\u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6\uff0c\u5bf9\u4e8e /sparkV1/spark-2.4.7 \u76ee\u5f55\u4e0b\u7684\u6587\u4ef6\u4ec5\u4f1a\u5728\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d\u4fdd\u7559 1\u4efd \u6570\u636e\u7f13\u5b58\u526f\u672c\uff0c\u800c\u5bf9\u4e8e\u6587\u4ef6 /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \uff0c\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u5c06\u4f1a\u4fdd\u7559 2\u4efd \u7f13\u5b58\u526f\u672c\u3002 \u539f\u5b50\u6027\u7f13\u5b58 \u00a7 \u5982\u60a8\u60f3\u4fdd\u6301\u7f13\u5b58\u4e00\u81f4\u6027\u548c\u539f\u5b50\u6027\uff0c\u90a3\u4e48\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b yaml \u6765\u5b8c\u6210\u8fd9\u4e2a\u529f\u80fd apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default options: atomicCache: \"true\" \u5355\u72ec\u8fdb\u884c\u5143\u6570\u636e\u7f13\u5b58\uff0c\u4e0d\u505a\u6570\u636e\u7f13\u5b58 \u00a7 \u5728\u8fdb\u884c dataload \u9884\u70ed\u65f6\uff0c\u9ed8\u8ba4\u662f\u4f1a\u505a\u6570\u636e\u7f13\u5b58\uff0c\u539f\u6570\u636e\u7f13\u5b58\u662f\u53ef\u9009\uff0c\u4f46\u5982\u679c\u60a8\u53ea\u60f3\u5143\u6570\u636e\u7f13\u5b58\u4e0d\u60f3\u505a\u6570\u636e\u7f13\u5b58\u7684\u8bdd\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b yaml \u6253\u5f00\u53ea\u505a\u5143\u6570\u636e\u7f13\u5b58\u7684\u5f00\u5173 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default options: loadMetadataOnly: \"true\" \u901a\u8fc7\u901a\u914d\u7b26\u8fdb\u884c\u7f13\u5b58\u8fc7\u6ee4 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 replicas: 1 options: filter: \"*.tgz\" cacheThread: \"2\" dryrun: \"false\" \u4f7f\u7528\u5217\u8868\u6587\u4ef6\u8fdb\u884c\u7f13\u5b58 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true options: accessKeyId: \"xxx\" accessKeySecret: \"xxx\" endpoint: \"oss-cn-shanghai-internal.aliyuncs.com\" url: \"oss://test-bucket/filelist\" dryrun: \"false\" \u7f13\u5b58\u8fdb\u5ea6\u53caLOG\u67e5\u770b \u00a7 $ kubectl get dataload spark-dataload NAME DATASET PHASE AGE spark-dataload spark Loading 2m13s \u67e5\u770b\u5f53\u524d\u7f13\u5b58\u7684\u76f8\u5173\u4fe1\u606f kubectl logs spark-dataload \u73af\u5883\u6e05\u7406 \u00a7 $ kubectl delete dataload spark-dataload","title":"\u6570\u636e\u9884\u52a0\u8f7d"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#_1","text":"\u914d\u7f6e\u5f85\u521b\u5efa\u7684DataLoad\u5bf9\u8c61 \u521b\u5efaDataLoad\u5bf9\u8c61 \u67e5\u770b\u521b\u5efa\u7684DataLoad\u5bf9\u8c61\u72b6\u6001 \u7b49\u5f85\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5b8c\u6210 DataLoad\u8fdb\u9636\u914d\u7f6e \u6307\u5b9a\u4e00\u4e2a\u6216\u591a\u4e2a\u6570\u636e\u96c6\u5b50\u76ee\u5f55\u8fdb\u884c\u52a0\u8f7d \u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf \u539f\u5b50\u6027\u7f13\u5b58 \u5355\u72ec\u8fdb\u884c\u5143\u6570\u636e\u7f13\u5b58\uff0c\u4e0d\u505a\u6570\u636e\u7f13\u5b58 \u7f13\u5b58\u8fdb\u5ea6\u53caLOG\u67e5\u770b \u73af\u5883\u6e05\u7406 \u4e3a\u4e86\u4fdd\u8bc1\u5e94\u7528\u5728\u8bbf\u95ee\u6570\u636e\u65f6\u7684\u6027\u80fd\uff0c\u53ef\u4ee5\u901a\u8fc7 \u6570\u636e\u9884\u52a0\u8f7d \u63d0\u524d\u5c06\u8fdc\u7a0b\u5b58\u50a8\u7cfb\u7edf\u4e2d\u7684\u6570\u636e\u62c9\u53d6\u5230\u9760\u8fd1\u8ba1\u7b97\u7ed3\u70b9\u7684\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d\uff0c\u4f7f\u5f97\u6d88\u8d39\u8be5\u6570\u636e\u96c6\u7684\u5e94\u7528\u80fd\u591f\u5728\u9996\u6b21\u8fd0\u884c\u65f6\u5373\u53ef\u4eab\u53d7\u5230\u7f13\u5b58\u5e26\u6765\u7684\u52a0\u901f\u6548\u679c\u3002 \u4e3a\u6b64\uff0c\u6211\u4eec\u63d0\u4f9b\u4e86DataLoad CRD, \u8be5CRD\u8ba9\u4f60\u53ef\u901a\u8fc7\u7b80\u5355\u7684\u914d\u7f6e\u5c31\u80fd\u5b8c\u6210\u6574\u4e2a\u6570\u636e\u9884\u52a0\u8f7d\u8fc7\u7a0b\uff0c\u5e76\u5bf9\u6570\u636e\u9884\u52a0\u8f7d\u7684\u8bb8\u591a\u884c\u4e3a\u8fdb\u884c\u81ea\u5b9a\u4e49\u63a7\u5236\u3002 \u5047\u8bbe\u6211\u4eec\u5df2\u7ecf\u6709\u5982\u4e0b\u4e00\u4e2aname\u4e3a spark \u7684dataset\uff1a apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: spark spec: mounts: - mountPoint: oss://test-bucket/spark name: sparkV1 accessModes: - ReadWriteMany \u63a5\u4e0b\u6765\u5bf9\u8fd9\u4e2adataset\u8fdb\u884c\u9884\u70ed\uff0c\u8bf7\u6ce8\u610f\u4e0b\u6587\u4e2d\u7684 /sparkV1 \u4e3a mounts[].name \u7684\u503c\uff0c\u5982\u679c\u60a8\u7684 mounts[].path \u4e3a / , \u90a3\u4e48\u4e0b\u6587\u4e2d\u7684 /sparkV1 \u53ef\u4ee5\u76f4\u63a5\u6539\u4e3a /","title":"\u6570\u636e\u9884\u52a0\u8f7d"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#dataload","text":"apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default target: - path: /sparkV1/ spec.dataset \u6307\u660e\u4e86\u9700\u8981\u8fdb\u884c\u6570\u636e\u9884\u52a0\u8f7d\u7684\u76ee\u6807\u6570\u636e\u96c6\uff0c\u5728\u8be5\u4f8b\u5b50\u4e2d\uff0c\u6211\u4eec\u7684\u6570\u636e\u9884\u52a0\u8f7d\u76ee\u6807\u4e3a default \u547d\u540d\u7a7a\u95f4\u4e0b\u540d\u4e3a spark \u7684\u6570\u636e\u96c6\uff0c\u5982\u679c\u8be5\u914d\u7f6e\u4e0e\u4f60\u6240\u5728\u7684\u5b9e\u9645\u73af\u5883\u4e0d\u7b26\uff0c\u8bf7\u6839\u636e\u4f60\u7684\u5b9e\u9645\u73af\u5883\u5bf9\u5176\u8fdb\u884c\u8c03\u6574\u3002","title":"\u914d\u7f6e\u5f85\u521b\u5efa\u7684DataLoad\u5bf9\u8c61"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#dataload_1","text":"kubectl create -f dataload.yaml","title":"\u521b\u5efaDataLoad\u5bf9\u8c61"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#dataload_2","text":"kubectl get dataload spark-dataload \u4e0a\u8ff0\u547d\u4ee4\u4f1a\u5f97\u5230\u7c7b\u4f3c\u4ee5\u4e0b\u7ed3\u679c\uff1a NAME DATASET PHASE AGE spark-dataload spark Loading 2m13s \u4f60\u4e5f\u53ef\u4ee5\u901a\u8fc7 kubectl describe \u83b7\u53d6\u6709\u5173\u8be5DataLoad\u7684\u66f4\u591a\u8be6\u7ec6\u4fe1\u606f\uff1a kubectl describe dataload spark-dataload \u5f97\u5230\u4ee5\u4e0b\u7ed3\u679c\uff1a Name: spark-dataload Namespace: default Labels: <none> Annotations: <none> API Version: data.fluid.io/v1alpha1 Kind: DataLoad ... Spec: Dataset: Name: spark Namespace: default Status: Conditions: Phase: Loading Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal DataLoadJobStarted 80s DataLoad The DataLoad job spark-dataload-loader-job started \u4e0a\u8ff0\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u6839\u636e\u4f60\u6240\u5728\u7684\u7f51\u7edc\u73af\u5883\u4e0d\u540c\uff0c\u53ef\u80fd\u4f1a\u8017\u8d39\u6570\u5206\u949f","title":"\u67e5\u770b\u521b\u5efa\u7684DataLoad\u5bf9\u8c61\u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#_2","text":"kubectl get dataload spark-dataload \u4f60\u4f1a\u770b\u5230\u8be5DataLoad\u7684 Phase \u72b6\u6001\u5df2\u7ecf\u4ece Loading \u53d8\u4e3a Complete \uff0c\u8fd9\u8868\u660e\u6574\u4e2a\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5df2\u7ecf\u5b8c\u6210 NAME DATASET PHASE AGE spark-dataload spark Complete 5m17s \u6b64\u65f6\u518d\u6b21\u67e5\u770bDataset\u5bf9\u8c61\u7684\u7f13\u5b58\u72b6\u6001\uff1a kubectl get dataset spark \u53ef\u53d1\u73b0\uff0c\u8fdc\u7a0b\u5b58\u50a8\u7cfb\u7edf\u4e2d\u7684\u5168\u90e8\u6570\u636e\u5747\u5df2\u6210\u529f\u7f13\u5b58\u5230\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE spark 1.92GiB 1.92GiB 4.00GiB 100.0% Bound 7m41s","title":"\u7b49\u5f85\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5b8c\u6210"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#dataload_3","text":"","title":"DataLoad\u8fdb\u9636\u914d\u7f6e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#_3","text":"\u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6\u53ef\u4ee5\u52a0\u8f7d\u6307\u5b9a\u7684\u5b50\u76ee\u5f55(\u6216\u6587\u4ef6)\uff0c\u800c\u4e0d\u662f\u6574\u4e2a\u6570\u636e\u96c6\uff0c\u4f8b\u5982\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 - path: /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \u4e0a\u8ff0DataLoad\u4ec5\u4f1a\u52a0\u8f7d /sparkV1/spark-2.4.7 \u76ee\u5f55\u4e0b\u7684\u5168\u90e8\u6587\u4ef6\uff0c\u4ee5\u53ca /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \u6587\u4ef6","title":"\u6307\u5b9a\u4e00\u4e2a\u6216\u591a\u4e2a\u6570\u636e\u96c6\u5b50\u76ee\u5f55\u8fdb\u884c\u52a0\u8f7d"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#_4","text":"\u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6,\u4f60\u4e5f\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u63a7\u5236\u52a0\u8f7d\u7684\u6570\u636e\u526f\u672c\u6570\u91cf\uff0c\u4f8b\u5982\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 replicas: 1 - path: /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz replicas: 2 \u4e0a\u8ff0DataLoad\u5728\u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6\uff0c\u5bf9\u4e8e /sparkV1/spark-2.4.7 \u76ee\u5f55\u4e0b\u7684\u6587\u4ef6\u4ec5\u4f1a\u5728\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d\u4fdd\u7559 1\u4efd \u6570\u636e\u7f13\u5b58\u526f\u672c\uff0c\u800c\u5bf9\u4e8e\u6587\u4ef6 /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \uff0c\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u5c06\u4f1a\u4fdd\u7559 2\u4efd \u7f13\u5b58\u526f\u672c\u3002","title":"\u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#_5","text":"\u5982\u60a8\u60f3\u4fdd\u6301\u7f13\u5b58\u4e00\u81f4\u6027\u548c\u539f\u5b50\u6027\uff0c\u90a3\u4e48\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b yaml \u6765\u5b8c\u6210\u8fd9\u4e2a\u529f\u80fd apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default options: atomicCache: \"true\"","title":"\u539f\u5b50\u6027\u7f13\u5b58"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#_6","text":"\u5728\u8fdb\u884c dataload \u9884\u70ed\u65f6\uff0c\u9ed8\u8ba4\u662f\u4f1a\u505a\u6570\u636e\u7f13\u5b58\uff0c\u539f\u6570\u636e\u7f13\u5b58\u662f\u53ef\u9009\uff0c\u4f46\u5982\u679c\u60a8\u53ea\u60f3\u5143\u6570\u636e\u7f13\u5b58\u4e0d\u60f3\u505a\u6570\u636e\u7f13\u5b58\u7684\u8bdd\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b yaml \u6253\u5f00\u53ea\u505a\u5143\u6570\u636e\u7f13\u5b58\u7684\u5f00\u5173 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default options: loadMetadataOnly: \"true\"","title":"\u5355\u72ec\u8fdb\u884c\u5143\u6570\u636e\u7f13\u5b58\uff0c\u4e0d\u505a\u6570\u636e\u7f13\u5b58"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#_7","text":"apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 replicas: 1 options: filter: \"*.tgz\" cacheThread: \"2\" dryrun: \"false\"","title":"\u901a\u8fc7\u901a\u914d\u7b26\u8fdb\u884c\u7f13\u5b58\u8fc7\u6ee4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#_8","text":"apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true options: accessKeyId: \"xxx\" accessKeySecret: \"xxx\" endpoint: \"oss-cn-shanghai-internal.aliyuncs.com\" url: \"oss://test-bucket/filelist\" dryrun: \"false\"","title":"\u4f7f\u7528\u5217\u8868\u6587\u4ef6\u8fdb\u884c\u7f13\u5b58"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#log","text":"$ kubectl get dataload spark-dataload NAME DATASET PHASE AGE spark-dataload spark Loading 2m13s \u67e5\u770b\u5f53\u524d\u7f13\u5b58\u7684\u76f8\u5173\u4fe1\u606f kubectl logs spark-dataload","title":"\u7f13\u5b58\u8fdb\u5ea6\u53caLOG\u67e5\u770b"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_dataload/#_9","text":"$ kubectl delete dataload spark-dataload","title":"\u73af\u5883\u6e05\u7406"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_download/","text":"\u4e0b\u8f7d 1.0.0-20230602 0.8.0-20221105 0.8.0-20220729 0.8.0-20220630 0.8.0-20220625 0.8.0-20220606 0.8.0-20220526 \u4e0b\u8f7d \u00a7 1.0.0-20230602 \u00a7 \u66f4\u65b0\u65e5\u671f\uff1a2022-11-05 Fluid 1.0.0 JindoRuntime\u76f8\u5173\u529f\u80fd\u5347\u7ea7\uff1a * JindoFSx \u5347\u7ea7\u5230 4.6.12 0.8.0-20221105 \u00a7 \u66f4\u65b0\u65e5\u671f\uff1a2022-11-05 Fluid 0.8.0 JindoRuntime\u76f8\u5173\u529f\u80fd\u5347\u7ea7\uff1a * JindoFSx \u5347\u7ea7\u5230 4.5.2 0.8.0-20220729 \u00a7 \u66f4\u65b0\u65e5\u671f\uff1a2022-07-29 Fluid 0.8.0 commitId:e7d4c18 JindoRuntime\u76f8\u5173\u529f\u80fd\u5347\u7ea7\uff1a * JindoFSx \u5347\u7ea7\u5230 4.5.1 0.8.0-20220630 \u00a7 \u66f4\u65b0\u65e5\u671f\uff1a2022-06-30 Fluid 0.8.0 commitId:32837a3 0.8.0-20220625 \u00a7 \u66f4\u65b0\u65e5\u671f\uff1a2022-06-25 Fluid 0.8.0 commitId:510b30b 0.8.0-20220606 \u00a7 \u66f4\u65b0\u65e5\u671f\uff1a2022-05-26 Fluid 0.8.0 commitId:e305ff1 0.8.0-20220526 \u00a7 \u66f4\u65b0\u65e5\u671f\uff1a2022-05-26 Fluid 0.8.0 commitId:e562a90 JindoRuntime\u76f8\u5173\u529f\u80fd\u5347\u7ea7\uff1a * JindoFSx \u5347\u7ea7\u5230 4.6.12","title":"Jindo fluid download"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_download/#_1","text":"","title":"\u4e0b\u8f7d"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_download/#100-20230602","text":"\u66f4\u65b0\u65e5\u671f\uff1a2022-11-05 Fluid 1.0.0 JindoRuntime\u76f8\u5173\u529f\u80fd\u5347\u7ea7\uff1a * JindoFSx \u5347\u7ea7\u5230 4.6.12","title":"1.0.0-20230602"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_download/#080-20221105","text":"\u66f4\u65b0\u65e5\u671f\uff1a2022-11-05 Fluid 0.8.0 JindoRuntime\u76f8\u5173\u529f\u80fd\u5347\u7ea7\uff1a * JindoFSx \u5347\u7ea7\u5230 4.5.2","title":"0.8.0-20221105"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_download/#080-20220729","text":"\u66f4\u65b0\u65e5\u671f\uff1a2022-07-29 Fluid 0.8.0 commitId:e7d4c18 JindoRuntime\u76f8\u5173\u529f\u80fd\u5347\u7ea7\uff1a * JindoFSx \u5347\u7ea7\u5230 4.5.1","title":"0.8.0-20220729"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_download/#080-20220630","text":"\u66f4\u65b0\u65e5\u671f\uff1a2022-06-30 Fluid 0.8.0 commitId:32837a3","title":"0.8.0-20220630"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_download/#080-20220625","text":"\u66f4\u65b0\u65e5\u671f\uff1a2022-06-25 Fluid 0.8.0 commitId:510b30b","title":"0.8.0-20220625"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_download/#080-20220606","text":"\u66f4\u65b0\u65e5\u671f\uff1a2022-05-26 Fluid 0.8.0 commitId:e305ff1","title":"0.8.0-20220606"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_download/#080-20220526","text":"\u66f4\u65b0\u65e5\u671f\uff1a2022-05-26 Fluid 0.8.0 commitId:e562a90 JindoRuntime\u76f8\u5173\u529f\u80fd\u5347\u7ea7\uff1a * JindoFSx \u5347\u7ea7\u5230 4.6.12","title":"0.8.0-20220526"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/","text":"\u52a0\u901fHDFS\u4e0a\u6570\u636e \u00a7 JindoFSx \u63d0\u4f9b\u4e86\u8bbf\u95ee HDFS \u7684\u80fd\u529b\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e HDFS \u4f5c\u4e3a JindoFSx \u7684\u540e\u7aef\u5b58\u50a8\uff0c\u4f7f Fluid \u901a\u8fc7 JindoFSx \u6765\u8bbf\u95ee HDFS \u4e0a\u7684\u6570\u636e\uff0c\u540c\u65f6 JindoFSx \u4e5f\u63d0\u4f9b\u4e86\u5bf9 HDFS \u4e0a\u7684\u6570\u636e\u4ee5\u53ca\u5143\u6570\u636e\u7684\u7f13\u5b58\u52a0\u901f\u529f\u80fd\u3002 \u672c\u6587\u6863\u5c55\u793a\u4e86\u5982\u4f55\u5728 Fluid \u4e2d\u4ee5\u58f0\u660e\u5f0f\u7684\u65b9\u5f0f\u5b8c\u6210 JindoFSx \u90e8\u7f72\uff0c\u5bf9\u63a5 HDFS \u6570\u636e\u6e90\u3002 \u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 \u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz \u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 1\u3001\u521b\u5efa dataset \u548c JindoRuntime \u00a7 1.1\u3001HA \u96c6\u7fa4\u4ece HDFS \u914d\u7f6e\u6587\u4ef6\u4e2d\u76f8\u5173\u4fe1\u606f\uff0c\u975e HA \u96c6\u7fa4\u8bf7\u7565\u8fc7\u6b64\u6b65\u9aa4 \u00a7 \u4ece HDFS \u96c6\u7fa4\u4e0a\u627e\u5230 hdfs-site.xml \u914d\u7f6e\u6587\u4ef6 * \u963f\u91cc\u4e91EMR\u96c6\u7fa4\u5728 /etc/ecm/hadoop-conf \u8def\u5f84\u4e0b * CDH \u96c6\u7fa4\u53ef\u53c2\u8003 What-is-the-Path-of-hdfs-site-xml-core-xml \u9700\u8981\u5c06\u5176\u4e2d rpc \u7684\u8bbf\u95ee\u5730\u5740\u4e2d\u57df\u540d\u7684\u90e8\u5206\u53c2\u6570\uff0c\u4fee\u6539\u6210\u53ef\u8bbf\u95ee\u7684\u5185\u7f51 IP \u5730\u5740\uff0c\u4f8b\u5982\uff1a <property> <name>dfs.namenode.rpc-address.emr-cluster.nn1</name> <value>emr-header-1.cluster-xxx:8020</value> </property> <property> <name>dfs.namenode.rpc-address.emr-cluster.nn2</name> <value>emr-header-2.cluster-xxx:8020</value> </property> \u4fee\u6539\u4e3a <property> <name>dfs.namenode.rpc-address.emr-cluster.nn1</name> <value>192.168.0.1:8020</value> </property> <property> <name>dfs.namenode.rpc-address.emr-cluster.nn2</name> <value>192.168.0.2:8020</value> </property> \u4ece\u914d\u7f6e\u6587\u4ef6\u91cc\u6211\u4eec\u53ef\u4ee5\u62ff\u5230\u5982\u4e0b\u51e0\u4e2a\u4fe1\u606f \u96c6\u7fa4namespaceService\u540d\u79f0\uff1aemr-cluster \u4e24\u4e2anamenode\u540d\u5b57\uff1ann1 \u548c nn2 \u4e24\u4e2anamenode\u7684\u8bbf\u95ee\u5730\u5740\uff1a192.168.0.1:8020 \u548c 192.168.0.2:8020 1.2\u3001\u521b\u5efa Dataset \u8d44\u6e90\u5bf9\u8c61 \u00a7 \u9996\u5148\u521b\u5efa\u4e00\u4e2a\u540d\u4e3a dataset.yaml \u7684\u6587\u4ef6\uff0c\u9488\u5bf9 HDFS \u96c6\u7fa4\u5206\u4e3a HA \u96c6\u7fa4\u548c\u975e HA \u96c6\u7fa4\u4e24\u79cd\u4e0d\u540c\u7684\u60c5\u51b5\u3002 HA \u96c6\u7fa4 \u00a7 \u5c061.1\u4e2d\u62ff\u5230\u7684\u51e0\u4e2a\u4fe1\u606f\u653e\u5230 mounts.options \u91cc\u9762\uff0cuser \u4ee3\u8868\u7528\u6237\u6743\u9650 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hdfs spec: mounts: - mountPoint: hdfs://emr-cluster/ name: hdfs options: fs.hdfs.emr-cluster.dfs.ha.namenodes = nn1,nn2 fs.hdfs.emr-cluster.dfs.namenode.rpc-address.nn1 = 192.168.0.1:8020 fs.hdfs.emr-cluster.dfs.namenode.rpc-address.nn2 = 192.168.0.2:8020 fs.hdfs.user = hadoop path: / accessModes: - ReadOnlyMany \u5176\u4e2d mountPoint \u4e3a\u9700\u8981\u8bbf\u95ee\u7684 HDFS \u8def\u5f84\uff08\u963f\u91cc\u4e91EMR\u96c6\u7fa4\u9ed8\u8ba4\u540d\u5b57\u4e3a emr-cluster\uff0c\u5982\u4e0b\u6240\u793a\uff09 - accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 <property> <name>fs.defaultFS</name> <value>hdfs://emr-cluster</value> </property> \u975e HA \u96c6\u7fa4 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hdfs spec: mounts: - mountPoint: hdfs://<namenodeIP>:<port>/ name: hdfs path: / accessModes: - ReadOnlyMany \u5176\u4e2d mountPoint \u4e3a HDFS \u96c6\u7fa4\u7684 namenode \u7684 IP \u5730\u5740\u548c\u7aef\u53e3\u7684\u7ec4\u5408 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 \u00a7 1.3\u3001\u521b\u5efa Dataset \u00a7 $ kubectl create -f dataset.yaml 1.4 \u521b\u5efa JindoRuntime \u8d44\u6e90\u5bf9\u8c61 \u00a7 \u521b\u5efa\u4e00\u4e2a JindoRuntime \u5bf9\u5e94\u7684 JindoFSx \u96c6\u7fa4 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hdfs spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /mnt/disk1/ quota: 290Gi high: \"0.9\" low: \"0.2\" \u591a\u5757\u76d8\u591a\u4e2aquota\u8bf7\u53c2\u8003 path: /mnt/disk1/,/mnt/disk2/,/mnt/disk3/ quotaList: 290G,290G,290G \u5176\u4e2d path \u548c quota \u7684\u6570\u91cf\u5e94\u8be5\u76f8\u540c\u3002 1.5\u3001\u521b\u5efa JindoRuntime \u00a7 \u6267\u884c create \u547d\u4ee4 $ kubectl create -f runtime.yaml \u67e5\u770b dataset \u72b6\u6001 \u00a7 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hdfs 180.01GiB 0.00B 261.00GiB 0.0% Bound 62m \u68c0\u67e5 PV\uff0cPVC \u521b\u5efa\u60c5\u51b5 \u00a7 $ kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hdfs 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hdfs Bound hadoop 100Gi RWX 58m \u68c0\u67e5 jindoruntime \u90e8\u7f72\u60c5\u51b5 \u00a7 $ kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hdfs Ready Ready 62m","title":"\u52a0\u901fHDFS\u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#hdfs","text":"JindoFSx \u63d0\u4f9b\u4e86\u8bbf\u95ee HDFS \u7684\u80fd\u529b\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e HDFS \u4f5c\u4e3a JindoFSx \u7684\u540e\u7aef\u5b58\u50a8\uff0c\u4f7f Fluid \u901a\u8fc7 JindoFSx \u6765\u8bbf\u95ee HDFS \u4e0a\u7684\u6570\u636e\uff0c\u540c\u65f6 JindoFSx \u4e5f\u63d0\u4f9b\u4e86\u5bf9 HDFS \u4e0a\u7684\u6570\u636e\u4ee5\u53ca\u5143\u6570\u636e\u7684\u7f13\u5b58\u52a0\u901f\u529f\u80fd\u3002 \u672c\u6587\u6863\u5c55\u793a\u4e86\u5982\u4f55\u5728 Fluid \u4e2d\u4ee5\u58f0\u660e\u5f0f\u7684\u65b9\u5f0f\u5b8c\u6210 JindoFSx \u90e8\u7f72\uff0c\u5bf9\u63a5 HDFS \u6570\u636e\u6e90\u3002","title":"\u52a0\u901fHDFS\u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#_1","text":"kubectl create ns fluid-system","title":"\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#fluid","text":"","title":"\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#fluid_1","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#1-dataset-jindoruntime","text":"","title":"1\u3001\u521b\u5efa dataset \u548c JindoRuntime"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#11ha-hdfs-ha","text":"\u4ece HDFS \u96c6\u7fa4\u4e0a\u627e\u5230 hdfs-site.xml \u914d\u7f6e\u6587\u4ef6 * \u963f\u91cc\u4e91EMR\u96c6\u7fa4\u5728 /etc/ecm/hadoop-conf \u8def\u5f84\u4e0b * CDH \u96c6\u7fa4\u53ef\u53c2\u8003 What-is-the-Path-of-hdfs-site-xml-core-xml \u9700\u8981\u5c06\u5176\u4e2d rpc \u7684\u8bbf\u95ee\u5730\u5740\u4e2d\u57df\u540d\u7684\u90e8\u5206\u53c2\u6570\uff0c\u4fee\u6539\u6210\u53ef\u8bbf\u95ee\u7684\u5185\u7f51 IP \u5730\u5740\uff0c\u4f8b\u5982\uff1a <property> <name>dfs.namenode.rpc-address.emr-cluster.nn1</name> <value>emr-header-1.cluster-xxx:8020</value> </property> <property> <name>dfs.namenode.rpc-address.emr-cluster.nn2</name> <value>emr-header-2.cluster-xxx:8020</value> </property> \u4fee\u6539\u4e3a <property> <name>dfs.namenode.rpc-address.emr-cluster.nn1</name> <value>192.168.0.1:8020</value> </property> <property> <name>dfs.namenode.rpc-address.emr-cluster.nn2</name> <value>192.168.0.2:8020</value> </property> \u4ece\u914d\u7f6e\u6587\u4ef6\u91cc\u6211\u4eec\u53ef\u4ee5\u62ff\u5230\u5982\u4e0b\u51e0\u4e2a\u4fe1\u606f \u96c6\u7fa4namespaceService\u540d\u79f0\uff1aemr-cluster \u4e24\u4e2anamenode\u540d\u5b57\uff1ann1 \u548c nn2 \u4e24\u4e2anamenode\u7684\u8bbf\u95ee\u5730\u5740\uff1a192.168.0.1:8020 \u548c 192.168.0.2:8020","title":"1.1\u3001HA \u96c6\u7fa4\u4ece HDFS \u914d\u7f6e\u6587\u4ef6\u4e2d\u76f8\u5173\u4fe1\u606f\uff0c\u975e HA \u96c6\u7fa4\u8bf7\u7565\u8fc7\u6b64\u6b65\u9aa4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#12-dataset","text":"\u9996\u5148\u521b\u5efa\u4e00\u4e2a\u540d\u4e3a dataset.yaml \u7684\u6587\u4ef6\uff0c\u9488\u5bf9 HDFS \u96c6\u7fa4\u5206\u4e3a HA \u96c6\u7fa4\u548c\u975e HA \u96c6\u7fa4\u4e24\u79cd\u4e0d\u540c\u7684\u60c5\u51b5\u3002","title":"1.2\u3001\u521b\u5efa Dataset \u8d44\u6e90\u5bf9\u8c61"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#ha","text":"\u5c061.1\u4e2d\u62ff\u5230\u7684\u51e0\u4e2a\u4fe1\u606f\u653e\u5230 mounts.options \u91cc\u9762\uff0cuser \u4ee3\u8868\u7528\u6237\u6743\u9650 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hdfs spec: mounts: - mountPoint: hdfs://emr-cluster/ name: hdfs options: fs.hdfs.emr-cluster.dfs.ha.namenodes = nn1,nn2 fs.hdfs.emr-cluster.dfs.namenode.rpc-address.nn1 = 192.168.0.1:8020 fs.hdfs.emr-cluster.dfs.namenode.rpc-address.nn2 = 192.168.0.2:8020 fs.hdfs.user = hadoop path: / accessModes: - ReadOnlyMany \u5176\u4e2d mountPoint \u4e3a\u9700\u8981\u8bbf\u95ee\u7684 HDFS \u8def\u5f84\uff08\u963f\u91cc\u4e91EMR\u96c6\u7fa4\u9ed8\u8ba4\u540d\u5b57\u4e3a emr-cluster\uff0c\u5982\u4e0b\u6240\u793a\uff09 - accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 <property> <name>fs.defaultFS</name> <value>hdfs://emr-cluster</value> </property>","title":"HA \u96c6\u7fa4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#ha_1","text":"apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hdfs spec: mounts: - mountPoint: hdfs://<namenodeIP>:<port>/ name: hdfs path: / accessModes: - ReadOnlyMany \u5176\u4e2d mountPoint \u4e3a HDFS \u96c6\u7fa4\u7684 namenode \u7684 IP \u5730\u5740\u548c\u7aef\u53e3\u7684\u7ec4\u5408 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199","title":"\u975e HA \u96c6\u7fa4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#_2","text":"","title":""},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#13-dataset","text":"$ kubectl create -f dataset.yaml","title":"1.3\u3001\u521b\u5efa Dataset"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#14-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a JindoRuntime \u5bf9\u5e94\u7684 JindoFSx \u96c6\u7fa4 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hdfs spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /mnt/disk1/ quota: 290Gi high: \"0.9\" low: \"0.2\" \u591a\u5757\u76d8\u591a\u4e2aquota\u8bf7\u53c2\u8003 path: /mnt/disk1/,/mnt/disk2/,/mnt/disk3/ quotaList: 290G,290G,290G \u5176\u4e2d path \u548c quota \u7684\u6570\u91cf\u5e94\u8be5\u76f8\u540c\u3002","title":"1.4 \u521b\u5efa JindoRuntime \u8d44\u6e90\u5bf9\u8c61"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#15-jindoruntime","text":"\u6267\u884c create \u547d\u4ee4 $ kubectl create -f runtime.yaml","title":"1.5\u3001\u521b\u5efa JindoRuntime"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#dataset","text":"$ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hdfs 180.01GiB 0.00B 261.00GiB 0.0% Bound 62m","title":"\u67e5\u770b dataset \u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#pvpvc","text":"$ kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hdfs 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hdfs Bound hadoop 100Gi RWX 58m","title":"\u68c0\u67e5 PV\uff0cPVC \u521b\u5efa\u60c5\u51b5"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_hdfs_ufs_example/#jindoruntime","text":"$ kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hdfs Ready Ready 62m","title":"\u68c0\u67e5 jindoruntime \u90e8\u7f72\u60c5\u51b5"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_monitor_metric/","text":"\u76d1\u63a7\u548c\u53ef\u89c2\u6d4b\u6027\u529f\u80fd \u00a7 \u672c\u6587\u4ecb\u7ecd\u5728\u542f\u52a8 Fluid JindoRuntime \u7684\u524d\u63d0\u4e0b\uff0c\u901a\u8fc7 Prometheus + Grafana \u6536\u96c6\u7f13\u5b58\u7cfb\u7edf\u76f8\u5173\u7684 metrics \u6307\u6807\uff0c\u8fdb\u884c\u53ef\u89c6\u5316\u5c55\u793a\u7684\u4f7f\u7528\u65b9\u6cd5\u3002 \u4f7f\u7528 In-Cluster \u6a21\u5f0f\u90e8\u7f72 prometheus \u548c grafana \u76d1\u63a7 Fluid \u5e94\u7528 \u00a7 1. \u90e8\u7f72\u6216\u914d\u7f6e Prometheus \u00a7 \u5982\u679c\u96c6\u7fa4\u5185\u65e0 prometheus\uff0c\u8bf7\u70b9\u51fb\u4e0b\u8f7d prometheus-jindofs.yaml \u6587\u4ef6 $ kubectl apply -f prometheus-jindofs.yaml \u5982\u96c6\u7fa4\u5185\u6709 prometheus,\u53ef\u5c06\u4ee5\u4e0b\u914d\u7f6e\u5199\u5230 prometheus \u914d\u7f6e\u6587\u4ef6\u4e2d: scrape_configs: - job_name: 'jindo runtime' metrics_path: /jindometrics/prometheus kubernetes_sd_configs: - role: endpoints relabel_configs: - source_labels: [__meta_kubernetes_service_label_role] regex: jindofs-master action: keep - source_labels: [__meta_kubernetes_endpoint_port_name] regex: rpc action: keep - source_labels: [__meta_kubernetes_namespace] target_label: namespace replacement: $1 action: replace - source_labels: [__meta_kubernetes_service_label_release] target_label: fluid_runtime replacement: $1 action: replace - source_labels: [__meta_kubernetes_endpoint_address_target_name] target_label: pod replacement: $1 action: replace 2. \u90e8\u7f72 grafana \u00a7 \u5982\u679c\u96c6\u7fa4\u5185\u65e0 grafana\uff0c\u8bf7\u70b9\u51fb\u4e0b\u8f7d grafana.yaml \u6587\u4ef6 $ kubectl apply -f grafana.yaml 3. \u914d\u7f6e grafana \u00a7 \u767b\u5f55 grafana \u627e\u5230 grafana \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\u548c\u6539\u8282\u70b9\u5bf9\u5916\u53ef\u8bbf\u95ee grafana-node-ip \uff0c\u5982\u679c\u60f3\u8981\u5728\u516c\u7f51\u8bbf\u95ee\uff0c\u53ef\u4ee5\u7ed1\u5b9a\u516c\u7f51IP\uff0c $ kubectl get pod -n kube-system -o wide | grep grafana monitoring-grafana-7dfcf6d5cd-5k8qw 1/1 Running 0 108m 10.54.1.11 cn-shanghai.192.168.1.1 <none> <none> \u627e\u5230 grafana \u670d\u52a1\u5bf9\u5e94\u7684 NodePort $ kubectl describe svc monitoring-grafana -n kube-system Name: monitoring-grafana Namespace: kube-system Labels: <none> Annotations: <none> Selector: app=grafana Type: NodePort IP Families: <none> IP: 172.16.83.188 IPs: 172.16.83.188 Port: <unset> 80/TCP TargetPort: 3000/TCP NodePort: <unset> 30396/TCP Endpoints: 10.54.1.75:3000 Session Affinity: None External Traffic Policy: Cluster Events: <none> \u53ef\u4ee5\u770b\u5230\u5f53\u524d\u66b4\u9732\u7684 NodePort \u4e3a 30396 \u8bbf\u95ee http://$grafana-node-ip:NodePort \uff0c\u9ed8\u8ba4\u8d26\u53f7\u5bc6\u7801 admin:admin \u67e5\u770b prometheus svc \u7aef\u53e3 \u540c\u6837\u5148\u627e\u5230 prometheus \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\uff0c\u627e\u5230\u8be5\u8282\u70b9\u7684 IP $ kubectl get pod -n kube-system -o wide | grep grafana prometheus-deployment-d6d8554b4-wfl98 1/1 Running 0 17d 10.54.1.204 cn-shanghai.192.168.1.1 <none> <none> kubectl get no cn-shanghai.192.168.31.107 -o wide NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP cn-shanghai.192.168.1.1 Ready <none> 5h51m v1.20.4-aliyun.1 192.168.1.1 101.132.66.11 \u901a\u8fc7\u67e5\u770b\u8282\u70b9\u7684 INTERNAL-IP \uff0c\u53ef\u4e86\u89e3\u8be5\u8282\u70b9\u7684\u5185\u90e8\u8bbf\u95ee IP \u4e3a 192.168.1.1 $ kubectl get svc -n kube-system | grep prometheus-svc prometheus-svc NodePort 10.100.0.144 <none> 9090:31225/TCP 22h $ kubectl describe svc prometheus-svc -n kube-system Name: prometheus-svc Namespace: kube-system Labels: kubernetes.io/name=Prometheus name=prometheus-svc Annotations: kubectl.kubernetes.io/last-applied-configuration: {\"apiVersion\":\"v1\",\"kind\":\"Service\",\"metadata\":{\"annotations\":{},\"labels\":{\"kubernetes.io/name\":\"Prometheus\",\"name\":\"prometheus-svc\"},\"nam... Selector: app=prometheus Type: NodePort IP: 10.100.0.144 Port: prometheus 9090/TCP TargetPort: 9090/TCP NodePort: prometheus 31225/TCP Endpoints: 10.99.224.138:9090 Session Affinity: None External Traffic Policy: Cluster Events: <none> \u53ef\u4ee5\u770b\u5230\u5f53\u524d prometheus svc \u66b4\u9732\u7684\u7aef\u53e3\u662f 31225 \uff0c\u540e\u7eed\u4f7f\u7528 http://IP:NodePort \u7684\u5f62\u5f0f\u6765\u53d1\u73b0 prometheus \u670d\u52a1 \u914d\u7f6e prometheus data source \u4f7f\u7528 http://IP:NodePort \u7684\u5f62\u5f0f\u6765\u53d1\u73b0 prometheus \u670d\u52a1 \u5bfc\u5165\u5b8c\u6210\u540e\u70b9\u51fbSave & Test \u663e\u793a Data source is working \u5373\u53ef \u5bfc\u5165\u6a21\u677f\u6587\u4ef6 grafana \u9009\u62e9\u5bfc\u5165\u6a21\u677f Json \u6587\u4ef6\uff0c\u70b9\u51fb\u6b64\u5904\u4e0b\u8f7d fluid-prometheus-grafana-monitor-jindofs.json \u67e5\u770b\u76d1\u63a7 \u5728 DashBoards \u4e2d\u627e\u5230\u5bfc\u5165\u7684\u6a21\u7248\u89c6\u56fe \u6ce8\uff1afluid_runtime \u5bf9\u5e94Fluid runtime name; namespace \u5bf9\u5e94 Fluid runtime namespace JindoRuntime Prometheus \u6307\u6807\u9884\u89c8(\u66f4\u65b0\u4e2d) \u00a7 \u4e0b\u9762\u5bf9 JindoRuntime \u5f53\u524d\u6536\u96c6\u7684\u6307\u6807\u8fdb\u884c\u9884\u89c8\u548c\u63cf\u8ff0\uff0c\u60a8\u53ef\u4ee5\u57fa\u4e8e\u5df2\u6536\u96c6\u6307\u6807\u5728 Grafana \u4e2d\u8fdb\u884c\u7ec4\u5408\u548c\u56fe\u8868\u5c55\u793a\uff0c\u5982\u60a8\u9700\u8981\u989d\u5916\u7684\u57fa\u672c metrics \u5c55\u793a\uff0c\u8bf7\u5f00 issue \u8054\u7cfb\u6211\u4eec # HELP jindofsx_ns_backend_read_bytes_time_total_window # TYPE jindofsx_ns_backend_read_bytes_time_total_window gauge jindofsx_ns_backend_read_bytes_time_total_window 20367553 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_read_bytes_total # TYPE jindofsx_ns_backend_read_bytes_total gauge jindofsx_ns_backend_read_bytes_total 166703347021 #\u603b\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_bytes_total_window # TYPE jindofsx_ns_backend_read_bytes_total_window gauge jindofsx_ns_backend_read_bytes_total_window 1174671374 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_time_total # TYPE jindofsx_ns_backend_read_time_total gauge jindofsx_ns_backend_read_time_total 8717632094 #\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_readop_num_total # TYPE jindofsx_ns_backend_readop_num_total gauge jindofsx_ns_backend_readop_num_total 181951 #\u5e95\u5c42\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_local_read_bytes_time_total_window # TYPE jindofsx_ns_local_read_bytes_time_total_window gauge jindofsx_ns_local_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_bytes_total # TYPE jindofsx_ns_local_read_bytes_total gauge jindofsx_ns_local_read_bytes_total 0 #\u603b\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_local_read_bytes_total_window # TYPE jindofsx_ns_local_read_bytes_total_window gauge jindofsx_ns_local_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_time_total # TYPE jindofsx_ns_local_read_time_total gauge jindofsx_ns_local_read_time_total 0 #\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_readop_num_total # TYPE jindofsx_ns_local_readop_num_total gauge jindofsx_ns_local_readop_num_total 0 #\u77ed\u8def\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_remote_read_bytes_time_total_window # TYPE jindofsx_ns_remote_read_bytes_time_total_window gauge jindofsx_ns_remote_read_bytes_time_total_window 73714663 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_read_bytes_total # TYPE jindofsx_ns_remote_read_bytes_total gauge jindofsx_ns_remote_read_bytes_total 112108045498 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_bytes_total_window # TYPE jindofsx_ns_remote_read_bytes_total_window gauge jindofsx_ns_remote_read_bytes_total_window 13380884043 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_time_total # TYPE jindofsx_ns_remote_read_time_total gauge jindofsx_ns_remote_read_time_total 765272359 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u65f6\u95f4\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_readop_num_total # TYPE jindofsx_ns_remote_readop_num_total gauge jindofsx_ns_remote_readop_num_total 107314 #\u8fdc\u7aef\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_slicelet_read_bytes_time_total_window # TYPE jindofsx_ns_slicelet_read_bytes_time_total_window gauge jindofsx_ns_slicelet_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_read_bytes_total # TYPE jindofsx_ns_slicelet_read_bytes_total gauge jindofsx_ns_slicelet_read_bytes_total 0 #\u603b\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_bytes_total_window # TYPE jindofsx_ns_slicelet_read_bytes_total_window gauge jindofsx_ns_slicelet_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_time_total # TYPE jindofsx_ns_slicelet_read_time_total gauge jindofsx_ns_slicelet_read_time_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_readop_num_total # TYPE jindofsx_ns_slicelet_readop_num_total gauge jindofsx_ns_slicelet_readop_num_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_total_disk_cap # TYPE jindofsx_ns_total_disk_cap gauge jindofsx_ns_total_disk_cap 840739848192 #\u78c1\u76d8\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_mem_cap # TYPE jindofsx_ns_total_mem_cap gauge jindofsx_ns_total_mem_cap 0 #\u5185\u5b58\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_stsnodes_num # TYPE jindofsx_ns_total_stsnodes_num gauge jindofsx_ns_total_stsnodes_num 3 #alive worker \u8282\u70b9\u7684\u6570\u91cf # HELP jindofsx_ns_total_used_disk_cap # TYPE jindofsx_ns_total_used_disk_cap gauge jindofsx_ns_total_used_disk_cap 153423446016 #\u5df2\u7528\u78c1\u76d8\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_used_mem_cap # TYPE jindofsx_ns_total_used_mem_cap gauge jindofsx_ns_total_used_mem_cap 0 #\u5df2\u7528\u5185\u5b58\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte","title":"\u76d1\u63a7\u548c\u53ef\u89c2\u6d4b\u6027\u529f\u80fd"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_monitor_metric/#_1","text":"\u672c\u6587\u4ecb\u7ecd\u5728\u542f\u52a8 Fluid JindoRuntime \u7684\u524d\u63d0\u4e0b\uff0c\u901a\u8fc7 Prometheus + Grafana \u6536\u96c6\u7f13\u5b58\u7cfb\u7edf\u76f8\u5173\u7684 metrics \u6307\u6807\uff0c\u8fdb\u884c\u53ef\u89c6\u5316\u5c55\u793a\u7684\u4f7f\u7528\u65b9\u6cd5\u3002","title":"\u76d1\u63a7\u548c\u53ef\u89c2\u6d4b\u6027\u529f\u80fd"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_monitor_metric/#in-cluster-prometheus-grafana-fluid","text":"","title":"\u4f7f\u7528 In-Cluster \u6a21\u5f0f\u90e8\u7f72 prometheus \u548c grafana \u76d1\u63a7 Fluid \u5e94\u7528"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_monitor_metric/#1-prometheus","text":"\u5982\u679c\u96c6\u7fa4\u5185\u65e0 prometheus\uff0c\u8bf7\u70b9\u51fb\u4e0b\u8f7d prometheus-jindofs.yaml \u6587\u4ef6 $ kubectl apply -f prometheus-jindofs.yaml \u5982\u96c6\u7fa4\u5185\u6709 prometheus,\u53ef\u5c06\u4ee5\u4e0b\u914d\u7f6e\u5199\u5230 prometheus \u914d\u7f6e\u6587\u4ef6\u4e2d: scrape_configs: - job_name: 'jindo runtime' metrics_path: /jindometrics/prometheus kubernetes_sd_configs: - role: endpoints relabel_configs: - source_labels: [__meta_kubernetes_service_label_role] regex: jindofs-master action: keep - source_labels: [__meta_kubernetes_endpoint_port_name] regex: rpc action: keep - source_labels: [__meta_kubernetes_namespace] target_label: namespace replacement: $1 action: replace - source_labels: [__meta_kubernetes_service_label_release] target_label: fluid_runtime replacement: $1 action: replace - source_labels: [__meta_kubernetes_endpoint_address_target_name] target_label: pod replacement: $1 action: replace","title":"1. \u90e8\u7f72\u6216\u914d\u7f6e Prometheus"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_monitor_metric/#2-grafana","text":"\u5982\u679c\u96c6\u7fa4\u5185\u65e0 grafana\uff0c\u8bf7\u70b9\u51fb\u4e0b\u8f7d grafana.yaml \u6587\u4ef6 $ kubectl apply -f grafana.yaml","title":"2. \u90e8\u7f72 grafana"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_monitor_metric/#3-grafana","text":"\u767b\u5f55 grafana \u627e\u5230 grafana \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\u548c\u6539\u8282\u70b9\u5bf9\u5916\u53ef\u8bbf\u95ee grafana-node-ip \uff0c\u5982\u679c\u60f3\u8981\u5728\u516c\u7f51\u8bbf\u95ee\uff0c\u53ef\u4ee5\u7ed1\u5b9a\u516c\u7f51IP\uff0c $ kubectl get pod -n kube-system -o wide | grep grafana monitoring-grafana-7dfcf6d5cd-5k8qw 1/1 Running 0 108m 10.54.1.11 cn-shanghai.192.168.1.1 <none> <none> \u627e\u5230 grafana \u670d\u52a1\u5bf9\u5e94\u7684 NodePort $ kubectl describe svc monitoring-grafana -n kube-system Name: monitoring-grafana Namespace: kube-system Labels: <none> Annotations: <none> Selector: app=grafana Type: NodePort IP Families: <none> IP: 172.16.83.188 IPs: 172.16.83.188 Port: <unset> 80/TCP TargetPort: 3000/TCP NodePort: <unset> 30396/TCP Endpoints: 10.54.1.75:3000 Session Affinity: None External Traffic Policy: Cluster Events: <none> \u53ef\u4ee5\u770b\u5230\u5f53\u524d\u66b4\u9732\u7684 NodePort \u4e3a 30396 \u8bbf\u95ee http://$grafana-node-ip:NodePort \uff0c\u9ed8\u8ba4\u8d26\u53f7\u5bc6\u7801 admin:admin \u67e5\u770b prometheus svc \u7aef\u53e3 \u540c\u6837\u5148\u627e\u5230 prometheus \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\uff0c\u627e\u5230\u8be5\u8282\u70b9\u7684 IP $ kubectl get pod -n kube-system -o wide | grep grafana prometheus-deployment-d6d8554b4-wfl98 1/1 Running 0 17d 10.54.1.204 cn-shanghai.192.168.1.1 <none> <none> kubectl get no cn-shanghai.192.168.31.107 -o wide NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP cn-shanghai.192.168.1.1 Ready <none> 5h51m v1.20.4-aliyun.1 192.168.1.1 101.132.66.11 \u901a\u8fc7\u67e5\u770b\u8282\u70b9\u7684 INTERNAL-IP \uff0c\u53ef\u4e86\u89e3\u8be5\u8282\u70b9\u7684\u5185\u90e8\u8bbf\u95ee IP \u4e3a 192.168.1.1 $ kubectl get svc -n kube-system | grep prometheus-svc prometheus-svc NodePort 10.100.0.144 <none> 9090:31225/TCP 22h $ kubectl describe svc prometheus-svc -n kube-system Name: prometheus-svc Namespace: kube-system Labels: kubernetes.io/name=Prometheus name=prometheus-svc Annotations: kubectl.kubernetes.io/last-applied-configuration: {\"apiVersion\":\"v1\",\"kind\":\"Service\",\"metadata\":{\"annotations\":{},\"labels\":{\"kubernetes.io/name\":\"Prometheus\",\"name\":\"prometheus-svc\"},\"nam... Selector: app=prometheus Type: NodePort IP: 10.100.0.144 Port: prometheus 9090/TCP TargetPort: 9090/TCP NodePort: prometheus 31225/TCP Endpoints: 10.99.224.138:9090 Session Affinity: None External Traffic Policy: Cluster Events: <none> \u53ef\u4ee5\u770b\u5230\u5f53\u524d prometheus svc \u66b4\u9732\u7684\u7aef\u53e3\u662f 31225 \uff0c\u540e\u7eed\u4f7f\u7528 http://IP:NodePort \u7684\u5f62\u5f0f\u6765\u53d1\u73b0 prometheus \u670d\u52a1 \u914d\u7f6e prometheus data source \u4f7f\u7528 http://IP:NodePort \u7684\u5f62\u5f0f\u6765\u53d1\u73b0 prometheus \u670d\u52a1 \u5bfc\u5165\u5b8c\u6210\u540e\u70b9\u51fbSave & Test \u663e\u793a Data source is working \u5373\u53ef \u5bfc\u5165\u6a21\u677f\u6587\u4ef6 grafana \u9009\u62e9\u5bfc\u5165\u6a21\u677f Json \u6587\u4ef6\uff0c\u70b9\u51fb\u6b64\u5904\u4e0b\u8f7d fluid-prometheus-grafana-monitor-jindofs.json \u67e5\u770b\u76d1\u63a7 \u5728 DashBoards \u4e2d\u627e\u5230\u5bfc\u5165\u7684\u6a21\u7248\u89c6\u56fe \u6ce8\uff1afluid_runtime \u5bf9\u5e94Fluid runtime name; namespace \u5bf9\u5e94 Fluid runtime namespace","title":"3. \u914d\u7f6e grafana"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_monitor_metric/#jindoruntime-prometheus","text":"\u4e0b\u9762\u5bf9 JindoRuntime \u5f53\u524d\u6536\u96c6\u7684\u6307\u6807\u8fdb\u884c\u9884\u89c8\u548c\u63cf\u8ff0\uff0c\u60a8\u53ef\u4ee5\u57fa\u4e8e\u5df2\u6536\u96c6\u6307\u6807\u5728 Grafana \u4e2d\u8fdb\u884c\u7ec4\u5408\u548c\u56fe\u8868\u5c55\u793a\uff0c\u5982\u60a8\u9700\u8981\u989d\u5916\u7684\u57fa\u672c metrics \u5c55\u793a\uff0c\u8bf7\u5f00 issue \u8054\u7cfb\u6211\u4eec # HELP jindofsx_ns_backend_read_bytes_time_total_window # TYPE jindofsx_ns_backend_read_bytes_time_total_window gauge jindofsx_ns_backend_read_bytes_time_total_window 20367553 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_read_bytes_total # TYPE jindofsx_ns_backend_read_bytes_total gauge jindofsx_ns_backend_read_bytes_total 166703347021 #\u603b\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_bytes_total_window # TYPE jindofsx_ns_backend_read_bytes_total_window gauge jindofsx_ns_backend_read_bytes_total_window 1174671374 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_time_total # TYPE jindofsx_ns_backend_read_time_total gauge jindofsx_ns_backend_read_time_total 8717632094 #\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_readop_num_total # TYPE jindofsx_ns_backend_readop_num_total gauge jindofsx_ns_backend_readop_num_total 181951 #\u5e95\u5c42\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_local_read_bytes_time_total_window # TYPE jindofsx_ns_local_read_bytes_time_total_window gauge jindofsx_ns_local_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_bytes_total # TYPE jindofsx_ns_local_read_bytes_total gauge jindofsx_ns_local_read_bytes_total 0 #\u603b\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_local_read_bytes_total_window # TYPE jindofsx_ns_local_read_bytes_total_window gauge jindofsx_ns_local_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_time_total # TYPE jindofsx_ns_local_read_time_total gauge jindofsx_ns_local_read_time_total 0 #\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_readop_num_total # TYPE jindofsx_ns_local_readop_num_total gauge jindofsx_ns_local_readop_num_total 0 #\u77ed\u8def\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_remote_read_bytes_time_total_window # TYPE jindofsx_ns_remote_read_bytes_time_total_window gauge jindofsx_ns_remote_read_bytes_time_total_window 73714663 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_read_bytes_total # TYPE jindofsx_ns_remote_read_bytes_total gauge jindofsx_ns_remote_read_bytes_total 112108045498 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_bytes_total_window # TYPE jindofsx_ns_remote_read_bytes_total_window gauge jindofsx_ns_remote_read_bytes_total_window 13380884043 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_time_total # TYPE jindofsx_ns_remote_read_time_total gauge jindofsx_ns_remote_read_time_total 765272359 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u65f6\u95f4\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_readop_num_total # TYPE jindofsx_ns_remote_readop_num_total gauge jindofsx_ns_remote_readop_num_total 107314 #\u8fdc\u7aef\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_slicelet_read_bytes_time_total_window # TYPE jindofsx_ns_slicelet_read_bytes_time_total_window gauge jindofsx_ns_slicelet_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_read_bytes_total # TYPE jindofsx_ns_slicelet_read_bytes_total gauge jindofsx_ns_slicelet_read_bytes_total 0 #\u603b\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_bytes_total_window # TYPE jindofsx_ns_slicelet_read_bytes_total_window gauge jindofsx_ns_slicelet_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_time_total # TYPE jindofsx_ns_slicelet_read_time_total gauge jindofsx_ns_slicelet_read_time_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_readop_num_total # TYPE jindofsx_ns_slicelet_readop_num_total gauge jindofsx_ns_slicelet_readop_num_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_total_disk_cap # TYPE jindofsx_ns_total_disk_cap gauge jindofsx_ns_total_disk_cap 840739848192 #\u78c1\u76d8\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_mem_cap # TYPE jindofsx_ns_total_mem_cap gauge jindofsx_ns_total_mem_cap 0 #\u5185\u5b58\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_stsnodes_num # TYPE jindofsx_ns_total_stsnodes_num gauge jindofsx_ns_total_stsnodes_num 3 #alive worker \u8282\u70b9\u7684\u6570\u91cf # HELP jindofsx_ns_total_used_disk_cap # TYPE jindofsx_ns_total_used_disk_cap gauge jindofsx_ns_total_used_disk_cap 153423446016 #\u5df2\u7528\u78c1\u76d8\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_used_mem_cap # TYPE jindofsx_ns_total_used_mem_cap gauge jindofsx_ns_total_used_mem_cap 0 #\u5df2\u7528\u5185\u5b58\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte","title":"JindoRuntime Prometheus \u6307\u6807\u9884\u89c8(\u66f4\u65b0\u4e2d)"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_nas_ufs_example/","text":"\u52a0\u901fOSS\u4e0a\u6570\u636e \u00a7 \u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002 1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001 \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907 \u00a7 5.1 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: nas spec: mounts: - mountPoint: local:///mnt/nas name: data path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: nas spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d NAS \u7684\u8def\u5f84\uff0c local:// \u4e3a\u672c\u5730\u8def\u5f84\u524d\u7f00\uff0c /mnt/nas \u8868\u793a\u672c\u5730\u771f\u5b9e\u8def\u5f84\uff0c\u8be5\u8def\u5f84\u9700\u8981\u5728\u5404\u4e2a\u8282\u70b9\u4e0a\u90fd\u5b58\u5728\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JinRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime nas NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE nas Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset nas NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE nas 511MiB 0.00B 180.00GiB 0.0% Bound 1h \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/nas 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/nas Bound hadoop 100Gi RWX 58m 6\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime nas \u5220\u9664 dataset kubectl delete dataset nas","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_nas_ufs_example/#oss","text":"\u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_nas_ufs_example/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_nas_ufs_example/#2-fluid","text":"","title":"2\u3001\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_nas_ufs_example/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_nas_ufs_example/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_nas_ufs_example/#5","text":"","title":"5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_nas_ufs_example/#51-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: nas spec: mounts: - mountPoint: local:///mnt/nas name: data path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: nas spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d NAS \u7684\u8def\u5f84\uff0c local:// \u4e3a\u672c\u5730\u8def\u5f84\u524d\u7f00\uff0c /mnt/nas \u8868\u793a\u672c\u5730\u771f\u5b9e\u8def\u5f84\uff0c\u8be5\u8def\u5f84\u9700\u8981\u5728\u5404\u4e2a\u8282\u70b9\u4e0a\u90fd\u5b58\u5728\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JinRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime nas NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE nas Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset nas NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE nas 511MiB 0.00B 180.00GiB 0.0% Bound 1h \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/nas 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/nas Bound hadoop 100Gi RWX 58m","title":"5.1 \u521b\u5efa dataset \u548c JindoRuntime"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_nas_ufs_example/#6","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime nas \u5220\u9664 dataset kubectl delete dataset nas","title":"6\u3001\u73af\u5883\u6e05\u7406"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/","text":"\u52a0\u901f OSS-HDFS \u4e0a\u6570\u636e \u00a7 \u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS-HDFS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002 1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001 \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907 \u00a7 5.1 \u51c6\u5907 OSS-HDFS \u670d\u52a1 \u00a7 \u60a8\u9700\u8981\u5f00\u901a\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1(Object Storage Service\uff0c\u7b80\u79f0OSS)\uff0c\u6309\u7167\u4f7f\u7528\u6587\u6863\u8fdb\u884cbucket\u521b\u5efa\uff0c\u5e76\u5728\u521b\u5efa\u65f6\u6307\u5b9abucket\u4e3a\u652f\u6301 OSS-HDFS \u6a21\u5f0f\uff0c\u8bbf\u95ee\u94fe\u63a5 https://github.com../../tree/master/docsdocs/user/4.x/4.6.x/4.6.12/jindofs/jindo_dls_quickstart.md 5.2 \u51c6\u5907\u6d4b\u8bd5\u6837\u4f8b\u6570\u636e \u00a7 \u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 Apache \u955c\u50cf\u7ad9\u70b9\u4e0a\u7684 Spark \u76f8\u5173\u8d44\u6e90\u4f5c\u4e3a\u6f14\u793a\u4e2d\u4f7f\u7528\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002\u8fd9\u4e2a\u9009\u62e9\u5e76\u6ca1\u6709\u4efb\u4f55\u7279\u6b8a\u4e4b\u5904\uff0c\u4f60\u53ef\u4ee5\u5c06\u8fd9\u4e2a\u8fdc\u7a0b\u6587\u4ef6\u4fee\u6539\u4e3a\u4efb\u610f\u4f60\u559c\u6b22\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002 \u73b0\u5728\u8fdc\u7a0b\u8d44\u6e90\u6587\u4ef6\u5230\u672c\u5730 mkdir tmp cd tmp wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz \u4e0a\u4f20\u672c\u5730\u6587\u4ef6\u5230 OSS-HDFS \u4e0a \u60a8\u53ef\u4ee5\u4f7f\u7528 JindoSDK \uff0c\u6309\u7167\u4f7f\u7528\u8bf4\u660e\u5c06\u672c\u5730\u4e0b\u8f7d\u7684\u6587\u4ef6\u4e0a\u4f20\u5230\u8fdc\u7a0b OSS-HDFS \u7684bucket\u4e0a\u3002\u5982\u60a8\u7684bucket\u540d\u5b57\u4e3a test-bucket \u4e14\u4f7f\u7528 JindoSDK \u4f5c\u4e3a\u5ba2\u6237\u7aef\u5de5\u5177\uff0c\u53ef\u6267\u884c\u5982\u4e0b\u547d\u4ee4\u4e0a\u4f20\u6587\u4ef6\u5230 OSS-HDFS \u4e0a\u9762\u3002 hadoop fs -put spark-2.4.7-bin-hadoop2.7.tgz oss://test-bucket/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz hadoop fs -put spark-3.1.1-bin-hadoop3.2.tgz oss://test-bucket/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz 5.3 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/dir/ options: fs.oss.accessKeyId: <OSS_HDFS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_HDFS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_HDFS_ENDPOINT> name: hadoop path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aOSS-HDFS bucket\u7684endpoint\u4fe1\u606f\uff0c\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08cn-xxx.oss-dls.aliyuncs.com\uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08oss-cn-xxx.aliyuncs.com\uff09\u4e0d\u540c\uff0c\u5982\u60a8\u7684\u670d\u52a1\u5728 hangzhou\uff0c\u90a3\u4e48\u5bf9\u5e94\u7684 endpoint \u4e3a\uff1acn-hangzhou.oss-dls.aliyuncs.com replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JinRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hadoop Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE HCFS URL TOTAL FILES CACHE HIT RATIO AGE hadoop [Calculating] 0.00B 200.00GiB Bound hadoop-jindofs-master-0.default:18844 [Calculating] 6m28s \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m 6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: hadoop \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u67e5\u770b\u6587\u4ef6\u5927\u5c0f $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 \u8fdb\u884c\u6587\u4ef6\u7684cp\u89c2\u5bdf\u65f6\u95f4\u6d88\u8017\u4e8618s $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m18.386s user 0m0.002s sys 0m0.105s \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 oss \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: hadoop-dataload spec: loadMetadata: true dataset: name: hadoop namespace: default \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload spark-dataload NAME DATASET PHASE AGE hadoop-dataload hadoop Completed 2m13s \u67e5\u770b\u6b64\u65f6 dataset \u7684\u7f13\u5b58\u60c5\u51b5\uff0c\u53d1\u73b0210MB\u7684\u6570\u636e\u5df2\u7ecf\u90fd\u7f13\u5b58\u5230\u4e86\u672c\u5730\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hadoop 210.00MiB 210.00MiB 180.00GiB 100.0% Bound 1h \u4e3a\u4e86\u907f\u514d\u5176\u4ed6\u56e0\u7d20(\u6bd4\u5982page cache)\u5bf9\u7ed3\u679c\u9020\u6210\u5f71\u54cd\uff0c\u6211\u4eec\u5c06\u5220\u9664\u4e4b\u524d\u7684\u5bb9\u5668\uff0c\u65b0\u5efa\u76f8\u540c\u7684\u5e94\u7528\uff0c\u5c1d\u8bd5\u8bbf\u95ee\u540c\u6837\u7684\u6587\u4ef6\u3002\u7531\u4e8e\u6b64\u65f6\u6587\u4ef6\u5df2\u7ecf\u88ab JindoFSx \u7f13\u5b58\uff0c\u53ef\u4ee5\u770b\u5230\u7b2c\u4e8c\u6b21\u8bbf\u95ee\u6240\u9700\u65f6\u95f4\u8fdc\u5c0f\u4e8e\u7b2c\u4e00\u6b21\u3002 kubectl delete -f app.yaml && kubectl create -f app.yaml \u8fdb\u884c\u6587\u4ef6\u7684\u62f7\u8d1d\u89c2\u5bdf\u65f6\u95f4\uff0c\u53d1\u73b0\u6d88\u801748ms\uff0c\u6574\u4e2a\u62f7\u8d1d\u7684\u65f6\u95f4\u7f29\u77ed\u4e86300\u500d $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m0.048s user 0m0.001s sys 0m0.046s 7\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"\u52a0\u901f OSS-HDFS \u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#oss-hdfs","text":"\u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS-HDFS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002","title":"\u52a0\u901f OSS-HDFS \u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#2-fluid","text":"","title":"2\u3001\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#5","text":"","title":"5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#51-oss-hdfs","text":"\u60a8\u9700\u8981\u5f00\u901a\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1(Object Storage Service\uff0c\u7b80\u79f0OSS)\uff0c\u6309\u7167\u4f7f\u7528\u6587\u6863\u8fdb\u884cbucket\u521b\u5efa\uff0c\u5e76\u5728\u521b\u5efa\u65f6\u6307\u5b9abucket\u4e3a\u652f\u6301 OSS-HDFS \u6a21\u5f0f\uff0c\u8bbf\u95ee\u94fe\u63a5 https://github.com../../tree/master/docsdocs/user/4.x/4.6.x/4.6.12/jindofs/jindo_dls_quickstart.md","title":"5.1 \u51c6\u5907 OSS-HDFS \u670d\u52a1"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#52","text":"\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 Apache \u955c\u50cf\u7ad9\u70b9\u4e0a\u7684 Spark \u76f8\u5173\u8d44\u6e90\u4f5c\u4e3a\u6f14\u793a\u4e2d\u4f7f\u7528\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002\u8fd9\u4e2a\u9009\u62e9\u5e76\u6ca1\u6709\u4efb\u4f55\u7279\u6b8a\u4e4b\u5904\uff0c\u4f60\u53ef\u4ee5\u5c06\u8fd9\u4e2a\u8fdc\u7a0b\u6587\u4ef6\u4fee\u6539\u4e3a\u4efb\u610f\u4f60\u559c\u6b22\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002 \u73b0\u5728\u8fdc\u7a0b\u8d44\u6e90\u6587\u4ef6\u5230\u672c\u5730 mkdir tmp cd tmp wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz \u4e0a\u4f20\u672c\u5730\u6587\u4ef6\u5230 OSS-HDFS \u4e0a \u60a8\u53ef\u4ee5\u4f7f\u7528 JindoSDK \uff0c\u6309\u7167\u4f7f\u7528\u8bf4\u660e\u5c06\u672c\u5730\u4e0b\u8f7d\u7684\u6587\u4ef6\u4e0a\u4f20\u5230\u8fdc\u7a0b OSS-HDFS \u7684bucket\u4e0a\u3002\u5982\u60a8\u7684bucket\u540d\u5b57\u4e3a test-bucket \u4e14\u4f7f\u7528 JindoSDK \u4f5c\u4e3a\u5ba2\u6237\u7aef\u5de5\u5177\uff0c\u53ef\u6267\u884c\u5982\u4e0b\u547d\u4ee4\u4e0a\u4f20\u6587\u4ef6\u5230 OSS-HDFS \u4e0a\u9762\u3002 hadoop fs -put spark-2.4.7-bin-hadoop2.7.tgz oss://test-bucket/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz hadoop fs -put spark-3.1.1-bin-hadoop3.2.tgz oss://test-bucket/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz","title":"5.2 \u51c6\u5907\u6d4b\u8bd5\u6837\u4f8b\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#53-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/dir/ options: fs.oss.accessKeyId: <OSS_HDFS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_HDFS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_HDFS_ENDPOINT> name: hadoop path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aOSS-HDFS bucket\u7684endpoint\u4fe1\u606f\uff0c\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08cn-xxx.oss-dls.aliyuncs.com\uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08oss-cn-xxx.aliyuncs.com\uff09\u4e0d\u540c\uff0c\u5982\u60a8\u7684\u670d\u52a1\u5728 hangzhou\uff0c\u90a3\u4e48\u5bf9\u5e94\u7684 endpoint \u4e3a\uff1acn-hangzhou.oss-dls.aliyuncs.com replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JinRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hadoop Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE HCFS URL TOTAL FILES CACHE HIT RATIO AGE hadoop [Calculating] 0.00B 200.00GiB Bound hadoop-jindofs-master-0.default:18844 [Calculating] 6m28s \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m","title":"5.3 \u521b\u5efa dataset \u548c JindoRuntime"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#6","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: hadoop \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u67e5\u770b\u6587\u4ef6\u5927\u5c0f $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 \u8fdb\u884c\u6587\u4ef6\u7684cp\u89c2\u5bdf\u65f6\u95f4\u6d88\u8017\u4e8618s $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m18.386s user 0m0.002s sys 0m0.105s \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 oss \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: hadoop-dataload spec: loadMetadata: true dataset: name: hadoop namespace: default \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload spark-dataload NAME DATASET PHASE AGE hadoop-dataload hadoop Completed 2m13s \u67e5\u770b\u6b64\u65f6 dataset \u7684\u7f13\u5b58\u60c5\u51b5\uff0c\u53d1\u73b0210MB\u7684\u6570\u636e\u5df2\u7ecf\u90fd\u7f13\u5b58\u5230\u4e86\u672c\u5730\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hadoop 210.00MiB 210.00MiB 180.00GiB 100.0% Bound 1h \u4e3a\u4e86\u907f\u514d\u5176\u4ed6\u56e0\u7d20(\u6bd4\u5982page cache)\u5bf9\u7ed3\u679c\u9020\u6210\u5f71\u54cd\uff0c\u6211\u4eec\u5c06\u5220\u9664\u4e4b\u524d\u7684\u5bb9\u5668\uff0c\u65b0\u5efa\u76f8\u540c\u7684\u5e94\u7528\uff0c\u5c1d\u8bd5\u8bbf\u95ee\u540c\u6837\u7684\u6587\u4ef6\u3002\u7531\u4e8e\u6b64\u65f6\u6587\u4ef6\u5df2\u7ecf\u88ab JindoFSx \u7f13\u5b58\uff0c\u53ef\u4ee5\u770b\u5230\u7b2c\u4e8c\u6b21\u8bbf\u95ee\u6240\u9700\u65f6\u95f4\u8fdc\u5c0f\u4e8e\u7b2c\u4e00\u6b21\u3002 kubectl delete -f app.yaml && kubectl create -f app.yaml \u8fdb\u884c\u6587\u4ef6\u7684\u62f7\u8d1d\u89c2\u5bdf\u65f6\u95f4\uff0c\u53d1\u73b0\u6d88\u801748ms\uff0c\u6574\u4e2a\u62f7\u8d1d\u7684\u65f6\u95f4\u7f29\u77ed\u4e86300\u500d $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m0.048s user 0m0.001s sys 0m0.046s","title":"6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_hdfs_ufs_example/#7","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"7\u3001\u73af\u5883\u6e05\u7406"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/","text":"\u52a0\u901fOSS\u4e0a\u6570\u636e \u00a7 \u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002 1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001 \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907 \u00a7 5.1 \u51c6\u5907 OSS \u670d\u52a1 \u00a7 \u60a8\u9700\u8981\u5f00\u901a\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1(Object Storage Service\uff0c\u7b80\u79f0OSS)\uff0c\u6309\u7167\u4f7f\u7528\u6587\u6863\u8fdb\u884cbucket\u521b\u5efa( https://www.aliyun.com/product/oss ) 5.2 \u51c6\u5907\u6d4b\u8bd5\u6837\u4f8b\u6570\u636e \u00a7 \u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 Apache \u955c\u50cf\u7ad9\u70b9\u4e0a\u7684 Spark \u76f8\u5173\u8d44\u6e90\u4f5c\u4e3a\u6f14\u793a\u4e2d\u4f7f\u7528\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002\u8fd9\u4e2a\u9009\u62e9\u5e76\u6ca1\u6709\u4efb\u4f55\u7279\u6b8a\u4e4b\u5904\uff0c\u4f60\u53ef\u4ee5\u5c06\u8fd9\u4e2a\u8fdc\u7a0b\u6587\u4ef6\u4fee\u6539\u4e3a\u4efb\u610f\u4f60\u559c\u6b22\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002 \u73b0\u5728\u8fdc\u7a0b\u8d44\u6e90\u6587\u4ef6\u5230\u672c\u5730 mkdir tmp cd tmp wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz \u4e0a\u4f20\u672c\u5730\u6587\u4ef6\u5230 OSS \u4e0a \u60a8\u53ef\u4ee5\u4f7f\u7528 OSS \u63d0\u4f9b\u7684\u5ba2\u6237\u7aef ossutil \u6216\u8005\u963f\u91cc\u4e91EMR\u56e2\u961f\u63d0\u4f9b\u7684 JindoSDK \uff0c\u6309\u7167\u4f7f\u7528\u8bf4\u660e\u5c06\u672c\u5730\u4e0b\u8f7d\u7684\u6587\u4ef6\u4e0a\u4f20\u5230\u8fdc\u7a0b OSS \u7684bucket\u4e0a\u3002\u5982\u60a8\u7684bucket\u540d\u5b57\u4e3a test-bucket \u4e14\u4f7f\u7528 JindoSDK \u4f5c\u4e3a\u5ba2\u6237\u7aef\u5de5\u5177\uff0c\u53ef\u6267\u884c\u5982\u4e0b\u547d\u4ee4\u4e0a\u4f20\u6587\u4ef6\u5230 OSS \u4e0a\u9762\u3002 hadoop fs -put spark-2.4.7-bin-hadoop2.7.tgz oss://test-bucket/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz hadoop fs -put spark-3.1.1-bin-hadoop3.2.tgz oss://test-bucket/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz 5.3 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/dir/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aoss bucket\u7684endpoint\u4fe1\u606f\uff0c\u516c\u7f51\u6216\u5185\u7f51\u5730\u5740\u7686\u53ef\uff0c\u5982\u60a8\u7684 bucket \u5728\u676d\u5dde Region\uff0c\u90a3\u4e48\u516c\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou.aliyuncs.com\uff0c\u5185\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou-internal.aliyuncs.com(\u5185\u7f51\u5730\u5740\u4f7f\u7528\u6761\u4ef6\u4e3a\u60a8\u7684 K8S \u96c6\u7fa4\u6240\u5728\u533a\u57df\u548coss\u533a\u57df\u76f8\u540c)\u3002 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JindoRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hadoop Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE HCFS URL TOTAL FILES CACHE HIT RATIO AGE hadoop [Calculating] 0.00B 200.00GiB Bound hadoop-jindofs-master-0.default:18844 [Calculating] 6m28s \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m 6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: hadoop \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u67e5\u770b\u6587\u4ef6\u5927\u5c0f $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 \u8fdb\u884c\u6587\u4ef6\u7684cp\u89c2\u5bdf\u65f6\u95f4\u6d88\u8017\u4e8618s $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m18.386s user 0m0.002s sys 0m0.105s \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 oss \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: hadoop-dataload spec: loadMetadata: true dataset: name: hadoop namespace: default \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload spark-dataload NAME DATASET PHASE AGE hadoop-dataload hadoop Completed 2m13s \u67e5\u770b\u6b64\u65f6 dataset \u7684\u7f13\u5b58\u60c5\u51b5\uff0c\u53d1\u73b0210MB\u7684\u6570\u636e\u5df2\u7ecf\u90fd\u7f13\u5b58\u5230\u4e86\u672c\u5730\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hadoop 210.00MiB 210.00MiB 180.00GiB 100.0% Bound 1h \u4e3a\u4e86\u907f\u514d\u5176\u4ed6\u56e0\u7d20(\u6bd4\u5982page cache)\u5bf9\u7ed3\u679c\u9020\u6210\u5f71\u54cd\uff0c\u6211\u4eec\u5c06\u5220\u9664\u4e4b\u524d\u7684\u5bb9\u5668\uff0c\u65b0\u5efa\u76f8\u540c\u7684\u5e94\u7528\uff0c\u5c1d\u8bd5\u8bbf\u95ee\u540c\u6837\u7684\u6587\u4ef6\u3002\u7531\u4e8e\u6b64\u65f6\u6587\u4ef6\u5df2\u7ecf\u88ab JindoFSx \u7f13\u5b58\uff0c\u53ef\u4ee5\u770b\u5230\u7b2c\u4e8c\u6b21\u8bbf\u95ee\u6240\u9700\u65f6\u95f4\u8fdc\u5c0f\u4e8e\u7b2c\u4e00\u6b21\u3002 kubectl delete -f app.yaml && kubectl create -f app.yaml \u8fdb\u884c\u6587\u4ef6\u7684\u62f7\u8d1d\u89c2\u5bdf\u65f6\u95f4\uff0c\u53d1\u73b0\u6d88\u801748ms\uff0c\u6574\u4e2a\u62f7\u8d1d\u7684\u65f6\u95f4\u7f29\u77ed\u4e86300\u500d $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m0.048s user 0m0.001s sys 0m0.046s 7\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#oss","text":"\u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#2-fluid","text":"","title":"2\u3001\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#5","text":"","title":"5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#51-oss","text":"\u60a8\u9700\u8981\u5f00\u901a\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1(Object Storage Service\uff0c\u7b80\u79f0OSS)\uff0c\u6309\u7167\u4f7f\u7528\u6587\u6863\u8fdb\u884cbucket\u521b\u5efa( https://www.aliyun.com/product/oss )","title":"5.1 \u51c6\u5907 OSS \u670d\u52a1"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#52","text":"\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 Apache \u955c\u50cf\u7ad9\u70b9\u4e0a\u7684 Spark \u76f8\u5173\u8d44\u6e90\u4f5c\u4e3a\u6f14\u793a\u4e2d\u4f7f\u7528\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002\u8fd9\u4e2a\u9009\u62e9\u5e76\u6ca1\u6709\u4efb\u4f55\u7279\u6b8a\u4e4b\u5904\uff0c\u4f60\u53ef\u4ee5\u5c06\u8fd9\u4e2a\u8fdc\u7a0b\u6587\u4ef6\u4fee\u6539\u4e3a\u4efb\u610f\u4f60\u559c\u6b22\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002 \u73b0\u5728\u8fdc\u7a0b\u8d44\u6e90\u6587\u4ef6\u5230\u672c\u5730 mkdir tmp cd tmp wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz \u4e0a\u4f20\u672c\u5730\u6587\u4ef6\u5230 OSS \u4e0a \u60a8\u53ef\u4ee5\u4f7f\u7528 OSS \u63d0\u4f9b\u7684\u5ba2\u6237\u7aef ossutil \u6216\u8005\u963f\u91cc\u4e91EMR\u56e2\u961f\u63d0\u4f9b\u7684 JindoSDK \uff0c\u6309\u7167\u4f7f\u7528\u8bf4\u660e\u5c06\u672c\u5730\u4e0b\u8f7d\u7684\u6587\u4ef6\u4e0a\u4f20\u5230\u8fdc\u7a0b OSS \u7684bucket\u4e0a\u3002\u5982\u60a8\u7684bucket\u540d\u5b57\u4e3a test-bucket \u4e14\u4f7f\u7528 JindoSDK \u4f5c\u4e3a\u5ba2\u6237\u7aef\u5de5\u5177\uff0c\u53ef\u6267\u884c\u5982\u4e0b\u547d\u4ee4\u4e0a\u4f20\u6587\u4ef6\u5230 OSS \u4e0a\u9762\u3002 hadoop fs -put spark-2.4.7-bin-hadoop2.7.tgz oss://test-bucket/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz hadoop fs -put spark-3.1.1-bin-hadoop3.2.tgz oss://test-bucket/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz","title":"5.2 \u51c6\u5907\u6d4b\u8bd5\u6837\u4f8b\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#53-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/dir/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aoss bucket\u7684endpoint\u4fe1\u606f\uff0c\u516c\u7f51\u6216\u5185\u7f51\u5730\u5740\u7686\u53ef\uff0c\u5982\u60a8\u7684 bucket \u5728\u676d\u5dde Region\uff0c\u90a3\u4e48\u516c\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou.aliyuncs.com\uff0c\u5185\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou-internal.aliyuncs.com(\u5185\u7f51\u5730\u5740\u4f7f\u7528\u6761\u4ef6\u4e3a\u60a8\u7684 K8S \u96c6\u7fa4\u6240\u5728\u533a\u57df\u548coss\u533a\u57df\u76f8\u540c)\u3002 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JindoRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hadoop Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE HCFS URL TOTAL FILES CACHE HIT RATIO AGE hadoop [Calculating] 0.00B 200.00GiB Bound hadoop-jindofs-master-0.default:18844 [Calculating] 6m28s \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m","title":"5.3 \u521b\u5efa dataset \u548c JindoRuntime"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#6","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: hadoop \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u67e5\u770b\u6587\u4ef6\u5927\u5c0f $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 \u8fdb\u884c\u6587\u4ef6\u7684cp\u89c2\u5bdf\u65f6\u95f4\u6d88\u8017\u4e8618s $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m18.386s user 0m0.002s sys 0m0.105s \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 oss \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: hadoop-dataload spec: loadMetadata: true dataset: name: hadoop namespace: default \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload spark-dataload NAME DATASET PHASE AGE hadoop-dataload hadoop Completed 2m13s \u67e5\u770b\u6b64\u65f6 dataset \u7684\u7f13\u5b58\u60c5\u51b5\uff0c\u53d1\u73b0210MB\u7684\u6570\u636e\u5df2\u7ecf\u90fd\u7f13\u5b58\u5230\u4e86\u672c\u5730\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hadoop 210.00MiB 210.00MiB 180.00GiB 100.0% Bound 1h \u4e3a\u4e86\u907f\u514d\u5176\u4ed6\u56e0\u7d20(\u6bd4\u5982page cache)\u5bf9\u7ed3\u679c\u9020\u6210\u5f71\u54cd\uff0c\u6211\u4eec\u5c06\u5220\u9664\u4e4b\u524d\u7684\u5bb9\u5668\uff0c\u65b0\u5efa\u76f8\u540c\u7684\u5e94\u7528\uff0c\u5c1d\u8bd5\u8bbf\u95ee\u540c\u6837\u7684\u6587\u4ef6\u3002\u7531\u4e8e\u6b64\u65f6\u6587\u4ef6\u5df2\u7ecf\u88ab JindoFSx \u7f13\u5b58\uff0c\u53ef\u4ee5\u770b\u5230\u7b2c\u4e8c\u6b21\u8bbf\u95ee\u6240\u9700\u65f6\u95f4\u8fdc\u5c0f\u4e8e\u7b2c\u4e00\u6b21\u3002 kubectl delete -f app.yaml && kubectl create -f app.yaml \u8fdb\u884c\u6587\u4ef6\u7684\u62f7\u8d1d\u89c2\u5bdf\u65f6\u95f4\uff0c\u53d1\u73b0\u6d88\u801748ms\uff0c\u6574\u4e2a\u62f7\u8d1d\u7684\u65f6\u95f4\u7f29\u77ed\u4e86300\u500d $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m0.048s user 0m0.001s sys 0m0.046s","title":"6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_oss_ufs_example/#7","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"7\u3001\u73af\u5883\u6e05\u7406"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_overview/","text":"Fluid \u7ed3\u5408 JindoCache \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4ecb\u7ecd\u548c\u4f7f\u7528 \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 Kubernetes version > 1.14, \u652f\u6301CSI Golang 1.12+ Helm 3 \u5b89\u88c5\u5305\u4e0b\u8f7d \u00a7 \u4e0b\u8f7d\u9875\u9762 \u4f7f\u7528\u6587\u6863 \u00a7 \u52a0\u901fOSS\u4e0a\u6570\u636e \u52a0\u901fHDFS\u4e0a\u6570\u636e \u52a0\u901fOSS-HDFS\u4e0a\u6570\u636e \u52a0\u901fS3\u4e0a\u6570\u636e \u52a0\u901fNAS\u4e0a\u6570\u636e \u52a0\u901fPVC\u4e0a\u6570\u636e \u76f4\u63a5\u6302\u8f7d OSS \u4e0a\u6570\u636e Fluid + JindoCache \u9ad8\u9636\u529f\u80fd\u4f7f\u7528 \u6570\u636e\u9884\u52a0\u8f7d \u76d1\u63a7\u53ca\u53ef\u89c2\u6d4b\u529f\u80fd \u95ee\u9898\u8bca\u65ad\u53ca\u5904\u7406 \u5bf9\u63a5\u963f\u91cc\u4e91SLS\u65e5\u5fd7\u670d\u52a1 \u6700\u4f73\u5b9e\u8df5\u7cfb\u5217 \u00a7 \u4e91\u539f\u751f AI \u5728\u81ea\u52a8\u9a7e\u9a76\u9886\u57df\u7684\u5e94\u7528\u63a2\u7d22\u4e0e\u6beb\u672b\u667a\u884c\u843d\u5730\u5b9e\u8df5 \u5fae\u535a\u6d77\u91cf\u6df1\u5ea6\u5b66\u4e60\u6a21\u578b\u8bad\u7ec3\u6548\u7387\u8dc3\u5347\u7684\u79d8\u5bc6 \u4f5c\u4e1a\u5e2e\u68c0\u7d22\u670d\u52a1\u57fa\u4e8e Fluid \u7684\u8ba1\u7b97\u5b58\u50a8\u5206\u79bb\u5b9e\u8df5","title":"Fluid \u7ed3\u5408 JindoCache \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4ecb\u7ecd\u548c\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_overview/#fluid-jindocache","text":"","title":"Fluid \u7ed3\u5408 JindoCache \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4ecb\u7ecd\u548c\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_overview/#_1","text":"Kubernetes version > 1.14, \u652f\u6301CSI Golang 1.12+ Helm 3","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_overview/#_2","text":"\u4e0b\u8f7d\u9875\u9762","title":"\u5b89\u88c5\u5305\u4e0b\u8f7d"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_overview/#_3","text":"\u52a0\u901fOSS\u4e0a\u6570\u636e \u52a0\u901fHDFS\u4e0a\u6570\u636e \u52a0\u901fOSS-HDFS\u4e0a\u6570\u636e \u52a0\u901fS3\u4e0a\u6570\u636e \u52a0\u901fNAS\u4e0a\u6570\u636e \u52a0\u901fPVC\u4e0a\u6570\u636e \u76f4\u63a5\u6302\u8f7d OSS \u4e0a\u6570\u636e Fluid + JindoCache \u9ad8\u9636\u529f\u80fd\u4f7f\u7528 \u6570\u636e\u9884\u52a0\u8f7d \u76d1\u63a7\u53ca\u53ef\u89c2\u6d4b\u529f\u80fd \u95ee\u9898\u8bca\u65ad\u53ca\u5904\u7406 \u5bf9\u63a5\u963f\u91cc\u4e91SLS\u65e5\u5fd7\u670d\u52a1","title":"\u4f7f\u7528\u6587\u6863"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_overview/#_4","text":"\u4e91\u539f\u751f AI \u5728\u81ea\u52a8\u9a7e\u9a76\u9886\u57df\u7684\u5e94\u7528\u63a2\u7d22\u4e0e\u6beb\u672b\u667a\u884c\u843d\u5730\u5b9e\u8df5 \u5fae\u535a\u6d77\u91cf\u6df1\u5ea6\u5b66\u4e60\u6a21\u578b\u8bad\u7ec3\u6548\u7387\u8dc3\u5347\u7684\u79d8\u5bc6 \u4f5c\u4e1a\u5e2e\u68c0\u7d22\u670d\u52a1\u57fa\u4e8e Fluid \u7684\u8ba1\u7b97\u5b58\u50a8\u5206\u79bb\u5b9e\u8df5","title":"\u6700\u4f73\u5b9e\u8df5\u7cfb\u5217"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/","text":"\u52a0\u901fPVC\u4e0a\u6570\u636e \u00a7 \u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c PVC \u6570\u636e\u5377\u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002 1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001 \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907 \u00a7 5.1 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u4e3a\u5df2\u5b58\u5728\u7684PVC\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 testPVC\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: pvc spec: mounts: - mountPoint: pvc://testPVC name: data path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: pvc spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d pvc \u7684\u8def\u5f84\uff0c pvc:// \u4e3a\u6302\u8f7d\u672c\u5730volume\u524d\u7f00\uff0c testPVC \u8868\u793a\u9700\u8981\u52a0\u901f\u7684PVC\u7684\u540d\u5b57\uff0c\u9700\u8981\u6709\u8bbf\u95ee\u6743\u9650\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JinRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime pvc NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE pvc Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset pvc NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE pvc 511MiB 0.00B 180.00GiB 0.0% Bound 1h \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/pvc 100Gi ROX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/pvc Bound hadoop 100Gi RWX 58m 6\u3001\u6570\u636e\u9884\u70ed \u00a7 \u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf \u00a7 \u53ef\u4ee5\u8fdb\u884c\u6570\u636e\u9884\u70ed\u63d0\u524d\u5c06\u6587\u4ef6\u7f13\u5b58\u5230\u672c\u5730\uff0c\u5047\u8bbe\u9700\u8981\u52a0\u901f\u8be5 pvc \u4e0b\u9762\u7684\u4e24\u4e2a\u5b50\u76ee\u5f55 /data1 \u548c /data1 \u540c\u65f6\u5206\u522b\u6307\u5b9a\u7f13\u5b58\u526f\u672c\u6570\u91cf\uff0c\u53ef\u4ee5\u53c2\u8003\u5982\u4e0bdataload.yaml\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: test spec: dataset: name: pvc namespace: default loadMetadata: true target: - path: /data1/ replicas: 1 - path: /data2/ replicas: 2 kubectl create dataload.yaml \u89c2\u5bdf dataload \u72b6\u6001 kubectl get dataload test \u5177\u4f53 dataload \u7684\u529f\u80fd\u53ef\u53c2\u8003 \u6570\u636e\u9884\u52a0\u8f7d 7\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime \u548c dataset kubectl delete dataset pvc","title":"\u52a0\u901fPVC\u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/#pvc","text":"\u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c PVC \u6570\u636e\u5377\u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002","title":"\u52a0\u901fPVC\u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/#2-fluid","text":"","title":"2\u3001\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/#5","text":"","title":"5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/#51-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u4e3a\u5df2\u5b58\u5728\u7684PVC\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 testPVC\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: pvc spec: mounts: - mountPoint: pvc://testPVC name: data path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: pvc spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d pvc \u7684\u8def\u5f84\uff0c pvc:// \u4e3a\u6302\u8f7d\u672c\u5730volume\u524d\u7f00\uff0c testPVC \u8868\u793a\u9700\u8981\u52a0\u901f\u7684PVC\u7684\u540d\u5b57\uff0c\u9700\u8981\u6709\u8bbf\u95ee\u6743\u9650\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JinRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime pvc NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE pvc Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset pvc NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE pvc 511MiB 0.00B 180.00GiB 0.0% Bound 1h \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/pvc 100Gi ROX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/pvc Bound hadoop 100Gi RWX 58m","title":"5.1 \u521b\u5efa dataset \u548c JindoRuntime"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/#6","text":"","title":"6\u3001\u6570\u636e\u9884\u70ed"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/#_1","text":"\u53ef\u4ee5\u8fdb\u884c\u6570\u636e\u9884\u70ed\u63d0\u524d\u5c06\u6587\u4ef6\u7f13\u5b58\u5230\u672c\u5730\uff0c\u5047\u8bbe\u9700\u8981\u52a0\u901f\u8be5 pvc \u4e0b\u9762\u7684\u4e24\u4e2a\u5b50\u76ee\u5f55 /data1 \u548c /data1 \u540c\u65f6\u5206\u522b\u6307\u5b9a\u7f13\u5b58\u526f\u672c\u6570\u91cf\uff0c\u53ef\u4ee5\u53c2\u8003\u5982\u4e0bdataload.yaml\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: test spec: dataset: name: pvc namespace: default loadMetadata: true target: - path: /data1/ replicas: 1 - path: /data2/ replicas: 2 kubectl create dataload.yaml \u89c2\u5bdf dataload \u72b6\u6001 kubectl get dataload test \u5177\u4f53 dataload \u7684\u529f\u80fd\u53ef\u53c2\u8003 \u6570\u636e\u9884\u52a0\u8f7d","title":"\u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_pvc_ufs_example/#7","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime \u548c dataset kubectl delete dataset pvc","title":"7\u3001\u73af\u5883\u6e05\u7406"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_question/","text":"\u95ee\u9898\u8bca\u65ad\u53ca\u5904\u7406 \u00a7 \u5982\u60a8\u7684\u73af\u5883\u51fa\u73b0\u95ee\u9898\uff0c\u4e5f\u53ef\u5f00 ISSUE \u7ed9\u6211\u4eec\uff0c\u4f1a\u7b2c\u4e00\u65f6\u95f4\u5904\u7406 Fluid \u5b89\u88c5\u4f7f\u7528\u76f8\u5173\u95ee\u9898 \u00a7 1. \u4e3a\u4ec0\u4e48\u6211\u4f7f\u7528Helm\u5b89\u88c5fluid\u5931\u8d25\u4e86\uff1f \u00a7 \u56de\u7b54 :\u63a8\u8350\u6309\u7167\u5b89\u88c5\u6587\u6863\u4f9d\u6b21\u786e\u8ba4Fluid\u7ec4\u4ef6\u662f\u5426\u6b63\u5e38\u8fd0\u884c\u3002 Fluid\u5b89\u88c5\u6587\u6863\u662f\u4ee5 Helm 3 \u4e3a\u4f8b\u8fdb\u884c\u90e8\u7f72\u7684\u3002\u5982\u679c\u60a8\u4f7f\u7528 Helm 3 \u4ee5\u4e0b\u7684\u7248\u672c\u90e8\u7f72Fluid\uff0c \u5e76\u4e14\u9047\u5230\u4e86 CRD\u6ca1\u6709\u6b63\u5e38\u542f\u52a8 \u7684\u60c5\u51b5\uff0c\u8fd9\u53ef\u80fd\u662f\u56e0\u4e3a Helm 3 \u53ca\u5176\u4ee5\u4e0a\u7248\u672c\u4f1a\u5728 helm install \u7684\u65f6\u5019\u81ea\u52a8\u5b89\u88c5CRD\uff0c \u800c\u4f4e\u7248\u672c\u7684Helm\u5219\u4e0d\u4f1a\u3002 \u53c2\u89c1 Helm\u5b98\u65b9\u6587\u6863\u8bf4\u660e \u3002 \u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u60a8\u9700\u8981\u624b\u52a8\u5b89\u88c5CRD\uff1a $ kubectl create -f fluid/crds 2. \u4e3a\u4ec0\u4e48\u6211\u65e0\u6cd5\u5220\u9664Runtime\uff1f \u00a7 \u56de\u7b54 :\u8bf7\u68c0\u67e5\u76f8\u5173Pod\u8fd0\u884c\u72b6\u6001\u548cRuntime\u7684Events\u3002 \u53ea\u8981\u6709\u4efb\u4f55\u6d3b\u8dc3Pod\u8fd8\u5728\u4f7f\u7528Fluid\u521b\u5efa\u7684Volume\uff0cFluid\u5c31\u4e0d\u4f1a\u5b8c\u6210\u5220\u9664\u64cd\u4f5c\u3002 \u5982\u4e0b\u7684\u547d\u4ee4\u53ef\u4ee5\u5feb\u901f\u5730\u627e\u51fa\u8fd9\u4e9b\u6d3b\u8dc3Pod\uff0c\u4f7f\u7528\u65f6\u628a <dataset_name> \u548c <dataset_namespace> \u6362\u6210\u81ea\u5df1\u7684\u5373\u53ef\uff1a kubectl describe pvc <dataset_name> -n <dataset_namespace> | \\ awk '/^Mounted/ {flag=1}; /^Events/ {flag=0}; flag' | \\ awk 'NR==1 {print $3}; NR!=1 {print $1}' | \\ xargs -I {} kubectl get po {} | \\ grep -E \"Running|Terminating|Pending\" | \\ cut -d \" \" -f 1 3. \u4e3a\u4ec0\u4e48\u6211\u5728\u521b\u5efa\u4efb\u52a1\u6302\u8f7d Runtime \u521b\u5efa\u7684 PVC \u7684\u65f6\u5019\u51fa\u73b0 driver name fuse.csi.fluid.io not found in the list of registered CSI drivers \u9519\u8bef\uff1f \u00a7 \u56de\u7b54 :\u8bf7\u67e5\u770b\u4efb\u52a1\u88ab\u8c03\u5ea6\u8282\u70b9\u6240\u5728\u7684 kubelet \u914d\u7f6e\u662f\u5426\u4e3a\u9ed8\u8ba4 /var/lib/kubelet \u3002 \u9996\u5148\u901a\u8fc7\u547d\u4ee4\u67e5\u770bFluid\u7684CSI\u7ec4\u4ef6\u662f\u5426\u6b63\u5e38 \u5982\u4e0b\u7684\u547d\u4ee4\u53ef\u4ee5\u5feb\u901f\u5730\u627e\u51faPod\uff0c\u4f7f\u7528\u65f6\u628a <node_name> \u548c <fluid_namespace> \u6362\u6210\u81ea\u5df1\u7684\u5373\u53ef\uff1a kubectl get pod -n <fluid_namespace> | grep <node_name> # <pod_name> \u4e3a\u4e0a\u4e00\u6b65pod\u540d kubectl logs <pod_name> node-driver-registrar -n <fluid_namespace> kubectl logs <pod_name> plugins -n <fluid_namespace> \u5982\u679c\u4e0a\u8ff0\u6b65\u9aa4\u7684Log\u65e0\u9519\u8bef\uff0c\u8bf7\u67e5\u770bcsidriver\u5bf9\u8c61\u662f\u5426\u5b58\u5728\uff1a kubectl get csidriver \u5982\u679ccsidriver\u5bf9\u8c61\u5b58\u5728\uff0c\u8bf7\u67e5\u770b\u67e5\u770bcsi\u6ce8\u518c\u8282\u70b9\u662f\u5426\u5305\u542b <node_name> \uff1a kubectl get csinode | grep <node_name> \u5982\u679c\u4e0a\u8ff0\u547d\u4ee4\u65e0\u8f93\u51fa\uff0c\u67e5\u770b\u4efb\u52a1\u88ab\u8c03\u5ea6\u8282\u70b9\u6240\u5728\u7684 kubelet \u914d\u7f6e\u662f\u5426\u4e3a\u9ed8\u8ba4 /var/lib/kubelet \u3002\u56e0\u4e3aFluid\u7684CSI\u7ec4\u4ef6\u901a\u8fc7\u56fa\u5b9a\u5730\u5740\u7684socket\u6ce8\u518c\u5230kubelet\uff0c\u9ed8\u8ba4\u4e3a --csi-address=/var/lib/kubelet/csi-plugins/fuse.csi.fluid.io/csi.sock --kubelet-registration-path=/var/lib/kubelet/csi-plugins/fuse.csi.fluid.io/csi.sock \u3002 4. \u4e3a\u4ec0\u4e48\u66f4\u65b0\u4e86fluid\u540e\uff0c\u4f7f\u7528 kubectl get \u67e5\u8be2\u66f4\u65b0\u524d\u521b\u5efa\u7684dataset\uff0c\u53d1\u73b0\u76f8\u6bd4\u65b0\u5efa\u7684dataset\u7f3a\u5c11\u4e86\u67d0\u4e9b\u5b57\u6bb5\uff1f \u00a7 \u56de\u7b54 :\u7531\u4e8e\u6211\u4eec\u5728fluid\u7684\u5347\u7ea7\u8fc7\u7a0b\u4e2d\u53ef\u80fd\u66f4\u65b0\u4e86CRD\uff0c\u4f60\u5728\u65e7\u7248\u672c\u521b\u5efa\u7684dataset\uff0c\u4f1a\u5c06CRD\u4e2d\u65b0\u589e\u7684\u5b57\u6bb5\u8bbe\u7f6e\u4e3a\u7a7a \u4f8b\u5982\uff0c\u5982\u679c\u4f60\u4ecev0.4\u6216\u66f4\u65e9\u7248\u672c\u5347\u7ea7\uff0c\u90a3\u65f6\u5019\u7684dataset\u6ca1\u6709FileNum\u5b57\u6bb5 \u66f4\u65b0fluid\u540e\uff0c\u5982\u679c\u4f60\u4f7f\u7528 kubectl get \u547d\u4ee4\uff0c\u65e0\u6cd5\u67e5\u8be2\u5230\u8be5dataset\u7684FileNum \u4f60\u53ef\u4ee5\u91cd\u5efadataset\uff0c\u65b0\u5efa\u7684dataset\u4f1a\u6b63\u5e38\u663e\u793a\u8fd9\u4e9b\u5b57\u6bb5 5. \u4e3a\u4ec0\u4e48\u5728\u5e94\u7528\u7a0b\u5e8f\u4e2d\u4f7f\u7528 PVC \u65f6\u4f1a\u4ea7\u751f\u4e86 Volume Attachment \u8d85\u65f6\u95ee\u9898\uff1f \u00a7 \u56de\u7b54 :Volume Attachment \u8d85\u65f6\u95ee\u9898\u662f Kubelet \u8fdb\u884c\u8bf7\u6c42 CSI Driver \u65f6\u672a\u6536\u5230 CSI Driver \u7684\u54cd\u5e94\u800c\u9020\u6210\u7684\u8d85\u65f6\u3002 \u8be5\u95ee\u9898\u662f\u7531\u4e8e Fluid \u7684 CSI Driver \u6ca1\u6709\u5b89\u88c5\uff0c\u6216\u8005kubelet\u6ca1\u6709\u8bbf\u95ee CSI Driver \u7684\u6743\u9650\u5bfc\u81f4\u7684\u3002 \u7531\u4e8e CSI Driver \u662f\u7531 Kubelet \u8fdb\u884c\u56de\u8c03\uff0c\u4f46\u662f\u5982\u679c Fluid \u6ca1\u6709\u5b89\u88c5 CSI Driver \u6216\u8005 Kubelet \u6ca1\u6709\u6743\u9650\u67e5\u770b CSI Driver\uff0c\u5c31\u4f1a\u5bfc\u81f4 CSI Plugin \u6ca1\u6709\u88ab\u6b63\u786e\u89e6\u53d1\u3002 \u9996\u5148\u9700\u8981\u4f7f\u7528\u547d\u4ee4 kubectl get csidriver \u67e5\u770b\u662f\u5426\u5b89\u88c5\u4e86 CSI Driver\u3002 \u5982\u679c\u6ca1\u6709\u5b89\u88c5\uff0c\u4f7f\u7528\u547d\u4ee4 kubectl apply -f charts/fluid/fluid/templates/csi/driver.yaml \u8fdb\u884c\u5b89\u88c5\uff0c\u7136\u540e\u89c2\u5bdf PVC \u662f\u5426\u6210\u529f\u6302\u8f7d\u5230\u5e94\u7528\u7a0b\u5e8f\u4e2d\u3002 \u5982\u679c\u4ecd\u672a\u80fd\u89e3\u51b3\uff0c\u4f7f\u7528 export KUBECONFIG=/etc/kubernetes/kubelet.conf && kubectl get csidriver \u6765\u67e5\u770b Kubelet \u80fd\u591f\u5177\u6709\u6743\u9650\u770b\u5230 CSI Driver","title":"\u95ee\u9898\u8bca\u65ad\u53ca\u5904\u7406"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_question/#_1","text":"\u5982\u60a8\u7684\u73af\u5883\u51fa\u73b0\u95ee\u9898\uff0c\u4e5f\u53ef\u5f00 ISSUE \u7ed9\u6211\u4eec\uff0c\u4f1a\u7b2c\u4e00\u65f6\u95f4\u5904\u7406","title":"\u95ee\u9898\u8bca\u65ad\u53ca\u5904\u7406"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_question/#fluid","text":"","title":"Fluid \u5b89\u88c5\u4f7f\u7528\u76f8\u5173\u95ee\u9898"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_question/#1-helmfluid","text":"\u56de\u7b54 :\u63a8\u8350\u6309\u7167\u5b89\u88c5\u6587\u6863\u4f9d\u6b21\u786e\u8ba4Fluid\u7ec4\u4ef6\u662f\u5426\u6b63\u5e38\u8fd0\u884c\u3002 Fluid\u5b89\u88c5\u6587\u6863\u662f\u4ee5 Helm 3 \u4e3a\u4f8b\u8fdb\u884c\u90e8\u7f72\u7684\u3002\u5982\u679c\u60a8\u4f7f\u7528 Helm 3 \u4ee5\u4e0b\u7684\u7248\u672c\u90e8\u7f72Fluid\uff0c \u5e76\u4e14\u9047\u5230\u4e86 CRD\u6ca1\u6709\u6b63\u5e38\u542f\u52a8 \u7684\u60c5\u51b5\uff0c\u8fd9\u53ef\u80fd\u662f\u56e0\u4e3a Helm 3 \u53ca\u5176\u4ee5\u4e0a\u7248\u672c\u4f1a\u5728 helm install \u7684\u65f6\u5019\u81ea\u52a8\u5b89\u88c5CRD\uff0c \u800c\u4f4e\u7248\u672c\u7684Helm\u5219\u4e0d\u4f1a\u3002 \u53c2\u89c1 Helm\u5b98\u65b9\u6587\u6863\u8bf4\u660e \u3002 \u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u60a8\u9700\u8981\u624b\u52a8\u5b89\u88c5CRD\uff1a $ kubectl create -f fluid/crds","title":"1. \u4e3a\u4ec0\u4e48\u6211\u4f7f\u7528Helm\u5b89\u88c5fluid\u5931\u8d25\u4e86\uff1f"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_question/#2-runtime","text":"\u56de\u7b54 :\u8bf7\u68c0\u67e5\u76f8\u5173Pod\u8fd0\u884c\u72b6\u6001\u548cRuntime\u7684Events\u3002 \u53ea\u8981\u6709\u4efb\u4f55\u6d3b\u8dc3Pod\u8fd8\u5728\u4f7f\u7528Fluid\u521b\u5efa\u7684Volume\uff0cFluid\u5c31\u4e0d\u4f1a\u5b8c\u6210\u5220\u9664\u64cd\u4f5c\u3002 \u5982\u4e0b\u7684\u547d\u4ee4\u53ef\u4ee5\u5feb\u901f\u5730\u627e\u51fa\u8fd9\u4e9b\u6d3b\u8dc3Pod\uff0c\u4f7f\u7528\u65f6\u628a <dataset_name> \u548c <dataset_namespace> \u6362\u6210\u81ea\u5df1\u7684\u5373\u53ef\uff1a kubectl describe pvc <dataset_name> -n <dataset_namespace> | \\ awk '/^Mounted/ {flag=1}; /^Events/ {flag=0}; flag' | \\ awk 'NR==1 {print $3}; NR!=1 {print $1}' | \\ xargs -I {} kubectl get po {} | \\ grep -E \"Running|Terminating|Pending\" | \\ cut -d \" \" -f 1","title":"2. \u4e3a\u4ec0\u4e48\u6211\u65e0\u6cd5\u5220\u9664Runtime\uff1f"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_question/#3-runtime-pvc-driver-name-fusecsifluidio-not-found-in-the-list-of-registered-csi-drivers","text":"\u56de\u7b54 :\u8bf7\u67e5\u770b\u4efb\u52a1\u88ab\u8c03\u5ea6\u8282\u70b9\u6240\u5728\u7684 kubelet \u914d\u7f6e\u662f\u5426\u4e3a\u9ed8\u8ba4 /var/lib/kubelet \u3002 \u9996\u5148\u901a\u8fc7\u547d\u4ee4\u67e5\u770bFluid\u7684CSI\u7ec4\u4ef6\u662f\u5426\u6b63\u5e38 \u5982\u4e0b\u7684\u547d\u4ee4\u53ef\u4ee5\u5feb\u901f\u5730\u627e\u51faPod\uff0c\u4f7f\u7528\u65f6\u628a <node_name> \u548c <fluid_namespace> \u6362\u6210\u81ea\u5df1\u7684\u5373\u53ef\uff1a kubectl get pod -n <fluid_namespace> | grep <node_name> # <pod_name> \u4e3a\u4e0a\u4e00\u6b65pod\u540d kubectl logs <pod_name> node-driver-registrar -n <fluid_namespace> kubectl logs <pod_name> plugins -n <fluid_namespace> \u5982\u679c\u4e0a\u8ff0\u6b65\u9aa4\u7684Log\u65e0\u9519\u8bef\uff0c\u8bf7\u67e5\u770bcsidriver\u5bf9\u8c61\u662f\u5426\u5b58\u5728\uff1a kubectl get csidriver \u5982\u679ccsidriver\u5bf9\u8c61\u5b58\u5728\uff0c\u8bf7\u67e5\u770b\u67e5\u770bcsi\u6ce8\u518c\u8282\u70b9\u662f\u5426\u5305\u542b <node_name> \uff1a kubectl get csinode | grep <node_name> \u5982\u679c\u4e0a\u8ff0\u547d\u4ee4\u65e0\u8f93\u51fa\uff0c\u67e5\u770b\u4efb\u52a1\u88ab\u8c03\u5ea6\u8282\u70b9\u6240\u5728\u7684 kubelet \u914d\u7f6e\u662f\u5426\u4e3a\u9ed8\u8ba4 /var/lib/kubelet \u3002\u56e0\u4e3aFluid\u7684CSI\u7ec4\u4ef6\u901a\u8fc7\u56fa\u5b9a\u5730\u5740\u7684socket\u6ce8\u518c\u5230kubelet\uff0c\u9ed8\u8ba4\u4e3a --csi-address=/var/lib/kubelet/csi-plugins/fuse.csi.fluid.io/csi.sock --kubelet-registration-path=/var/lib/kubelet/csi-plugins/fuse.csi.fluid.io/csi.sock \u3002","title":"3. \u4e3a\u4ec0\u4e48\u6211\u5728\u521b\u5efa\u4efb\u52a1\u6302\u8f7d Runtime \u521b\u5efa\u7684 PVC \u7684\u65f6\u5019\u51fa\u73b0 driver name fuse.csi.fluid.io not found in the list of registered CSI drivers \u9519\u8bef\uff1f"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_question/#4-fluid-kubectl-get-datasetdataset","text":"\u56de\u7b54 :\u7531\u4e8e\u6211\u4eec\u5728fluid\u7684\u5347\u7ea7\u8fc7\u7a0b\u4e2d\u53ef\u80fd\u66f4\u65b0\u4e86CRD\uff0c\u4f60\u5728\u65e7\u7248\u672c\u521b\u5efa\u7684dataset\uff0c\u4f1a\u5c06CRD\u4e2d\u65b0\u589e\u7684\u5b57\u6bb5\u8bbe\u7f6e\u4e3a\u7a7a \u4f8b\u5982\uff0c\u5982\u679c\u4f60\u4ecev0.4\u6216\u66f4\u65e9\u7248\u672c\u5347\u7ea7\uff0c\u90a3\u65f6\u5019\u7684dataset\u6ca1\u6709FileNum\u5b57\u6bb5 \u66f4\u65b0fluid\u540e\uff0c\u5982\u679c\u4f60\u4f7f\u7528 kubectl get \u547d\u4ee4\uff0c\u65e0\u6cd5\u67e5\u8be2\u5230\u8be5dataset\u7684FileNum \u4f60\u53ef\u4ee5\u91cd\u5efadataset\uff0c\u65b0\u5efa\u7684dataset\u4f1a\u6b63\u5e38\u663e\u793a\u8fd9\u4e9b\u5b57\u6bb5","title":"4. \u4e3a\u4ec0\u4e48\u66f4\u65b0\u4e86fluid\u540e\uff0c\u4f7f\u7528 kubectl get \u67e5\u8be2\u66f4\u65b0\u524d\u521b\u5efa\u7684dataset\uff0c\u53d1\u73b0\u76f8\u6bd4\u65b0\u5efa\u7684dataset\u7f3a\u5c11\u4e86\u67d0\u4e9b\u5b57\u6bb5\uff1f"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_question/#5-pvc-volume-attachment","text":"\u56de\u7b54 :Volume Attachment \u8d85\u65f6\u95ee\u9898\u662f Kubelet \u8fdb\u884c\u8bf7\u6c42 CSI Driver \u65f6\u672a\u6536\u5230 CSI Driver \u7684\u54cd\u5e94\u800c\u9020\u6210\u7684\u8d85\u65f6\u3002 \u8be5\u95ee\u9898\u662f\u7531\u4e8e Fluid \u7684 CSI Driver \u6ca1\u6709\u5b89\u88c5\uff0c\u6216\u8005kubelet\u6ca1\u6709\u8bbf\u95ee CSI Driver \u7684\u6743\u9650\u5bfc\u81f4\u7684\u3002 \u7531\u4e8e CSI Driver \u662f\u7531 Kubelet \u8fdb\u884c\u56de\u8c03\uff0c\u4f46\u662f\u5982\u679c Fluid \u6ca1\u6709\u5b89\u88c5 CSI Driver \u6216\u8005 Kubelet \u6ca1\u6709\u6743\u9650\u67e5\u770b CSI Driver\uff0c\u5c31\u4f1a\u5bfc\u81f4 CSI Plugin \u6ca1\u6709\u88ab\u6b63\u786e\u89e6\u53d1\u3002 \u9996\u5148\u9700\u8981\u4f7f\u7528\u547d\u4ee4 kubectl get csidriver \u67e5\u770b\u662f\u5426\u5b89\u88c5\u4e86 CSI Driver\u3002 \u5982\u679c\u6ca1\u6709\u5b89\u88c5\uff0c\u4f7f\u7528\u547d\u4ee4 kubectl apply -f charts/fluid/fluid/templates/csi/driver.yaml \u8fdb\u884c\u5b89\u88c5\uff0c\u7136\u540e\u89c2\u5bdf PVC \u662f\u5426\u6210\u529f\u6302\u8f7d\u5230\u5e94\u7528\u7a0b\u5e8f\u4e2d\u3002 \u5982\u679c\u4ecd\u672a\u80fd\u89e3\u51b3\uff0c\u4f7f\u7528 export KUBECONFIG=/etc/kubernetes/kubelet.conf && kubectl get csidriver \u6765\u67e5\u770b Kubelet \u80fd\u591f\u5177\u6709\u6743\u9650\u770b\u5230 CSI Driver","title":"5. \u4e3a\u4ec0\u4e48\u5728\u5e94\u7528\u7a0b\u5e8f\u4e2d\u4f7f\u7528 PVC \u65f6\u4f1a\u4ea7\u751f\u4e86 Volume Attachment \u8d85\u65f6\u95ee\u9898\uff1f"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_s3_ufs_example/","text":"\u62e5\u62b1\u4e91\u539f\u751f\uff0cFluid \u7ed3\u5408 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff1a\u52a0\u901f\u5bf9\u8c61\u5b58\u50a8\u4f7f\u7528 S3 \u534f\u8bae \u00a7 JindoFSx \u63d0\u4f9b\u4e86\u8bbf\u95ee S3 \u534f\u8bae\u7684\u80fd\u529b\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e S3 \u4f5c\u4e3a JindoFSx \u7684\u540e\u7aef\u5b58\u50a8\uff0c\u4f7f Fluid \u901a\u8fc7 JindoFSx \u6765\u8bbf\u95ee S3 \u4e0a\u7684\u6570\u636e\uff0c\u540c\u65f6 JindoFSx \u4e5f\u63d0\u4f9b\u4e86\u5bf9 S3 \u4e0a\u7684\u6570\u636e\u4ee5\u53ca\u5143\u6570\u636e\u7684\u7f13\u5b58\u52a0\u901f\u529f\u80fd\u3002 \u672c\u6587\u6863\u5c55\u793a\u4e86\u5982\u4f55\u5728 Fluid \u4e2d\u4ee5\u58f0\u660e\u5f0f\u7684\u65b9\u5f0f\u5b8c\u6210 JindoFSx \u90e8\u7f72\uff0c\u5bf9\u63a5 S3 \u6570\u636e\u6e90\uff08\u5305\u62ec\u63d0\u4f9b S3 \u517c\u5bb9\u7684\u6240\u6709\u5bf9\u8c61\u5b58\u50a8\uff09\u3002 \u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 \u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz \u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: s3 spec: mounts: - mountPoint: s3://<bucket>/<dir> options: fs.s3.accessKeyId: xxx fs.s3.accessKeySecret: xxx fs.s3.endpoint: s3.ap-east-1.amazonaws.com fs.s3.region: ap-east-1 name: s3 path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: s3 spec: replicas: 2 tieredstore: levels: - mediumtype: MEM path: /dev/shm quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d s3 \u7684\u8def\u5f84\uff0c\u652f\u6301\u6807\u51c6 s3 \u534f\u8bae\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u7684 worker \u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c\u5355\u4f4dGi\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b dataset \u7684\u60c5\u51b5 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE s3 210MiB 0.00B 180.00GiB 0.0% Bound 1h \u6267\u884c create \u547d\u4ee4 $ kubectl create -f runtime.yaml \u67e5\u770b dataset \u72b6\u6001 \u00a7 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE s3 180.01GiB 0.00B 261.00GiB 0.0% Bound 62m \u68c0\u67e5 PV\uff0cPVC \u521b\u5efa\u60c5\u51b5 \u00a7 $ kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/s3 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/s3 Bound hadoop 100Gi RWX 58m","title":"\u62e5\u62b1\u4e91\u539f\u751f\uff0cFluid \u7ed3\u5408 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff1a\u52a0\u901f\u5bf9\u8c61\u5b58\u50a8\u4f7f\u7528 S3 \u534f\u8bae"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_s3_ufs_example/#fluid-jindofsx-s3","text":"JindoFSx \u63d0\u4f9b\u4e86\u8bbf\u95ee S3 \u534f\u8bae\u7684\u80fd\u529b\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e S3 \u4f5c\u4e3a JindoFSx \u7684\u540e\u7aef\u5b58\u50a8\uff0c\u4f7f Fluid \u901a\u8fc7 JindoFSx \u6765\u8bbf\u95ee S3 \u4e0a\u7684\u6570\u636e\uff0c\u540c\u65f6 JindoFSx \u4e5f\u63d0\u4f9b\u4e86\u5bf9 S3 \u4e0a\u7684\u6570\u636e\u4ee5\u53ca\u5143\u6570\u636e\u7684\u7f13\u5b58\u52a0\u901f\u529f\u80fd\u3002 \u672c\u6587\u6863\u5c55\u793a\u4e86\u5982\u4f55\u5728 Fluid \u4e2d\u4ee5\u58f0\u660e\u5f0f\u7684\u65b9\u5f0f\u5b8c\u6210 JindoFSx \u90e8\u7f72\uff0c\u5bf9\u63a5 S3 \u6570\u636e\u6e90\uff08\u5305\u62ec\u63d0\u4f9b S3 \u517c\u5bb9\u7684\u6240\u6709\u5bf9\u8c61\u5b58\u50a8\uff09\u3002","title":"\u62e5\u62b1\u4e91\u539f\u751f\uff0cFluid \u7ed3\u5408 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff1a\u52a0\u901f\u5bf9\u8c61\u5b58\u50a8\u4f7f\u7528 S3 \u534f\u8bae"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_s3_ufs_example/#_1","text":"kubectl create ns fluid-system","title":"\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_s3_ufs_example/#fluid","text":"","title":"\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_s3_ufs_example/#helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_s3_ufs_example/#fluid_1","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_s3_ufs_example/#dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: s3 spec: mounts: - mountPoint: s3://<bucket>/<dir> options: fs.s3.accessKeyId: xxx fs.s3.accessKeySecret: xxx fs.s3.endpoint: s3.ap-east-1.amazonaws.com fs.s3.region: ap-east-1 name: s3 path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: s3 spec: replicas: 2 tieredstore: levels: - mediumtype: MEM path: /dev/shm quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d s3 \u7684\u8def\u5f84\uff0c\u652f\u6301\u6807\u51c6 s3 \u534f\u8bae\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u7684 worker \u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c\u5355\u4f4dGi\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b dataset \u7684\u60c5\u51b5 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE s3 210MiB 0.00B 180.00GiB 0.0% Bound 1h \u6267\u884c create \u547d\u4ee4 $ kubectl create -f runtime.yaml","title":"\u521b\u5efa dataset \u548c JindoRuntime"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_s3_ufs_example/#dataset","text":"$ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE s3 180.01GiB 0.00B 261.00GiB 0.0% Bound 62m","title":"\u67e5\u770b dataset \u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_s3_ufs_example/#pvpvc","text":"$ kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/s3 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/s3 Bound hadoop 100Gi RWX 58m","title":"\u68c0\u67e5 PV\uff0cPVC \u521b\u5efa\u60c5\u51b5"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/","text":"Fluid + JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u9ad8\u9636\u529f\u80fd\u4f7f\u7528 \u00a7 \u4f7f\u7528\u5217\u8868 \u6302\u8f7d\u70b9\u5728\u6839\u76ee\u5f55\u4e0b Secret \u52a0\u5bc6 AK \u53c2\u6570 Raft 3 master \u6a21\u5f0f \u4f7f\u7528 Placement \u90e8\u7f72\u591a\u4e2a runtime \u4f7f\u7528 NoseSelector \u90e8\u7f72\u8282\u70b9 \u4f7f\u7528 dataset nodeAffinity \u529f\u80fd Worker \u4e2a\u6570\u6269\u7f29\u5bb9 \u4f7f\u7528 tolerations \u529f\u80fd Resource \u8d44\u6e90 Fuse \u56de\u6536\u7b56\u7565 JindoFSx \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528 JindoFSx Fuse \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528 FuseOnly \u4f7f\u7528\u65b9\u5f0f \u591a\u6302\u8f7d\u70b9 master\u5143\u6570\u636e\u6301\u4e45\u5316 \u6302\u8f7d\u70b9\u5728\u6839\u76ee\u5f55\u4e0b \u00a7 \u9ed8\u8ba4\u4f7f\u7528 JindoRuntime \u4f1a\u5728\u6302\u8f7d\u70b9\u591a\u4e00\u5c42 / \u7684\u76ee\u5f55\uff0c\u5982\u679c\u60f3\u6302\u8f7d\u5728\u6839\u76ee\u5f55\u4e0b\u53ef\u4ee5\u5728 dataset \u91cc\u8fdb\u884c\u53c2\u6570\u6307\u5b9a apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase path: / \u6307\u5b9a spec.mounts.path = /\uff0c\u5219\u4f1a\u5c06\u6587\u4ef6\u6302\u8f7d\u5728\u6839\u76ee\u5f55\u4e0b Secret \u52a0\u5bc6 AK \u53c2\u6570 \u00a7 apiVersion: v1 kind: Secret metadata: name: mysecret stringData: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> \u5728 dataset \u91cc\u4f7f\u7528 secret apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://<OSS_BUCKET>/<OSS_DIRECTORY>/ name: hadoop options: fs.oss.endpoint: <OSS_ENDPOINT> encryptOptions: - name: fs.oss.accessKeyId valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeyId - name: fs.oss.accessKeySecret valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeySecret Raft 3 master \u6a21\u5f0f \u00a7 JindoRuntime \u6a21\u5f0f\u542f\u52a8\u4e00\u4e2a master\uff0c\u53ef\u4ee5\u901a\u8fc7\u4f7f\u7528 master.replicas \u6765\u542f\u52a8 3 \u4e2a master \u8fdb\u884c HA \u8f6c\u6362\u3002 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 3 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 2G high: \"0.8\" low: \"0.7\" master: replicas: 3 master.replicas = 3 : \u542f\u52a8 3 \u4e2a master pod \u4f7f\u7528 Placement \u90e8\u7f72\u591a\u4e2a runtime \u00a7 JindoRuntime \u542f\u52a8\u540e\uff0c\u9ed8\u8ba4 worker \u8282\u70b9\u4e0a\u53ea\u80fd\u542f\u52a8\u4e00\u4e2a runtime \u7684worker\uff0c\u5c5e\u4e8e\u72ec\u5360\u6a21\u5f0f\uff0c\u5982\u679c\u60f3\u8981\u5728\u540c\u4e00\u4e2a\u8282\u70b9\u4e0a\u652f\u6301\u90e8\u7f72\u591a\u4e2a worker\uff0c\u53ef\u4ee5\u4f7f\u7528 shared \u6a21\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase placement: \"Shared\" \u4f7f\u7528 NoseSelector \u90e8\u7f72\u8282\u70b9 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /mnt/disk1 quota: 10Gi high: \"0.9\" low: \"0.8\" master: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1 worker: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1 fuse: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1 \u4f7f\u7528 dataset nodeAffinity \u529f\u80fd \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: hbase-cache operator: In values: - \"true\" Worker \u4e2a\u6570\u6269\u7f29\u5bb9 \u00a7 \u4f7f\u7528 kubectl scale \u5bf9 Runtime \u7684 Worker \u6570\u91cf\u8fdb\u884c\u8c03\u6574 $ kubectl scale jindoruntime <runtime_name> --replicas=<replica_num> \u5176\u4e2d * runtime_name\uff1aruntime \u7684\u540d\u5b57 * replica_num\uff1a\u8868\u793a\u6269\u6240\u5bb9\u7684 Worker \u6570\u91cf \u4f7f\u7528 tolerations \u529f\u80fd \u00a7 \u60a8\u53ef\u4ee5\u5728 dataset \u91cc\u5b9a\u4e49 tolerations \u6307\u5b9a worker \u8282\u70b9\u7684\u8c03\u5ea6 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase tolerations: - key: hbase operator: Equal value: \"true\" \u4e5f\u53ef\u4ee5\u4e3a master/worker/fuse \u5355\u72ec\u6307\u5b9a tolerations \u8c03\u5ea6\u7b56\u7565 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 3 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 2Gi high: \"0.8\" low: \"0.7\" master: tolerations: - key: hbase operator: Equal value: \"true\" worker: tolerations: - key: hbase operator: Equal value: \"true\" fuse: tolerations: - key: hbase operator: Equal value: \"true\" Resource \u8d44\u6e90 \u00a7 \u53ef\u4ee5\u6307\u5b9a master/worker \u7b49\u7684 resource \u8d44\u6e90 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" master: resources: limits: cpu: \"4\" memory: \"8Gi\" requests: cpu: \"2\" memory: \"3Gi\" worker: resources: limits: cpu: \"4\" memory: \"8Gi\" requests: cpu: \"2\" memory: \"3Gi\" Fuse \u56de\u6536\u7b56\u7565 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: cleanPolicy: OnDemand fuse.cleanPolicy: onDemand / OnRuntimeDeleted \u5206\u522b\u8868\u793a\u6309\u9700\u542f\u52a8\uff0c\u4efb\u52a1\u7ed3\u675f fuse \u4e5f\u7ed3\u675f\u548c\u5728 runtime \u9500\u6bc1\u7684\u65f6\u5019\u548c\u5176\u4ed6\u7ec4\u4ef6\u4e00\u8d77\u9500\u6bc1\u6389 JindoFSx \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528 \u00a7 \u4e0b\u9762\u4ecb\u7ecd\u4e00\u4e9b\u548c JindoFSx \u7f13\u5b58\u76f8\u5173\u7684\u53c2\u6570\uff0c\u53ef\u4ee5\u6309\u7167\u5b9e\u9645\u573a\u666f\u8fdb\u884c\u9009\u62e9 Parameter Value Description fs.jindofsx.data.cache.enable true/false \u662f\u5426\u6253\u5f00\u6570\u636e\u7f13\u5b58\uff0c\u9ed8\u8ba4\u4e3a\u6253\u5f00 fs.jindofsx.meta.cache.enable true/false \u662f\u5426\u6253\u5f00\u5143\u6570\u636e\u7f13\u5b58\uff0c\u9ed8\u8ba4\u4e3a\u5173\u95ed fs.jindofsx.slicelet.cache.enable true/false \u662f\u5426\u6253\u5f00\u5c0f\u6587\u4ef6\u7f13\u5b58\u4f18\u5316(\u5199\u78c1\u76d8)\uff0c\u9ed8\u8ba4\u4e3a\u5173\u95ed fs.jindofsx.short.circuit.enable true/false \u662f\u5426\u6253\u5f00\u77ed\u8def\u8bfb\uff0c\u9ed8\u8ba4\u4e3a\u6253\u5f00 \u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: properties: fs.jindofsx.data.cache.enable: \"false\" fs.jindofsx.meta.cache.enable: \"true\" fs.jindofsx.slicelet.cache.enable: \"false\" fs.jindofsx.short.circuit.enable: \"false\" \u4ee5 kv \u65b9\u5f0f\u5199\u5728 spec.fuse.properties \u91cc\uff0c\u6309\u7167\u9700\u8981\u586b\u5199\u5373\u53ef\u3002 JindoFSx Fuse \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528 \u00a7 \u53c2\u6570\u540d\u79f0 \u53c2\u6570\u8bf4\u660e \u4f7f\u7528\u8303\u4f8b auto_unmount fuse\u8fdb\u7a0b\u9000\u51fa\u540e\u81ea\u52a8umount\u6302\u8f7d\u8282\u70b9\u3002 -oauto_unmount ro \u53ea\u8bfb\u6302\u8f7d\uff0c\u542f\u7528\u53c2\u6570\u540e\u4e0d\u5141\u8bb8\u5199\u64cd\u4f5c\u3002 -oro direct_io \u5f00\u542f\u540e\uff0c\u8bfb\u5199\u6587\u4ef6\u53ef\u4ee5\u7ed5\u8fc7page cache\u3002 -odirect_io kernel_cache \u5f00\u542f\u540e\uff0c\u5229\u7528\u5185\u6838\u7f13\u5b58\u4f18\u5316\u8bfb\u6027\u80fd\u3002 -okernel_cache auto_cache \u9ed8\u8ba4\u5f00\u542f\uff0c\u4e0ekernel_cache \u4e8c\u9009\u4e00\uff0c\u4e0ekernel_cache\u4e0d\u540c\u7684\u662f\uff0c\u5982\u679c\u6587\u4ef6\u5927\u5c0f\u6216\u4fee\u6539\u65f6\u95f4\u53d1\u751f\u53d8\u5316\uff0c\u7f13\u5b58\u5c31\u4f1a\u5931\u6548\u3002 -oauto_cache entry_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-oentry_timeout=60 attr_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-oattr_timeout=60 negative_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u5931\u8d25\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-onegative_timeout=30 \u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: args: - -oro - -oattr_timeout=60 - -oentry_timeout=60 - -onegative_timeout=60 \u4ee5\u6570\u7ec4\u65b9\u5f0f\u5199\u5728 spec.fuse.args \u91cc\uff0c\u6309\u7167\u9700\u8981\u586b\u5199\u5373\u53ef \u5982\u5355\u72ec\u914d\u7f6e\u5ba2\u6237\u7aefpread\u53c2\u6570\uff1a apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: args: - -oro - -ometrics_port=0 - -okernel_cache - -oattr_timeout=7200 - -oentry_timeout=7200 - -onegative_timeout=7200 - -opread FuseOnly \u4f7f\u7528\u65b9\u5f0f \u00a7 \u5982\u60a8\u60f3\u4f7f\u7528 FuseOnly \u65b9\u5f0f\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: master: disabled: true worker: disabled: true \u591a\u6302\u8f7d\u70b9 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: spark - mountPoint: oss://test-bucket-2/dir2/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop \u5728\u6302\u8f7d pvc \u548c fuse \u4ee5\u53ca\u505a dataload \u7684\u65f6\u5019\uff0c\u8fd9\u4e24\u4e2a\u6570\u636e\u6e90\u7684\u65b9\u5f0f\u5c06\u4ee5 mounts.name \u4f5c\u4e3a\u533a\u5206\uff0c\u6bd4\u5982 /spark \u8def\u5f84\u4e0b\u8bbf\u95ee oss://test-bucket-1/dir1/ \u4e0b\u7684\u6587\u4ef6\uff0c/hadoop \u8def\u5f84\u4e0b\u8bbf\u95ee oss://test-bucket-2/dir2/ \u4e0b\u7684\u6587\u4ef6\u5185\u5bb9 master\u5143\u6570\u636e\u6301\u4e45\u5316 \u00a7 \u5982\u679c\u60f3\u901a\u8fc7\u7ed9master\u6302\u8f7dvolume\u7684\u65b9\u5f0f\uff0c\u8ba9master\u7684\u5143\u6570\u636e\u6301\u4e45\u5316\u5230\u6307\u5b9a\u7684\u5b58\u50a8\u4e0a\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" volumes: - name: nas persistentVolumeClaim: claimName: nas-test master: volumeMounts: - name: nas subPath: test mountPath: /var/cache/ properties: namespace.meta-dir: \"/var/cache/\" \u8be5\u4f8b\u5b50\u5c06\u5143\u6570\u636e\u76ee\u5f55 namespace.meta-dir \u6307\u5b9a\u5230\u6302\u8f7d\u7684volume\u4e0a\uff0c\u5e76\u8fdb\u884c\u6301\u4e45\u5316\uff0c\u6ce8\u610f\u6302\u8f7d\u7684vomlume\u5fc5\u987b\u5c31\u6709\u8bfb/\u5199/\u5220\u9664\u6743\u9650\uff0c\u4e0d\u53ef\u6307\u5b9a\u4e3a\u53ea\u8bfb","title":"Fluid + JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u9ad8\u9636\u529f\u80fd\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#fluid-jindofsx","text":"\u4f7f\u7528\u5217\u8868 \u6302\u8f7d\u70b9\u5728\u6839\u76ee\u5f55\u4e0b Secret \u52a0\u5bc6 AK \u53c2\u6570 Raft 3 master \u6a21\u5f0f \u4f7f\u7528 Placement \u90e8\u7f72\u591a\u4e2a runtime \u4f7f\u7528 NoseSelector \u90e8\u7f72\u8282\u70b9 \u4f7f\u7528 dataset nodeAffinity \u529f\u80fd Worker \u4e2a\u6570\u6269\u7f29\u5bb9 \u4f7f\u7528 tolerations \u529f\u80fd Resource \u8d44\u6e90 Fuse \u56de\u6536\u7b56\u7565 JindoFSx \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528 JindoFSx Fuse \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528 FuseOnly \u4f7f\u7528\u65b9\u5f0f \u591a\u6302\u8f7d\u70b9 master\u5143\u6570\u636e\u6301\u4e45\u5316","title":"Fluid + JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u9ad8\u9636\u529f\u80fd\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#_1","text":"\u9ed8\u8ba4\u4f7f\u7528 JindoRuntime \u4f1a\u5728\u6302\u8f7d\u70b9\u591a\u4e00\u5c42 / \u7684\u76ee\u5f55\uff0c\u5982\u679c\u60f3\u6302\u8f7d\u5728\u6839\u76ee\u5f55\u4e0b\u53ef\u4ee5\u5728 dataset \u91cc\u8fdb\u884c\u53c2\u6570\u6307\u5b9a apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase path: / \u6307\u5b9a spec.mounts.path = /\uff0c\u5219\u4f1a\u5c06\u6587\u4ef6\u6302\u8f7d\u5728\u6839\u76ee\u5f55\u4e0b","title":"\u6302\u8f7d\u70b9\u5728\u6839\u76ee\u5f55\u4e0b"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#secret-ak","text":"apiVersion: v1 kind: Secret metadata: name: mysecret stringData: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> \u5728 dataset \u91cc\u4f7f\u7528 secret apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://<OSS_BUCKET>/<OSS_DIRECTORY>/ name: hadoop options: fs.oss.endpoint: <OSS_ENDPOINT> encryptOptions: - name: fs.oss.accessKeyId valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeyId - name: fs.oss.accessKeySecret valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeySecret","title":"Secret \u52a0\u5bc6 AK \u53c2\u6570"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#raft-3-master","text":"JindoRuntime \u6a21\u5f0f\u542f\u52a8\u4e00\u4e2a master\uff0c\u53ef\u4ee5\u901a\u8fc7\u4f7f\u7528 master.replicas \u6765\u542f\u52a8 3 \u4e2a master \u8fdb\u884c HA \u8f6c\u6362\u3002 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 3 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 2G high: \"0.8\" low: \"0.7\" master: replicas: 3 master.replicas = 3 : \u542f\u52a8 3 \u4e2a master pod","title":"Raft 3 master \u6a21\u5f0f"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#placement-runtime","text":"JindoRuntime \u542f\u52a8\u540e\uff0c\u9ed8\u8ba4 worker \u8282\u70b9\u4e0a\u53ea\u80fd\u542f\u52a8\u4e00\u4e2a runtime \u7684worker\uff0c\u5c5e\u4e8e\u72ec\u5360\u6a21\u5f0f\uff0c\u5982\u679c\u60f3\u8981\u5728\u540c\u4e00\u4e2a\u8282\u70b9\u4e0a\u652f\u6301\u90e8\u7f72\u591a\u4e2a worker\uff0c\u53ef\u4ee5\u4f7f\u7528 shared \u6a21\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase placement: \"Shared\"","title":"\u4f7f\u7528 Placement \u90e8\u7f72\u591a\u4e2a runtime"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#noseselector","text":"apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /mnt/disk1 quota: 10Gi high: \"0.9\" low: \"0.8\" master: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1 worker: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1 fuse: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1","title":"\u4f7f\u7528 NoseSelector \u90e8\u7f72\u8282\u70b9"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#dataset-nodeaffinity","text":"apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: hbase-cache operator: In values: - \"true\"","title":"\u4f7f\u7528 dataset nodeAffinity \u529f\u80fd"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#worker","text":"\u4f7f\u7528 kubectl scale \u5bf9 Runtime \u7684 Worker \u6570\u91cf\u8fdb\u884c\u8c03\u6574 $ kubectl scale jindoruntime <runtime_name> --replicas=<replica_num> \u5176\u4e2d * runtime_name\uff1aruntime \u7684\u540d\u5b57 * replica_num\uff1a\u8868\u793a\u6269\u6240\u5bb9\u7684 Worker \u6570\u91cf","title":"Worker \u4e2a\u6570\u6269\u7f29\u5bb9"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#tolerations","text":"\u60a8\u53ef\u4ee5\u5728 dataset \u91cc\u5b9a\u4e49 tolerations \u6307\u5b9a worker \u8282\u70b9\u7684\u8c03\u5ea6 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase tolerations: - key: hbase operator: Equal value: \"true\" \u4e5f\u53ef\u4ee5\u4e3a master/worker/fuse \u5355\u72ec\u6307\u5b9a tolerations \u8c03\u5ea6\u7b56\u7565 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 3 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 2Gi high: \"0.8\" low: \"0.7\" master: tolerations: - key: hbase operator: Equal value: \"true\" worker: tolerations: - key: hbase operator: Equal value: \"true\" fuse: tolerations: - key: hbase operator: Equal value: \"true\"","title":"\u4f7f\u7528 tolerations \u529f\u80fd"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#resource","text":"\u53ef\u4ee5\u6307\u5b9a master/worker \u7b49\u7684 resource \u8d44\u6e90 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" master: resources: limits: cpu: \"4\" memory: \"8Gi\" requests: cpu: \"2\" memory: \"3Gi\" worker: resources: limits: cpu: \"4\" memory: \"8Gi\" requests: cpu: \"2\" memory: \"3Gi\"","title":"Resource \u8d44\u6e90"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#fuse","text":"apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: cleanPolicy: OnDemand fuse.cleanPolicy: onDemand / OnRuntimeDeleted \u5206\u522b\u8868\u793a\u6309\u9700\u542f\u52a8\uff0c\u4efb\u52a1\u7ed3\u675f fuse \u4e5f\u7ed3\u675f\u548c\u5728 runtime \u9500\u6bc1\u7684\u65f6\u5019\u548c\u5176\u4ed6\u7ec4\u4ef6\u4e00\u8d77\u9500\u6bc1\u6389","title":"Fuse \u56de\u6536\u7b56\u7565"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#jindofsx","text":"\u4e0b\u9762\u4ecb\u7ecd\u4e00\u4e9b\u548c JindoFSx \u7f13\u5b58\u76f8\u5173\u7684\u53c2\u6570\uff0c\u53ef\u4ee5\u6309\u7167\u5b9e\u9645\u573a\u666f\u8fdb\u884c\u9009\u62e9 Parameter Value Description fs.jindofsx.data.cache.enable true/false \u662f\u5426\u6253\u5f00\u6570\u636e\u7f13\u5b58\uff0c\u9ed8\u8ba4\u4e3a\u6253\u5f00 fs.jindofsx.meta.cache.enable true/false \u662f\u5426\u6253\u5f00\u5143\u6570\u636e\u7f13\u5b58\uff0c\u9ed8\u8ba4\u4e3a\u5173\u95ed fs.jindofsx.slicelet.cache.enable true/false \u662f\u5426\u6253\u5f00\u5c0f\u6587\u4ef6\u7f13\u5b58\u4f18\u5316(\u5199\u78c1\u76d8)\uff0c\u9ed8\u8ba4\u4e3a\u5173\u95ed fs.jindofsx.short.circuit.enable true/false \u662f\u5426\u6253\u5f00\u77ed\u8def\u8bfb\uff0c\u9ed8\u8ba4\u4e3a\u6253\u5f00 \u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: properties: fs.jindofsx.data.cache.enable: \"false\" fs.jindofsx.meta.cache.enable: \"true\" fs.jindofsx.slicelet.cache.enable: \"false\" fs.jindofsx.short.circuit.enable: \"false\" \u4ee5 kv \u65b9\u5f0f\u5199\u5728 spec.fuse.properties \u91cc\uff0c\u6309\u7167\u9700\u8981\u586b\u5199\u5373\u53ef\u3002","title":"JindoFSx \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#jindofsx-fuse","text":"\u53c2\u6570\u540d\u79f0 \u53c2\u6570\u8bf4\u660e \u4f7f\u7528\u8303\u4f8b auto_unmount fuse\u8fdb\u7a0b\u9000\u51fa\u540e\u81ea\u52a8umount\u6302\u8f7d\u8282\u70b9\u3002 -oauto_unmount ro \u53ea\u8bfb\u6302\u8f7d\uff0c\u542f\u7528\u53c2\u6570\u540e\u4e0d\u5141\u8bb8\u5199\u64cd\u4f5c\u3002 -oro direct_io \u5f00\u542f\u540e\uff0c\u8bfb\u5199\u6587\u4ef6\u53ef\u4ee5\u7ed5\u8fc7page cache\u3002 -odirect_io kernel_cache \u5f00\u542f\u540e\uff0c\u5229\u7528\u5185\u6838\u7f13\u5b58\u4f18\u5316\u8bfb\u6027\u80fd\u3002 -okernel_cache auto_cache \u9ed8\u8ba4\u5f00\u542f\uff0c\u4e0ekernel_cache \u4e8c\u9009\u4e00\uff0c\u4e0ekernel_cache\u4e0d\u540c\u7684\u662f\uff0c\u5982\u679c\u6587\u4ef6\u5927\u5c0f\u6216\u4fee\u6539\u65f6\u95f4\u53d1\u751f\u53d8\u5316\uff0c\u7f13\u5b58\u5c31\u4f1a\u5931\u6548\u3002 -oauto_cache entry_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-oentry_timeout=60 attr_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-oattr_timeout=60 negative_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u5931\u8d25\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-onegative_timeout=30 \u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: args: - -oro - -oattr_timeout=60 - -oentry_timeout=60 - -onegative_timeout=60 \u4ee5\u6570\u7ec4\u65b9\u5f0f\u5199\u5728 spec.fuse.args \u91cc\uff0c\u6309\u7167\u9700\u8981\u586b\u5199\u5373\u53ef \u5982\u5355\u72ec\u914d\u7f6e\u5ba2\u6237\u7aefpread\u53c2\u6570\uff1a apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: args: - -oro - -ometrics_port=0 - -okernel_cache - -oattr_timeout=7200 - -oentry_timeout=7200 - -onegative_timeout=7200 - -opread","title":"JindoFSx Fuse \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#fuseonly","text":"\u5982\u60a8\u60f3\u4f7f\u7528 FuseOnly \u65b9\u5f0f\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: master: disabled: true worker: disabled: true","title":"FuseOnly \u4f7f\u7528\u65b9\u5f0f"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#_2","text":"apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: spark - mountPoint: oss://test-bucket-2/dir2/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop \u5728\u6302\u8f7d pvc \u548c fuse \u4ee5\u53ca\u505a dataload \u7684\u65f6\u5019\uff0c\u8fd9\u4e24\u4e2a\u6570\u636e\u6e90\u7684\u65b9\u5f0f\u5c06\u4ee5 mounts.name \u4f5c\u4e3a\u533a\u5206\uff0c\u6bd4\u5982 /spark \u8def\u5f84\u4e0b\u8bbf\u95ee oss://test-bucket-1/dir1/ \u4e0b\u7684\u6587\u4ef6\uff0c/hadoop \u8def\u5f84\u4e0b\u8bbf\u95ee oss://test-bucket-2/dir2/ \u4e0b\u7684\u6587\u4ef6\u5185\u5bb9","title":"\u591a\u6302\u8f7d\u70b9"},{"location":"backup/jindofsx/jindo_fluid/jindo_fluid_ways_to_use/#master","text":"\u5982\u679c\u60f3\u901a\u8fc7\u7ed9master\u6302\u8f7dvolume\u7684\u65b9\u5f0f\uff0c\u8ba9master\u7684\u5143\u6570\u636e\u6301\u4e45\u5316\u5230\u6307\u5b9a\u7684\u5b58\u50a8\u4e0a\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" volumes: - name: nas persistentVolumeClaim: claimName: nas-test master: volumeMounts: - name: nas subPath: test mountPath: /var/cache/ properties: namespace.meta-dir: \"/var/cache/\" \u8be5\u4f8b\u5b50\u5c06\u5143\u6570\u636e\u76ee\u5f55 namespace.meta-dir \u6307\u5b9a\u5230\u6302\u8f7d\u7684volume\u4e0a\uff0c\u5e76\u8fdb\u884c\u6301\u4e45\u5316\uff0c\u6ce8\u610f\u6302\u8f7d\u7684vomlume\u5fc5\u987b\u5c31\u6709\u8bfb/\u5199/\u5220\u9664\u6743\u9650\uff0c\u4e0d\u53ef\u6307\u5b9a\u4e3a\u53ea\u8bfb","title":"master\u5143\u6570\u636e\u6301\u4e45\u5316"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_nas_ufs_example/","text":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001\u4e0b\u8f7d\u5b89\u88c5\u5305 \u00a7 \u70b9\u51fb\u4e0b\u8f7d 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-0.7.0.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907 \u00a7 5.1 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: nas spec: mounts: - mountPoint: local:///mnt/nas name: nasData --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: nas spec: replicas: 2 tieredstore: levels: - mediumtype: MEM path: /dev/shm quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d NAS \u7684\u8def\u5f84\uff0c local:// \u4e3a\u672c\u5730\u8def\u5f84\u524d\u7f00\uff0c /mnt/nas \u8868\u793a\u672c\u5730\u771f\u5b9e\u8def\u5f84\uff0c\u8be5\u8def\u5f84\u9700\u8981\u5728\u5bbf\u4e3b\u673a\u4e0a\u53ef\u89c1\u3002 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JindoRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime nas NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE nas Ready Ready 62m \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/nas 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/nas Bound hadoop 100Gi RWX 58m \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 NAS \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: test spec: dataset: name: nas namespace: default target: - path: /nasData replicas: 1 \u5176\u4e2d /nasData \u4e2d nasData \u4e3a dataset \u4e2d\u6302\u8f7d\u70b9\u7684 name\uff0c\u53ef\u4ee5\u4f7f\u7528 /nasData/xxx \u6765\u9884\u70ed\u5176\u4e2d\u7684\u5b50\u76ee\u5f55\u3002 replicas \u8868\u793a\u7f13\u5b58\u7684\u5907\u4efd\u6570\uff0c\u9ed8\u8ba4\u662f1\u3002 \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload test-dataload NAME DATASET PHASE AGE dataload nas Completed 2m13s 6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: nas \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u8bbf\u95ee\u6302\u8f7d\u70b9\u5bf9\u5e94\u7684\u5e95\u5c42\u6587\u4ef6 $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 7\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime nas \u5220\u9664 dataset kubectl delete dataset nas","title":"Jindo fluid nas ufs example"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_nas_ufs_example/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_nas_ufs_example/#2","text":"\u70b9\u51fb\u4e0b\u8f7d","title":"2\u3001\u4e0b\u8f7d\u5b89\u88c5\u5305"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_nas_ufs_example/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-0.7.0.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_nas_ufs_example/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_nas_ufs_example/#5","text":"","title":"5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_nas_ufs_example/#51-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: nas spec: mounts: - mountPoint: local:///mnt/nas name: nasData --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: nas spec: replicas: 2 tieredstore: levels: - mediumtype: MEM path: /dev/shm quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d NAS \u7684\u8def\u5f84\uff0c local:// \u4e3a\u672c\u5730\u8def\u5f84\u524d\u7f00\uff0c /mnt/nas \u8868\u793a\u672c\u5730\u771f\u5b9e\u8def\u5f84\uff0c\u8be5\u8def\u5f84\u9700\u8981\u5728\u5bbf\u4e3b\u673a\u4e0a\u53ef\u89c1\u3002 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JindoRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime nas NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE nas Ready Ready 62m \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/nas 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/nas Bound hadoop 100Gi RWX 58m \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 NAS \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: test spec: dataset: name: nas namespace: default target: - path: /nasData replicas: 1 \u5176\u4e2d /nasData \u4e2d nasData \u4e3a dataset \u4e2d\u6302\u8f7d\u70b9\u7684 name\uff0c\u53ef\u4ee5\u4f7f\u7528 /nasData/xxx \u6765\u9884\u70ed\u5176\u4e2d\u7684\u5b50\u76ee\u5f55\u3002 replicas \u8868\u793a\u7f13\u5b58\u7684\u5907\u4efd\u6570\uff0c\u9ed8\u8ba4\u662f1\u3002 \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload test-dataload NAME DATASET PHASE AGE dataload nas Completed 2m13s","title":"5.1 \u521b\u5efa dataset \u548c JindoRuntime"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_nas_ufs_example/#6","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: nas \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u8bbf\u95ee\u6302\u8f7d\u70b9\u5bf9\u5e94\u7684\u5e95\u5c42\u6587\u4ef6 $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","title":"6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_nas_ufs_example/#7","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime nas \u5220\u9664 dataset kubectl delete dataset nas","title":"7\u3001\u73af\u5883\u6e05\u7406"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_oss_append_fuseonly/","text":"\u52a0\u901fOSS\u4e0a\u6570\u636e \u00a7 \u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002 1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001 \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s 5\u3001\u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop path: / accessModes: - ReadWriteMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: master: disabled: true worker: disabled: true mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aoss bucket\u7684endpoint\u4fe1\u606f\uff0c\u516c\u7f51\u6216\u5185\u7f51\u5730\u5740\u7686\u53ef\uff0c\u5982\u60a8\u7684 bucket \u5728\u676d\u5dde Region\uff0c\u90a3\u4e48\u516c\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou.aliyuncs.com\uff0c\u5185\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou-internal.aliyuncs.com(\u5185\u7f51\u5730\u5740\u4f7f\u7528\u6761\u4ef6\u4e3a\u60a8\u7684 K8S \u96c6\u7fa4\u6240\u5728\u533a\u57df\u548coss\u533a\u57df\u76f8\u540c)\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m 6\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_oss_append_fuseonly/#oss","text":"\u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_oss_append_fuseonly/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_oss_append_fuseonly/#2-fluid","text":"","title":"2\u3001\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_oss_append_fuseonly/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_oss_append_fuseonly/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_oss_append_fuseonly/#5-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop path: / accessModes: - ReadWriteMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: master: disabled: true worker: disabled: true mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aoss bucket\u7684endpoint\u4fe1\u606f\uff0c\u516c\u7f51\u6216\u5185\u7f51\u5730\u5740\u7686\u53ef\uff0c\u5982\u60a8\u7684 bucket \u5728\u676d\u5dde Region\uff0c\u90a3\u4e48\u516c\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou.aliyuncs.com\uff0c\u5185\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou-internal.aliyuncs.com(\u5185\u7f51\u5730\u5740\u4f7f\u7528\u6761\u4ef6\u4e3a\u60a8\u7684 K8S \u96c6\u7fa4\u6240\u5728\u533a\u57df\u548coss\u533a\u57df\u76f8\u540c)\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m","title":"5\u3001\u521b\u5efa dataset \u548c JindoRuntime"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_oss_append_fuseonly/#6","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"6\u3001\u73af\u5883\u6e05\u7406"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_sdk_connect_example/","text":"\u4f7f\u7528 JindoSDK \u8bbf\u95ee Fluid JindoRuntime \u96c6\u7fa4 \u00a7 \u6b65\u9aa4 \u00a7 \u524d\u63d0\uff1a\u542f\u52a8 Fluid JindoRuntime \u96c6\u7fa4 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e core-site \u00a7 \u767b\u9646\u5230 xxx-jindofs-master-0 \u5bb9\u5668\u91cc kubectl exec -ti xxx-jindofs-master-0 bash \u627e\u5230 /hdfs-3.2.1/etc/hadoop/core-site.xml \u6587\u4ef6\uff0c\u5c06\u6587\u4ef6\u5185\u5bb9\u5168\u90e8\u52a0\u5165\u5230 hadoop \u7684 classpath \u4e0b\u7684 core-site \u6587\u4ef6\u4e2d 4. shell\u547d\u4ee4 \u00a7 \u53ef\u4ee5\u6267\u884c hadoop shell \u547d\u4ee4\u6765\u5224\u65ad\u670d\u52a1\u662f\u5426\u8fde\u63a5\u6210\u529f hadoop fs -ls oss://<Bucket>/<Dir> 6. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoFSx \u53c2\u6570\u8bf4\u660e JindoFSx \u5305\u542b\u4e00\u4e9b\u7f13\u5b58\u76f8\u5173\u7684\u53c2\u6570\uff0c\u8bf7\u53c2\u8003\u6587\u6863 JindoFSx \u5ba2\u6237\u7aef\u914d\u7f6e\u5217\u8868 * JindoSDK \u53c2\u6570\u8bf4\u660e JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 7. \u6743\u9650\u7ba1\u7406 \u00a7 \u81ea\u5efa\u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u963f\u91cc\u4e91 E-MapReduce \u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848","title":"\u4f7f\u7528 JindoSDK \u8bbf\u95ee Fluid JindoRuntime \u96c6\u7fa4"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_sdk_connect_example/#jindosdk-fluid-jindoruntime","text":"","title":"\u4f7f\u7528 JindoSDK \u8bbf\u95ee Fluid JindoRuntime \u96c6\u7fa4"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_sdk_connect_example/#_1","text":"\u524d\u63d0\uff1a\u542f\u52a8 Fluid JindoRuntime \u96c6\u7fa4","title":"\u6b65\u9aa4"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_sdk_connect_example/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_sdk_connect_example/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_sdk_connect_example/#3-core-site","text":"\u767b\u9646\u5230 xxx-jindofs-master-0 \u5bb9\u5668\u91cc kubectl exec -ti xxx-jindofs-master-0 bash \u627e\u5230 /hdfs-3.2.1/etc/hadoop/core-site.xml \u6587\u4ef6\uff0c\u5c06\u6587\u4ef6\u5185\u5bb9\u5168\u90e8\u52a0\u5165\u5230 hadoop \u7684 classpath \u4e0b\u7684 core-site \u6587\u4ef6\u4e2d","title":"3. \u914d\u7f6e core-site"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_sdk_connect_example/#4-shell","text":"\u53ef\u4ee5\u6267\u884c hadoop shell \u547d\u4ee4\u6765\u5224\u65ad\u670d\u52a1\u662f\u5426\u8fde\u63a5\u6210\u529f hadoop fs -ls oss://<Bucket>/<Dir>","title":"4. shell\u547d\u4ee4"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_sdk_connect_example/#6","text":"JindoFSx \u53c2\u6570\u8bf4\u660e JindoFSx \u5305\u542b\u4e00\u4e9b\u7f13\u5b58\u76f8\u5173\u7684\u53c2\u6570\uff0c\u8bf7\u53c2\u8003\u6587\u6863 JindoFSx \u5ba2\u6237\u7aef\u914d\u7f6e\u5217\u8868 * JindoSDK \u53c2\u6570\u8bf4\u660e JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"6. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_sdk_connect_example/#7","text":"\u81ea\u5efa\u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u963f\u91cc\u4e91 E-MapReduce \u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848","title":"7. \u6743\u9650\u7ba1\u7406"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_update_version_example/","text":"Fluid JindoRuntime \u955c\u50cf\u5347\u7ea7 \u00a7 1\u3001\u4fee\u6539 jindoruntime-controller \u7684\u4fe1\u606f kubectl edit deployment jindoruntime-controller -n fluid-system 2\u3001\u7f16\u8f91 controller\uff0c\u627e\u5230\u5982\u4e0b\u5185\u5bb9 - name: JINDO_SMARTDATA_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:4.5.2 - name: JINDO_FUSE_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:4.5.2 3\u3001\u4f8b\u5982\u4ece 4.5.1 \u5347\u7ea7\u5230 4.5.2\uff0c\u4fee\u6539\u5982\u4e0b\u955c\u50cf\u5373\u53ef - name: JINDO_SMARTDATA_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:4.5.2 - name: JINDO_FUSE_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:4.5.2 \u5982\u679c\u662f fuseOnly \u6a21\u5f0f\uff0c\u90a3\u4e48\u53ea\u9700\u4fee\u6539 JINDO_FUSE_IMAGE_ENV \u5b57\u6bb5\u5373\u53ef 4\u3001\u4fdd\u5b58\u4fee\u6539\uff0c\u91cd\u65b0\u521b\u5efa\u96c6\u7fa4\u5373\u53ef","title":"Fluid JindoRuntime \u955c\u50cf\u5347\u7ea7"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_fluid_update_version_example/#fluid-jindoruntime","text":"1\u3001\u4fee\u6539 jindoruntime-controller \u7684\u4fe1\u606f kubectl edit deployment jindoruntime-controller -n fluid-system 2\u3001\u7f16\u8f91 controller\uff0c\u627e\u5230\u5982\u4e0b\u5185\u5bb9 - name: JINDO_SMARTDATA_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:4.5.2 - name: JINDO_FUSE_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:4.5.2 3\u3001\u4f8b\u5982\u4ece 4.5.1 \u5347\u7ea7\u5230 4.5.2\uff0c\u4fee\u6539\u5982\u4e0b\u955c\u50cf\u5373\u53ef - name: JINDO_SMARTDATA_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:4.5.2 - name: JINDO_FUSE_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:4.5.2 \u5982\u679c\u662f fuseOnly \u6a21\u5f0f\uff0c\u90a3\u4e48\u53ea\u9700\u4fee\u6539 JINDO_FUSE_IMAGE_ENV \u5b57\u6bb5\u5373\u53ef 4\u3001\u4fdd\u5b58\u4fee\u6539\uff0c\u91cd\u65b0\u521b\u5efa\u96c6\u7fa4\u5373\u53ef","title":"Fluid JindoRuntime \u955c\u50cf\u5347\u7ea7"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_util_intro/","text":"jindo-util \u4f7f\u7528\u65b9\u6cd5 \u00a7 \u4e0b\u8f7d jindo-util \u5de5\u5177\uff08\u5f53\u524d\u53ea\u652f\u6301 Linux \u73af\u5883\u4f7f\u7528\uff09 \u00a7 1\u3001\u4e0b\u8f7d jindofsx-4.5.2.tar.gz 2\u3001\u89e3\u538b tar \u5305 tar -zxvf jindofsx-4.5.2.tar.gz 3\u3001\u5728 /jindofsx-4.5.2/bin/ \u76ee\u5f55\u4e0b\u627e\u5230 jindo-util \u4e8c\u8fdb\u5236\u6587\u4ef6 chmod 700 jindo-util mv jindo-util jindo 4\u3001\u67e5\u770b jindo-util \u6240\u6709\u652f\u6301\u7684\u547d\u4ee4\u548c\u4f7f\u7528\u65b9\u6cd5 # jindo fs -help Usage: jindo fs [-appendToFile <localsrc> ... <dst>] [-archive [-i] [-c] <path>] [-cat <src> ...] [-count [-p] [-r] <path> ...] [-cp [-f] <src> ... <dst>] [-checksum <src> ...] [-chgrp [-R] GROUP PATH...] [-chmod [-R] <MODE[,MODE]... | OCTALMODE> PATH...] [-chown [-R] [OWNER][:[GROUP]] PATH...] [-copyFromLocal [-f] <localsrc> ... <dst>] [-copyToLocal [-f] <src> ... <localdst>] [-createSnapshot <snapshotDir> [<snapshotName>]] [-checkStoragePolicy -path <path>] [-deleteSnapshot <snapshotDir> <snapshotName>] [-du [-s] [-p] [-r] <path> ...] [-decommission -hosts <hosts>] [-formatCache] [-get [-f] <src> ... <localdst>] [-getfacl [-R] <path>] [-getfattr [-R] {-n name | -d} <path>] [-getStoragePolicy -path <path>] [-ls [-R] <path>] [-listPolicies] [-load [-meta] [-data] [-s] [-m] [-R] [-replica value] <path>] [-mkdir <path>] [-mv <src> ... <dst>] [-moveFromLocal <localsrc> ... <dst>] [-moveToLocal <src> <localdst>] [-metaDiff <path>] [-put [-f] <localsrc> ... <dst>] [-rm [-f] [-r|-R] <src> ...] [-rmdir <src> ...] [-renameSnapshot <snapshotDir> <oldName> <newName>] [-restore [-days value] <path>] [-stat [format] <path> ...] [-setStoragePolicy -path <path> -policy <policy>] [-setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>]] [-setfattr {-n name [-v value] | -x name} <path>] [-touchz <path> ...] [-truncate [-w] <length> <path> ...] [-unarchive [-i] [-a] <path>] [-uncache <path>] [-unsetStoragePolicy -path <path>] jindo admin [-mount <mountpoint> <path>] [-report] [-reportMetrics] [-umount <mountpoint>] jindo admin [-addProxyUser -dlsUri <path> -proxyUser <value> -users|-groups <value> -hosts <value>] [-allowSnapshot -dlsUri <path>] [-addUserGroupsMapping -dlsUri <path> -user <value> -groups <value>] [-disallowSnapshot -dlsUri <path>] [-deleteProxyUser -dlsUri <path> -proxyUser <value>] [-deleteUserGroupsMapping -dlsUri <path> -user <value>] [-dumpFile -dlsUri <path>] [-dumpBlockFiles -in <path> -out <path>] [-listProxyUsers -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-listUserGroupsMappings -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-snapshotDiff -dlsUri <path> -fromSnapshot <value> -toSnapshot <value>] jindo benchmark [-io -read -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-memCheck]] [-io -pread -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-seekTimes <value>] [-memCheck]] [-io -write -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-memCheck]] [-meta -mkdir -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-memCheck]] [-meta -listDirectory -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]] [-meta -getFileStatus -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]] [-meta -renameFile -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]] \u4f7f\u7528 jindo-util \u8bbf\u95ee\u6807\u51c6 OSS \u00a7 1\u3001\u65b0\u5efa\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg [common] logger.dir = /tmp/jindo-util/ logger.sync = false logger.consolelogger = false logger.level = 0 logger.verbose = 0 logger.cleaner.enable = true hadoopConf.enable = false [jindosdk] fs.oss.endpoint = <OSS_ENDPOINT> fs.oss.accessKeyId = <OSS_ACCESSKEY_ID> fs.oss.accessKeySecret = <OSS_ACCESSKEY_SECRET> : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u7684 endpoint : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u7684 AccessKeyId : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u7684 AccessKeySecret 2\u3001\u6dfb\u52a0\u73af\u5883\u53d8\u91cf export JINDOSDK_CONF_DIR=<JINDOSDK_CFG_DIR> : jindosdk.cfg \u914d\u7f6e\u6587\u4ef6\u6240\u5728\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u6bd4\u5982 /etc/ 3\u3001\u4f7f\u7528 jindo-util \u8bbf\u95ee OSS jindo fs -ls oss://<bucket>/<dir> \u4f7f\u7528 jindo-util \u8bbf\u95ee OSS-HDFS \u00a7 1\u3001\u65b0\u5efa\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg [common] logger.dir = /tmp/jindo-util/ logger.sync = false logger.consolelogger = false logger.level = 0 logger.verbose = 0 logger.cleaner.enable = true hadoopConf.enable = false [jindosdk] fs.oss.bucket.<OSS_HDFS_BUCKET>.accessKeyId = <OSS_HDFS_ACCESSKEY_ID> fs.oss.bucket.<OSS_HDFS_BUCKET>.accessKeySecret = <OSS_HDFS_ACCESSKEY_SECRET> fs.oss.bucket.<OSS_HDFS_BUCKET>.endpoint = <OSS_HDFS_ENDPOINT> fs.oss.bucket.<OSS_HDFS_BUCKET>.data.lake.storage.enable = true : \u9700\u8981\u8bbf\u95ee\u7684 OSS-HDFS \u7684 bucket : \u9700\u8981\u8bbf\u95ee\u7684 OSS-HDFS \u7684 endpoint : \u9700\u8981\u8bbf\u95ee\u7684 OSS-HDFS \u7684 AccessKeyId : \u9700\u8981\u8bbf\u95ee\u7684 OSS-HDFS \u7684 AccessKeySecret 2\u3001\u6dfb\u52a0\u73af\u5883\u53d8\u91cf export JINDOSDK_CONF_DIR=<JINDOSDK_CFG_DIR> : jindosdk.cfg \u914d\u7f6e\u6587\u4ef6\u6240\u5728\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u6bd4\u5982 /etc/ 3\u3001\u4f7f\u7528 jindo-util \u8bbf\u95ee OSS-HDFS \u670d\u52a1 jindo fs -ls oss://<bucket>/<dir>","title":"jindo-util \u4f7f\u7528\u65b9\u6cd5"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_util_intro/#jindo-util","text":"","title":"jindo-util \u4f7f\u7528\u65b9\u6cd5"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_util_intro/#jindo-util-linux","text":"1\u3001\u4e0b\u8f7d jindofsx-4.5.2.tar.gz 2\u3001\u89e3\u538b tar \u5305 tar -zxvf jindofsx-4.5.2.tar.gz 3\u3001\u5728 /jindofsx-4.5.2/bin/ \u76ee\u5f55\u4e0b\u627e\u5230 jindo-util \u4e8c\u8fdb\u5236\u6587\u4ef6 chmod 700 jindo-util mv jindo-util jindo 4\u3001\u67e5\u770b jindo-util \u6240\u6709\u652f\u6301\u7684\u547d\u4ee4\u548c\u4f7f\u7528\u65b9\u6cd5 # jindo fs -help Usage: jindo fs [-appendToFile <localsrc> ... <dst>] [-archive [-i] [-c] <path>] [-cat <src> ...] [-count [-p] [-r] <path> ...] [-cp [-f] <src> ... <dst>] [-checksum <src> ...] [-chgrp [-R] GROUP PATH...] [-chmod [-R] <MODE[,MODE]... | OCTALMODE> PATH...] [-chown [-R] [OWNER][:[GROUP]] PATH...] [-copyFromLocal [-f] <localsrc> ... <dst>] [-copyToLocal [-f] <src> ... <localdst>] [-createSnapshot <snapshotDir> [<snapshotName>]] [-checkStoragePolicy -path <path>] [-deleteSnapshot <snapshotDir> <snapshotName>] [-du [-s] [-p] [-r] <path> ...] [-decommission -hosts <hosts>] [-formatCache] [-get [-f] <src> ... <localdst>] [-getfacl [-R] <path>] [-getfattr [-R] {-n name | -d} <path>] [-getStoragePolicy -path <path>] [-ls [-R] <path>] [-listPolicies] [-load [-meta] [-data] [-s] [-m] [-R] [-replica value] <path>] [-mkdir <path>] [-mv <src> ... <dst>] [-moveFromLocal <localsrc> ... <dst>] [-moveToLocal <src> <localdst>] [-metaDiff <path>] [-put [-f] <localsrc> ... <dst>] [-rm [-f] [-r|-R] <src> ...] [-rmdir <src> ...] [-renameSnapshot <snapshotDir> <oldName> <newName>] [-restore [-days value] <path>] [-stat [format] <path> ...] [-setStoragePolicy -path <path> -policy <policy>] [-setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>]] [-setfattr {-n name [-v value] | -x name} <path>] [-touchz <path> ...] [-truncate [-w] <length> <path> ...] [-unarchive [-i] [-a] <path>] [-uncache <path>] [-unsetStoragePolicy -path <path>] jindo admin [-mount <mountpoint> <path>] [-report] [-reportMetrics] [-umount <mountpoint>] jindo admin [-addProxyUser -dlsUri <path> -proxyUser <value> -users|-groups <value> -hosts <value>] [-allowSnapshot -dlsUri <path>] [-addUserGroupsMapping -dlsUri <path> -user <value> -groups <value>] [-disallowSnapshot -dlsUri <path>] [-deleteProxyUser -dlsUri <path> -proxyUser <value>] [-deleteUserGroupsMapping -dlsUri <path> -user <value>] [-dumpFile -dlsUri <path>] [-dumpBlockFiles -in <path> -out <path>] [-listProxyUsers -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-listUserGroupsMappings -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-snapshotDiff -dlsUri <path> -fromSnapshot <value> -toSnapshot <value>] jindo benchmark [-io -read -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-memCheck]] [-io -pread -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-seekTimes <value>] [-memCheck]] [-io -write -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-fileSize <value>] [-memCheck]] [-meta -mkdir -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-memCheck]] [-meta -listDirectory -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]] [-meta -getFileStatus -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]] [-meta -renameFile -basePath <path> [-n <value>] [-c <value>] [-d <value>] [-entryNum <value>] [-memCheck]]","title":"\u4e0b\u8f7d jindo-util \u5de5\u5177\uff08\u5f53\u524d\u53ea\u652f\u6301 Linux \u73af\u5883\u4f7f\u7528\uff09"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_util_intro/#jindo-util-oss","text":"1\u3001\u65b0\u5efa\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg [common] logger.dir = /tmp/jindo-util/ logger.sync = false logger.consolelogger = false logger.level = 0 logger.verbose = 0 logger.cleaner.enable = true hadoopConf.enable = false [jindosdk] fs.oss.endpoint = <OSS_ENDPOINT> fs.oss.accessKeyId = <OSS_ACCESSKEY_ID> fs.oss.accessKeySecret = <OSS_ACCESSKEY_SECRET> : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u7684 endpoint : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u7684 AccessKeyId : \u9700\u8981\u8bbf\u95ee\u7684 OSS \u7684 AccessKeySecret 2\u3001\u6dfb\u52a0\u73af\u5883\u53d8\u91cf export JINDOSDK_CONF_DIR=<JINDOSDK_CFG_DIR> : jindosdk.cfg \u914d\u7f6e\u6587\u4ef6\u6240\u5728\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u6bd4\u5982 /etc/ 3\u3001\u4f7f\u7528 jindo-util \u8bbf\u95ee OSS jindo fs -ls oss://<bucket>/<dir>","title":"\u4f7f\u7528 jindo-util \u8bbf\u95ee\u6807\u51c6 OSS"},{"location":"backup/jindofsx/jindo_fluid/case/jindo_util_intro/#jindo-util-oss-hdfs","text":"1\u3001\u65b0\u5efa\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg [common] logger.dir = /tmp/jindo-util/ logger.sync = false logger.consolelogger = false logger.level = 0 logger.verbose = 0 logger.cleaner.enable = true hadoopConf.enable = false [jindosdk] fs.oss.bucket.<OSS_HDFS_BUCKET>.accessKeyId = <OSS_HDFS_ACCESSKEY_ID> fs.oss.bucket.<OSS_HDFS_BUCKET>.accessKeySecret = <OSS_HDFS_ACCESSKEY_SECRET> fs.oss.bucket.<OSS_HDFS_BUCKET>.endpoint = <OSS_HDFS_ENDPOINT> fs.oss.bucket.<OSS_HDFS_BUCKET>.data.lake.storage.enable = true : \u9700\u8981\u8bbf\u95ee\u7684 OSS-HDFS \u7684 bucket : \u9700\u8981\u8bbf\u95ee\u7684 OSS-HDFS \u7684 endpoint : \u9700\u8981\u8bbf\u95ee\u7684 OSS-HDFS \u7684 AccessKeyId : \u9700\u8981\u8bbf\u95ee\u7684 OSS-HDFS \u7684 AccessKeySecret 2\u3001\u6dfb\u52a0\u73af\u5883\u53d8\u91cf export JINDOSDK_CONF_DIR=<JINDOSDK_CFG_DIR> : jindosdk.cfg \u914d\u7f6e\u6587\u4ef6\u6240\u5728\u7684\u7edd\u5bf9\u8def\u5f84\uff0c\u6bd4\u5982 /etc/ 3\u3001\u4f7f\u7528 jindo-util \u8bbf\u95ee OSS-HDFS \u670d\u52a1 jindo fs -ls oss://<bucket>/<dir>","title":"\u4f7f\u7528 jindo-util \u8bbf\u95ee OSS-HDFS"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/","text":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002JindoFuse \u4f7f\u7528\u539f\u6765\u7684\u5730\u5740 (oss://) \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c\u8be5\u670d\u52a1\u4e0a\u7684\u6587\u4ef6\uff0c\u540c\u65f6\u901a\u8fc7\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002 \u57fa\u672c\u4f7f\u7528 \u00a7 \u73af\u5883\u51c6\u5907 \u00a7 # CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u914d\u7f6e\u670d\u52a1\u7aef \u00a7 \u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-filestore] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = cn-xxx.oss-dls.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e\u5ba2\u6237\u7aef \u00a7 \u914d\u7f6e\u76ee\u5f55 \u00a7 \u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf \u914d\u7f6e\u6587\u4ef6 \u00a7 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u914d\u7f6e\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint= <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1\u7684 AccessKey ID \u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7 AccessKey \u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> # \u5982\u679c\u6709\u591a\u4e2a bucket \u5bf9\u5e94\u4e0d\u540c\u7684AccessKey ID \u548c AccessKey Secret\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u914d\u7f6e\u65b9\u5f0f fs.oss.bucket.<your-bucket>.accessKeyId = <your_key_id> fs.oss.bucket.<your-bucket>.accessKeySecret = <your_key_secret> # \u914d\u7f6eNamespace service \u5730\u5740 fs.jindofsx.namespace.rpc.address = <hostname>:<port> # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable = false \u6302\u8f7d JindoFuse \u00a7 \u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002 \u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 mkdir -p <mount_point> \u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 jindo-fuse <mount_point> -ouri=[<oss_path>] -oxengine=jindofsx -ouri \u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684 oss \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a Bucket \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002 \u8bbf\u95ee JindoFuse \u00a7 \u5982\u679c\u5c06 OSS-HDFS \u670d\u52a1\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1 ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/ \u5378\u8f7d JindoFuse \u00a7 \u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point> \u81ea\u52a8\u5378\u8f7d JindoFuse \u00a7 \u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u9ad8\u9636\u4f7f\u7528 \u00a7 \u914d\u7f6e\u9009\u9879 \u00a7 \u66f4\u591a jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<oss_path>] -ofs.jindofsx.data.cache.enable=false","title":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#jindofuse-oss-hdfs-jindofs","text":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002JindoFuse \u4f7f\u7528\u539f\u6765\u7684\u5730\u5740 (oss://) \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c\u8be5\u670d\u52a1\u4e0a\u7684\u6587\u4ef6\uff0c\u540c\u65f6\u901a\u8fc7\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002","title":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#_1","text":"","title":"\u57fa\u672c\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#_2","text":"# CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev","title":"\u73af\u5883\u51c6\u5907"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#_3","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#_4","text":"\u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-filestore] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = cn-xxx.oss-dls.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002","title":"\u914d\u7f6e\u670d\u52a1\u7aef"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#jindofsx","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#_5","text":"","title":"\u914d\u7f6e\u5ba2\u6237\u7aef"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#_6","text":"\u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf","title":"\u914d\u7f6e\u76ee\u5f55"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#_7","text":"\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u914d\u7f6e\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint= <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1\u7684 AccessKey ID \u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7 AccessKey \u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> # \u5982\u679c\u6709\u591a\u4e2a bucket \u5bf9\u5e94\u4e0d\u540c\u7684AccessKey ID \u548c AccessKey Secret\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u914d\u7f6e\u65b9\u5f0f fs.oss.bucket.<your-bucket>.accessKeyId = <your_key_id> fs.oss.bucket.<your-bucket>.accessKeySecret = <your_key_secret> # \u914d\u7f6eNamespace service \u5730\u5740 fs.jindofsx.namespace.rpc.address = <hostname>:<port> # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable = false","title":"\u914d\u7f6e\u6587\u4ef6"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#jindofuse","text":"\u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002","title":"\u6302\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#_8","text":"mkdir -p <mount_point>","title":"\u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#fuse","text":"jindo-fuse <mount_point> -ouri=[<oss_path>] -oxengine=jindofsx -ouri \u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684 oss \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a Bucket \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002","title":"\u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#jindofuse_1","text":"\u5982\u679c\u5c06 OSS-HDFS \u670d\u52a1\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1 ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/","title":"\u8bbf\u95ee JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#jindofuse_2","text":"\u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point>","title":"\u5378\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#jindofuse_3","text":"\u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002","title":"\u81ea\u52a8\u5378\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#_9","text":"","title":"\u9ad8\u9636\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_dls/#_10","text":"\u66f4\u591a jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<oss_path>] -ofs.jindofsx.data.cache.enable=false","title":"\u914d\u7f6e\u9009\u9879"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/","text":"JindoFuse \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f \u00a7 \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002JindoFuse \u4f7f\u7528\u7edf\u4e00\u6302\u8f7d\u7684\u65b9\u5f0f (jindo://) \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c\u8be5\u670d\u52a1\u4e0a\u7684\u6587\u4ef6\uff0c\u540c\u65f6\u901a\u8fc7\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002 \u57fa\u672c\u4f7f\u7528 \u00a7 \u73af\u5883\u51c6\u5907 \u00a7 # CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u914d\u7f6e\u670d\u52a1\u7aef \u00a7 \u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-filestore] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = cn-xxx.oss-dls.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e\u5ba2\u6237\u7aef \u00a7 \u914d\u7f6e\u76ee\u5f55 \u00a7 \u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf \u914d\u7f6e\u6587\u4ef6 \u00a7 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u914d\u7f6e\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 # \u6216\u914d\u7f6e\u963f\u91cc\u4e91 OSS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou-internal.aliyuncs.com\u3002 fs.oss.endpoint= <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1\u7684 AccessKey ID \u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7 AccessKey \u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> # \u914d\u7f6eNamespace service \u5730\u5740 fs.jindofsx.namespace.rpc.address = <hostname>:<port> # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable = false \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55 \u00a7 \u6302\u8f7d\u547d\u4ee4 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>/ \u6302\u8f7d JindoFuse \u00a7 \u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002 \u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 mkdir -p <mount_point> \u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 jindo-fuse <mount_point> -ouri=[<jindo_path>] -oxengine=jindofsx -ouri \u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684 jindo \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a\u7edf\u4e00\u6302\u8f7d\uff0c\u6bd4\u5982\u4e4b\u524d\u6302\u8f7d\u7684 jindo:///jindooss \u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002 \u8bbf\u95ee JindoFuse \u00a7 \u5982\u679c\u5c06 jindo \u8def\u5f84\uff08jindo://\uff09\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/jindo/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/jindo/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/jindo/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/jindo/dir1 ls /mnt/jindo/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/jindo/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/jindo/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/jindo/dir1/ \u5378\u8f7d JindoFuse \u00a7 \u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point> \u81ea\u52a8\u5378\u8f7d JindoFuse \u00a7 \u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u9ad8\u9636\u4f7f\u7528 \u00a7 \u914d\u7f6e\u9009\u9879 \u00a7 \u66f4\u591a jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<jindo_path>] -ofs.jindofsx.data.cache.enable=false","title":"JindoFuse \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#jindofuse","text":"\u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002JindoFuse \u4f7f\u7528\u7edf\u4e00\u6302\u8f7d\u7684\u65b9\u5f0f (jindo://) \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c\u8be5\u670d\u52a1\u4e0a\u7684\u6587\u4ef6\uff0c\u540c\u65f6\u901a\u8fc7\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002","title":"JindoFuse \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#_1","text":"","title":"\u57fa\u672c\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#_2","text":"# CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev","title":"\u73af\u5883\u51c6\u5907"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#_3","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#_4","text":"\u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-filestore] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = cn-xxx.oss-dls.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002","title":"\u914d\u7f6e\u670d\u52a1\u7aef"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#jindofsx","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#_5","text":"","title":"\u914d\u7f6e\u5ba2\u6237\u7aef"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#_6","text":"\u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf","title":"\u914d\u7f6e\u76ee\u5f55"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#_7","text":"\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u914d\u7f6e\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3acn-hangzhou.oss-dls.aliyuncs.com\u3002 # \u6216\u914d\u7f6e\u963f\u91cc\u4e91 OSS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou-internal.aliyuncs.com\u3002 fs.oss.endpoint= <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1\u7684 AccessKey ID \u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7 AccessKey \u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> # \u914d\u7f6eNamespace service \u5730\u5740 fs.jindofsx.namespace.rpc.address = <hostname>:<port> # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable = false","title":"\u914d\u7f6e\u6587\u4ef6"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#oss-oss-hdfs","text":"\u6302\u8f7d\u547d\u4ee4 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>/","title":"\u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#jindofuse_1","text":"\u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002","title":"\u6302\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#_8","text":"mkdir -p <mount_point>","title":"\u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#fuse","text":"jindo-fuse <mount_point> -ouri=[<jindo_path>] -oxengine=jindofsx -ouri \u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684 jindo \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a\u7edf\u4e00\u6302\u8f7d\uff0c\u6bd4\u5982\u4e4b\u524d\u6302\u8f7d\u7684 jindo:///jindooss \u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002","title":"\u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#jindofuse_2","text":"\u5982\u679c\u5c06 jindo \u8def\u5f84\uff08jindo://\uff09\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/jindo/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/jindo/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/jindo/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/jindo/dir1 ls /mnt/jindo/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/jindo/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/jindo/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/jindo/dir1/","title":"\u8bbf\u95ee JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#jindofuse_3","text":"\u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point>","title":"\u5378\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#jindofuse_4","text":"\u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002","title":"\u81ea\u52a8\u5378\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#_9","text":"","title":"\u9ad8\u9636\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_jindo/#_10","text":"\u66f4\u591a jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<jindo_path>] -ofs.jindofsx.data.cache.enable=false","title":"\u914d\u7f6e\u9009\u9879"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/","text":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f NAS \u00a7 JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002JindoFuse \u4f7f\u7528 NAS \u7684\u5730\u5740 (local://) \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c\u8be5\u670d\u52a1\u4e0a\u7684\u6587\u4ef6\uff0c\u540c\u65f6\u901a\u8fc7\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002 \u57fa\u672c\u4f7f\u7528 \u00a7 \u73af\u5883\u51c6\u5907 \u00a7 # CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u96c6\u7fa4\u5df2\u6302\u8f7d NAS \u6587\u4ef6\u7cfb\u7edf \u6302\u8f7d\u6b65\u9aa4\u8bf7\u53c2\u8003 Linux\u7cfb\u7edf\u6302\u8f7dNFS\u6587\u4ef6\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u914d\u7f6e\u5ba2\u6237\u7aef \u00a7 \u914d\u7f6e\u76ee\u5f55 \u00a7 \u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf \u914d\u7f6e\u6587\u4ef6 \u00a7 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u914d\u7f6eNamespace service \u5730\u5740 fs.jindofsx.namespace.rpc.address = <hostname>:<port> # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable = false \u6302\u8f7d JindoFuse \u00a7 \u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002 \u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 mkdir -p <mount_point> \u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 jindo-fuse <mount_point> -ouri=[<nas_path>] -oxengine=jindofsx -ouri \u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684 nas \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a nas \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\uff0c\u6bd4\u5982 local:///tmp/nas/ \u3002\u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002 \u8bbf\u95ee JindoFuse \u00a7 \u5982\u679c\u5c06 nas \u8def\u5f84\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/nas/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/nas/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/nas/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/nas/dir1 ls /mnt/nas/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/nas/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/nas/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/nas/dir1/ \u5378\u8f7d JindoFuse \u00a7 \u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point> \u81ea\u52a8\u5378\u8f7d JindoFuse \u00a7 \u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u9ad8\u9636\u4f7f\u7528 \u00a7 \u914d\u7f6e\u9009\u9879 \u00a7 \u66f4\u591a jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<nas_path>] -ofs.jindofsx.data.cache.enable=false","title":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f NAS"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#jindofuse-nas","text":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002JindoFuse \u4f7f\u7528 NAS \u7684\u5730\u5740 (local://) \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c\u8be5\u670d\u52a1\u4e0a\u7684\u6587\u4ef6\uff0c\u540c\u65f6\u901a\u8fc7\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002","title":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f NAS"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#_1","text":"","title":"\u57fa\u672c\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#_2","text":"# CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev","title":"\u73af\u5883\u51c6\u5907"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#_3","text":"\u96c6\u7fa4\u5df2\u6302\u8f7d NAS \u6587\u4ef6\u7cfb\u7edf \u6302\u8f7d\u6b65\u9aa4\u8bf7\u53c2\u8003 Linux\u7cfb\u7edf\u6302\u8f7dNFS\u6587\u4ef6\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#_4","text":"","title":"\u914d\u7f6e\u5ba2\u6237\u7aef"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#_5","text":"\u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf","title":"\u914d\u7f6e\u76ee\u5f55"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#_6","text":"\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u914d\u7f6eNamespace service \u5730\u5740 fs.jindofsx.namespace.rpc.address = <hostname>:<port> # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable = false","title":"\u914d\u7f6e\u6587\u4ef6"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#jindofuse","text":"\u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002","title":"\u6302\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#_7","text":"mkdir -p <mount_point>","title":"\u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#fuse","text":"jindo-fuse <mount_point> -ouri=[<nas_path>] -oxengine=jindofsx -ouri \u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684 nas \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a nas \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\uff0c\u6bd4\u5982 local:///tmp/nas/ \u3002\u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002","title":"\u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#jindofuse_1","text":"\u5982\u679c\u5c06 nas \u8def\u5f84\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/nas/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/nas/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/nas/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/nas/dir1 ls /mnt/nas/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/nas/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/nas/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/nas/dir1/","title":"\u8bbf\u95ee JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#jindofuse_2","text":"\u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point>","title":"\u5378\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#jindofuse_3","text":"\u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002","title":"\u81ea\u52a8\u5378\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#_8","text":"","title":"\u9ad8\u9636\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_nas/#_9","text":"\u66f4\u591a jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<nas_path>] -ofs.jindofsx.data.cache.enable=false","title":"\u914d\u7f6e\u9009\u9879"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/","text":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f OSS \u00a7 JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002JindoFuse \u4f7f\u7528\u539f\u6765\u7684\u5730\u5740 (oss://) \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c\u8be5\u670d\u52a1\u4e0a\u7684\u6587\u4ef6\uff0c\u540c\u65f6\u901a\u8fc7\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002 \u57fa\u672c\u4f7f\u7528 \u00a7 \u73af\u5883\u51c6\u5907 \u00a7 # CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u914d\u7f6e\u670d\u52a1\u7aef \u00a7 \u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-filestore] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = cn-xxx.oss-dls.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e\u5ba2\u6237\u7aef \u00a7 \u914d\u7f6e\u76ee\u5f55 \u00a7 \u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf \u914d\u7f6e\u6587\u4ef6 \u00a7 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u6216\u914d\u7f6e\u963f\u91cc\u4e91 OSS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou-internal.aliyuncs.com\u3002 fs.oss.endpoint= <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1\u7684 AccessKey ID \u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7 AccessKey \u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> # \u914d\u7f6eNamespace service \u5730\u5740 fs.jindofsx.namespace.rpc.address = <hostname>:<port> # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable = false \u6302\u8f7d JindoFuse \u00a7 \u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002 \u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 mkdir -p <mount_point> \u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 jindo-fuse <mount_point> -ouri=[<oss_path>] -oxengine=jindofsx -ouri \u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684 oss \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a Bucket \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002 \u8bbf\u95ee JindoFuse \u00a7 \u5982\u679c\u5c06 OSS-HDFS \u670d\u52a1\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1 ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/ \u5378\u8f7d JindoFuse \u00a7 \u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point> \u81ea\u52a8\u5378\u8f7d JindoFuse \u00a7 \u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u9ad8\u9636\u4f7f\u7528 \u00a7 \u914d\u7f6e\u9009\u9879 \u00a7 \u66f4\u591a jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<oss_path>] -ofs.jindofsx.data.cache.enable=false","title":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f OSS"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#jindofuse-oss","text":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f\u53ef\u4ee5\u5229\u7528\u8ba1\u7b97\u96c6\u7fa4\u7684\u95f2\u7f6e\u5b58\u50a8\u8d44\u6e90\u8fdb\u884c\u6570\u636e\u7f13\u5b58\u6765\u52a0\u901f\u8ba1\u7b97\u670d\u52a1\uff0c\u907f\u514d\u4e86\u8ba1\u7b97\u96c6\u7fa4/\u670d\u52a1\u5360\u7528\u6838\u5fc3\u96c6\u7fa4\u8fc7\u591a\u5e26\u5bbd\u3002JindoFuse \u4f7f\u7528\u539f\u6765\u7684\u5730\u5740 (oss://) \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c\u8be5\u670d\u52a1\u4e0a\u7684\u6587\u4ef6\uff0c\u540c\u65f6\u901a\u8fc7\u7f13\u5b58\u52a0\u901f\u8bbf\u95ee\u901f\u5ea6\u3002","title":"JindoFuse \u900f\u660e\u7f13\u5b58\u52a0\u901f OSS"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#_1","text":"","title":"\u57fa\u672c\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#_2","text":"# CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev","title":"\u73af\u5883\u51c6\u5907"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#_3","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#_4","text":"\u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-filestore] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = cn-xxx.oss-dls.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002","title":"\u914d\u7f6e\u670d\u52a1\u7aef"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#jindofsx","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#_5","text":"","title":"\u914d\u7f6e\u5ba2\u6237\u7aef"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#_6","text":"\u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf","title":"\u914d\u7f6e\u76ee\u5f55"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#_7","text":"\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u6216\u914d\u7f6e\u963f\u91cc\u4e91 OSS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou-internal.aliyuncs.com\u3002 fs.oss.endpoint= <your_endpoint> # \u7528\u4e8e\u8bbf\u95ee OSS \u6216 OSS-HDFS \u670d\u52a1\u7684 AccessKey ID \u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7 AccessKey \u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> # \u914d\u7f6eNamespace service \u5730\u5740 fs.jindofsx.namespace.rpc.address = <hostname>:<port> # \u6570\u636e\u7f13\u5b58\u5f00\u5173 fs.jindofsx.data.cache.enable = false","title":"\u914d\u7f6e\u6587\u4ef6"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#jindofuse","text":"\u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002","title":"\u6302\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#_8","text":"mkdir -p <mount_point>","title":"\u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#fuse","text":"jindo-fuse <mount_point> -ouri=[<oss_path>] -oxengine=jindofsx -ouri \u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684 oss \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a Bucket \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002","title":"\u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#jindofuse_1","text":"\u5982\u679c\u5c06 OSS-HDFS \u670d\u52a1\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1 ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/","title":"\u8bbf\u95ee JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#jindofuse_2","text":"\u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point>","title":"\u5378\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#jindofuse_3","text":"\u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002","title":"\u81ea\u52a8\u5378\u8f7d JindoFuse"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#_9","text":"","title":"\u9ad8\u9636\u4f7f\u7528"},{"location":"backup/jindofsx/jindo_fuse/jindo_fuse_on_oss/#_10","text":"\u66f4\u591a jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindofsx \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<oss_path>] -ofs.jindofsx.data.cache.enable=false","title":"\u914d\u7f6e\u9009\u9879"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u900f\u660e\u7f13\u5b58\u52a0\u901f \u00a7 \u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u900f\u660e\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u670d\u52a1\u7aef\u914d\u7f6e OSS-HDFS \u670d\u52a1 AccessKey \u00a7 \u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS-HDFS \u670d\u52a1 bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = cn-xxx.oss-dls.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = cn-xxx.oss-dls.aliyuncs.com jindofsx.oss.bucket.YYY.user = xxx #Storage Service \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u4f7f\u7528\u7684\u7528\u6237\u540d \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS-HDFS \u670d\u52a1 bucket \u540d\u79f0\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS-HDFS \u670d\u52a1 bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06 OSS-HDFS \u670d\u52a1 \u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u914d\u7f6e\u7f13\u5b58\u52a0\u901f \u00a7 \u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u900f\u660e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs/#oss-hdfs-jindofs","text":"\u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u900f\u660e\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u900f\u660e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs/#oss-hdfs-accesskey","text":"\u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS-HDFS \u670d\u52a1 bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = cn-xxx.oss-dls.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = cn-xxx.oss-dls.aliyuncs.com jindofsx.oss.bucket.YYY.user = xxx #Storage Service \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u4f7f\u7528\u7684\u7528\u6237\u540d \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS-HDFS \u670d\u52a1 bucket \u540d\u79f0\u3002","title":"\u670d\u52a1\u7aef\u914d\u7f6e OSS-HDFS \u670d\u52a1 AccessKey"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs/#jindofsx","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS-HDFS \u670d\u52a1 bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs/#jindosdk","text":"\u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06 OSS-HDFS \u670d\u52a1 \u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs/#_2","text":"\u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e","title":"\u914d\u7f6e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs/#_3","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs_jindo/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f \u00a7 \u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u670d\u52a1\u7aef\u914d\u7f6e OSS-HDFS \u670d\u52a1 AccessKey \u00a7 \u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS-HDFS \u670d\u52a1 bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = cn-xxx.oss-dls.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.user = xxx #Storage Service \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u4f7f\u7528\u7684\u7528\u6237\u540d \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS-HDFS \u670d\u52a1 bucket \u7684\u540d\u79f0\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS-HDFS \u670d\u52a1 bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AccessKey \u5c06 OSS-HDFS \u670d\u52a1 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002\u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u6302\u8f7d OSS-HDFS \u670d\u52a1 \u00a7 \u6302\u8f7d OSS-HDFS \u670d\u52a1\u76ee\u5f55 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindodls oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219/jindodls \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/ [root@emr-header-1 ~]# hdfs dfs -ls jindo://emr-header-1:8101/ Found 1 items ---------- 1 0 1970-01-01 08:00 jindo://emr-header-1:8101/jindodls \u5373\u8bbf\u95ee jindo://emr-header-1:8101/jindodls/ \u7b49\u4ef7\u4e8e\u8bbf\u95ee oss://<Bucket>.<Endpoint>/ \u914d\u7f6e\u7f13\u5b58\u52a0\u901f \u00a7 \u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u00a7 \u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u5728\u6570\u636e\u7f13\u5b58\u5f00\u5173\u6253\u5f00\u65f6\u901a\u8fc7 jindo:// \u524d\u7f00\u8bfb\u53d6 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 jindo:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002 \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs_jindo/#oss-hdfs-jindofs","text":"\u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs_jindo/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs_jindo/#oss-hdfs-accesskey","text":"\u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS-HDFS \u670d\u52a1 bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = cn-xxx.oss-dls.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.user = xxx #Storage Service \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u4f7f\u7528\u7684\u7528\u6237\u540d \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS-HDFS \u670d\u52a1 bucket \u7684\u540d\u79f0\u3002","title":"\u670d\u52a1\u7aef\u914d\u7f6e OSS-HDFS \u670d\u52a1 AccessKey"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs_jindo/#jindofsx","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS-HDFS \u670d\u52a1 bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs_jindo/#jindosdk","text":"\u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 AccessKey \u5c06 OSS-HDFS \u670d\u52a1 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002\u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs_jindo/#oss-hdfs","text":"\u6302\u8f7d OSS-HDFS \u670d\u52a1\u76ee\u5f55 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindodls oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219/jindodls \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/ [root@emr-header-1 ~]# hdfs dfs -ls jindo://emr-header-1:8101/ Found 1 items ---------- 1 0 1970-01-01 08:00 jindo://emr-header-1:8101/jindodls \u5373\u8bbf\u95ee jindo://emr-header-1:8101/jindodls/ \u7b49\u4ef7\u4e8e\u8bbf\u95ee oss://<Bucket>.<Endpoint>/","title":"\u6302\u8f7d OSS-HDFS \u670d\u52a1"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs_jindo/#_2","text":"\u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e","title":"\u914d\u7f6e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs_jindo/#oss-hdfs_1","text":"\u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u5728\u6570\u636e\u7f13\u5b58\u5f00\u5173\u6253\u5f00\u65f6\u901a\u8fc7 jindo:// \u524d\u7f00\u8bfb\u53d6 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 jindo:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002","title":"\u8bbf\u95ee OSS-HDFS \u670d\u52a1"},{"location":"backup/jindofsx/jindofs/jindofsx_on_jindofs_jindo/#_3","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/jindotable/table_cache/","text":"\u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e \u00a7 JindoTable \u53ef\u4ee5\u4f7f\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e\uff0c\u4ece\u800c\u63d0\u9ad8\u8bbf\u95ee\u901f\u5ea6\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u00a7 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u00a7 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883 \u00a7 \u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e cache \u547d\u4ee4 \u00a7 \u529f\u80fd \u00a7 \u8868\u793a\u7f13\u5b58\u6307\u5b9a\u8868\u6216\u5206\u533a\u7684\u6570\u636e\u81f3\u96c6\u7fa4\u672c\u5730\u78c1\u76d8\u4e0a\u3002 \u8868\u6216\u5206\u533a\u7684\u8def\u5f84\u9700\u8981\u4f4d\u4e8e OSS \u6216 OSS-HDFS \u670d\u52a1\u3002 \u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f \u00a7 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help cache \u53c2\u6570\u8bf4\u660e \u00a7 jindotable -cache [-pin] [-s] [-l] [-m] [-r replica] [-b batchsize] -t <dbName.tableName> [-p <partitionSpec>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u6307\u5b9a\u8981\u7f13\u5b58\u7684\u8868 \u662f -p <partitionSpec> \u6307\u5b9a\u8981\u7f13\u5b58\u7684\u5206\u533a\uff0c\u4f7f\u7528partitionCol1=1,partitionCol2=2,...\u7684\u683c\u5f0f \u5426 -pin \u5728\u7f13\u5b58\u7a7a\u95f4\u4e0d\u8db3\u65f6\u5c3d\u91cf\u4e0d\u5220\u9664\u76f8\u5173\u6570\u636e\u3002 \u5426 -s \u662f\u5426\u540c\u6b65 \u5426 -l \u662f\u5426\u662f\u5c0f\u6587\u4ef6 \u5426 -m \u662f\u5426\u5728\u5185\u5b58\u7f13\u5b58 \u5426 -r replica \u7f13\u5b58\u7684\u526f\u672c\u6570 \u5426 -b batchsize \u7f13\u5b58\u7684\u5e76\u53d1\u6570 \u5426 \u793a\u4f8b \u00a7 \u7f13\u5b582020-03-16\u65e5 db1.t1 \u8868\u7684\u6570\u636e\u81f3\u672c\u5730\u78c1\u76d8\u4e0a\u3002 jindotable -cache -t db1.t1 -p date=2020-03-16 uncache \u547d\u4ee4 \u00a7 \u529f\u80fd \u00a7 \u8868\u793a\u5220\u9664\u96c6\u7fa4\u672c\u5730\u78c1\u76d8\u4e0a\u6307\u5b9a\u8868\u6216\u5206\u533a\u7684\u7f13\u5b58\u6570\u636e\u3002 \u5bf9\u5e94\u7684\u8def\u5f84\u9700\u8981\u4f4d\u4e8e OSS \u6216 OSS-HDFS \u670d\u52a1\u3002 \u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f \u00a7 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help uncache \u53c2\u6570\u8bf4\u660e \u00a7 jindotable -uncache [-s] -t <dbName.tableName> [-p <partitionSpec>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u6307\u5b9a\u8981\u5220\u9664\u7f13\u5b58\u7684\u8868 \u662f -p <partitionSpec> \u6307\u5b9a\u8981\u5220\u9664\u7f13\u5b58\u7684\u5206\u533a\uff0c\u4f7f\u7528partitionCol1=1,partitionCol2=2,...\u7684\u683c\u5f0f \u5426 \u793a\u4f8b \u00a7 \u5220\u9664\u96c6\u7fa4\u672c\u5730\u78c1\u76d8\u4e0a\u8868 db1.t2 \u7684\u7f13\u5b58\u6570\u636e\u3002 jindotable -uncache -t db1.t2 \u5220\u9664\u96c6\u7fa4\u672c\u5730\u78c1\u76d8\u4e0a\u8868 db1.t1 \u4e2d\u6307\u5b9a\u5206\u533a\u7684\u7f13\u5b58\u6570\u636e\u3002 jindotable -uncache -t db1.t1 -p date=2020-03-16,category=1","title":"\u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e"},{"location":"backup/jindofsx/jindotable/table_cache/#jindotable-hive","text":"JindoTable \u53ef\u4ee5\u4f7f\u7528 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e\uff0c\u4ece\u800c\u63d0\u9ad8\u8bbf\u95ee\u901f\u5ea6\u3002","title":"\u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e"},{"location":"backup/jindofsx/jindotable/table_cache/#_1","text":"","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/jindofsx/jindotable/table_cache/#jindofsx","text":"\u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf","title":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf"},{"location":"backup/jindofsx/jindotable/table_cache/#jindosdk","text":"\u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u5df2\u90e8\u7f72 JindoSDK"},{"location":"backup/jindofsx/jindotable/table_cache/#hadoop-hive","text":"\u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e","title":"\u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883"},{"location":"backup/jindofsx/jindotable/table_cache/#cache","text":"","title":"cache \u547d\u4ee4"},{"location":"backup/jindofsx/jindotable/table_cache/#_2","text":"\u8868\u793a\u7f13\u5b58\u6307\u5b9a\u8868\u6216\u5206\u533a\u7684\u6570\u636e\u81f3\u96c6\u7fa4\u672c\u5730\u78c1\u76d8\u4e0a\u3002 \u8868\u6216\u5206\u533a\u7684\u8def\u5f84\u9700\u8981\u4f4d\u4e8e OSS \u6216 OSS-HDFS \u670d\u52a1\u3002","title":"\u529f\u80fd"},{"location":"backup/jindofsx/jindotable/table_cache/#_3","text":"\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help cache","title":"\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f"},{"location":"backup/jindofsx/jindotable/table_cache/#_4","text":"jindotable -cache [-pin] [-s] [-l] [-m] [-r replica] [-b batchsize] -t <dbName.tableName> [-p <partitionSpec>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u6307\u5b9a\u8981\u7f13\u5b58\u7684\u8868 \u662f -p <partitionSpec> \u6307\u5b9a\u8981\u7f13\u5b58\u7684\u5206\u533a\uff0c\u4f7f\u7528partitionCol1=1,partitionCol2=2,...\u7684\u683c\u5f0f \u5426 -pin \u5728\u7f13\u5b58\u7a7a\u95f4\u4e0d\u8db3\u65f6\u5c3d\u91cf\u4e0d\u5220\u9664\u76f8\u5173\u6570\u636e\u3002 \u5426 -s \u662f\u5426\u540c\u6b65 \u5426 -l \u662f\u5426\u662f\u5c0f\u6587\u4ef6 \u5426 -m \u662f\u5426\u5728\u5185\u5b58\u7f13\u5b58 \u5426 -r replica \u7f13\u5b58\u7684\u526f\u672c\u6570 \u5426 -b batchsize \u7f13\u5b58\u7684\u5e76\u53d1\u6570 \u5426","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"backup/jindofsx/jindotable/table_cache/#_5","text":"\u7f13\u5b582020-03-16\u65e5 db1.t1 \u8868\u7684\u6570\u636e\u81f3\u672c\u5730\u78c1\u76d8\u4e0a\u3002 jindotable -cache -t db1.t1 -p date=2020-03-16","title":"\u793a\u4f8b"},{"location":"backup/jindofsx/jindotable/table_cache/#uncache","text":"","title":"uncache \u547d\u4ee4"},{"location":"backup/jindofsx/jindotable/table_cache/#_6","text":"\u8868\u793a\u5220\u9664\u96c6\u7fa4\u672c\u5730\u78c1\u76d8\u4e0a\u6307\u5b9a\u8868\u6216\u5206\u533a\u7684\u7f13\u5b58\u6570\u636e\u3002 \u5bf9\u5e94\u7684\u8def\u5f84\u9700\u8981\u4f4d\u4e8e OSS \u6216 OSS-HDFS \u670d\u52a1\u3002","title":"\u529f\u80fd"},{"location":"backup/jindofsx/jindotable/table_cache/#_7","text":"\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help uncache","title":"\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f"},{"location":"backup/jindofsx/jindotable/table_cache/#_8","text":"jindotable -uncache [-s] -t <dbName.tableName> [-p <partitionSpec>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u6307\u5b9a\u8981\u5220\u9664\u7f13\u5b58\u7684\u8868 \u662f -p <partitionSpec> \u6307\u5b9a\u8981\u5220\u9664\u7f13\u5b58\u7684\u5206\u533a\uff0c\u4f7f\u7528partitionCol1=1,partitionCol2=2,...\u7684\u683c\u5f0f \u5426","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"backup/jindofsx/jindotable/table_cache/#_9","text":"\u5220\u9664\u96c6\u7fa4\u672c\u5730\u78c1\u76d8\u4e0a\u8868 db1.t2 \u7684\u7f13\u5b58\u6570\u636e\u3002 jindotable -uncache -t db1.t2 \u5220\u9664\u96c6\u7fa4\u672c\u5730\u78c1\u76d8\u4e0a\u8868 db1.t1 \u4e2d\u6307\u5b9a\u5206\u533a\u7684\u7f13\u5b58\u6570\u636e\u3002 jindotable -uncache -t db1.t1 -p date=2020-03-16,category=1","title":"\u793a\u4f8b"},{"location":"backup/jindofsx/multi-cloud/jindofsx_on_multi_cloud/","text":"S3\u3001COS\u3001OBS \u591a\u4e91\u5b58\u50a8\u652f\u6301 \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u4e86\u900f\u660e\u7f13\u5b58\u548c\u7edf\u4e00\u6302\u8f7d\u547d\u540d\u7a7a\u95f4\u7684\u4f7f\u7528\u65b9\u5f0f\uff0c\u652f\u6301\u591a\u79cd scheme \u7684\u4e91\u5b58\u50a8\u3002\u6587\u4ef6\u90fd\u4ee5\u5bf9\u8c61\u7684\u5f62\u5f0f\u5b58\u50a8\u5728\u4e91\u5b58\u50a8\u4e0a\uff0c\u6bcf\u4e2a\u6587\u4ef6\u6839\u636e\u5b9e\u9645\u8bbf\u95ee\u60c5\u51b5\u4f1a\u5728\u672c\u5730\u8fdb\u884c\u7f13\u5b58\uff0c\u63d0\u5347\u8bbf\u95ee\u4e91\u5b58\u50a8\u7684\u6548\u7387\uff0c\u540c\u65f6\u517c\u5bb9\u4e86\u539f\u6709\u4e91\u5b58\u50a8\u6587\u4ef6\u5f62\u5f0f\uff0c\u6570\u636e\u8bbf\u95ee\u4e0a\u80fd\u591f\u4e0e\u5176\u4ed6\u4e91\u5b58\u50a8\u5ba2\u6237\u7aef\u5b8c\u5168\u517c\u5bb9\uff0c\u4f5c\u4e1a\u8bbf\u95ee\u4e91\u5b58\u50a8\u7684\u65b9\u5f0f\u65e0\u9700\u505a\u4efb\u4f55\u4fee\u6539\u3002 \u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301 S3\u3001COS\u3001OBS \u900f\u660e\u7f13\u5b58\u52a0\u901f\u548c\u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u670d\u52a1\u7aef\u914d\u7f6e\u591a\u4e91 AccessKey \u00a7 \u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] jindofsx.s3.bucket.XXX.accessKeyId = xxx jindofsx.s3.bucket.XXX.accessKeySecret = xxx jindofsx.s3.bucket.XXX.endpoint = s3.xxx.amazonaws.com jindofsx.cos.bucket.YYY.accessKeyId = xxx jindofsx.cos.bucket.YYY.accessKeySecret = xxx jindofsx.cos.bucket.YYY.endpoint = cos.xxx.myqcloud.com jindofsx.obs.bucket.ZZZ.accessKeyId = xxx jindofsx.obs.bucket.ZZZ.accessKeySecret = xxx jindofsx.obs.bucket.ZZZ.endpoint = obs.xxx.myhuaweicloud.com \u8bf4\u660e: XXX\u3001YYY\u3001ZZZ \u5206\u522b\u4e3a S3 bucket\u3001COS bucket\u3001OBS bucket \u7684\u540d\u79f0\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u591a\u4e91\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1\u591a\u4e91\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> <!-- S3 --> <property> <name>fs.AbstractFileSystem.s3.impl</name> <value>com.aliyun.jindodata.s3.S3</value> </property> <property> <name>fs.s3.impl</name> <value>com.aliyun.jindodata.s3.JindoS3FileSystem</value> </property> <!-- COS --> <property> <name>fs.AbstractFileSystem.cos.impl</name> <value>com.aliyun.jindodata.cos.Cos</value> </property> <property> <name>fs.cos.impl</name> <value>com.aliyun.jindodata.cos.JindoCosFileSystem</value> </property> <!-- OBS --> <property> <name>fs.AbstractFileSystem.obs.impl</name> <value>com.aliyun.jindodata.obs.OBS</value> </property> <property> <name>fs.obs.impl</name> <value>com.aliyun.jindodata.obs.JindoObsFileSystem</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06\u591a\u4e91\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <!-- S3 --> <property> <name>fs.s3.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.s3.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.s3.endpoint</name> <value>xxx</value> </property> <!-- COS --> <property> <name>fs.cos.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.cos.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.cos.endpoint</name> <value>xxx</value> </property> <!-- OBS --> <property> <name>fs.obs.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.obs.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.obs.endpoint</name> <value>xxx</value> </property> </configuration> \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u6302\u8f7d\u591a\u4e91\u5b58\u50a8 \u00a7 \u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u4ee5\u6302\u8f7d AWS S3 \u76ee\u5f55\u4e3a\u4f8b\uff0ccos\u548cobs\u4f7f\u7528\u65b9\u6cd5\u7c7b\u4f3c\u3002 * \u6302\u8f7d S3 \u76ee\u5f55 jindo admin -mount <path> <realpath> \u4f8b\uff1a jindo admin -mount /jindos3 s3://<Bucket>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindos3 \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f s3://<Bucket>/ [root@emr-header-1 ~]# hdfs dfs -ls jindo://emr-header-1:8101/ Found 1 items ---------- 1 0 1970-01-01 08:00 jindo://emr-header-1:8101/jindos3 \u5373\u8bbf\u95ee jindo://emr-header-1:8101/jindos3/ \u7b49\u4ef7\u4e8e\u8bbf\u95ee s3://<Bucket>/ \u914d\u7f6e\u7f13\u5b58\u52a0\u901f \u00a7 \u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u8bbf\u95ee AWS S3 \u00a7 \u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u5728\u6570\u636e\u7f13\u5b58\u5f00\u5173\u6253\u5f00\u65f6\u901a\u8fc7 jindo:// \u524d\u7f00\u8bfb\u53d6 S3 \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 jindo:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002 \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"S3\u3001COS\u3001OBS \u591a\u4e91\u5b58\u50a8\u652f\u6301"},{"location":"backup/jindofsx/multi-cloud/jindofsx_on_multi_cloud/#s3cosobs","text":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u4e86\u900f\u660e\u7f13\u5b58\u548c\u7edf\u4e00\u6302\u8f7d\u547d\u540d\u7a7a\u95f4\u7684\u4f7f\u7528\u65b9\u5f0f\uff0c\u652f\u6301\u591a\u79cd scheme \u7684\u4e91\u5b58\u50a8\u3002\u6587\u4ef6\u90fd\u4ee5\u5bf9\u8c61\u7684\u5f62\u5f0f\u5b58\u50a8\u5728\u4e91\u5b58\u50a8\u4e0a\uff0c\u6bcf\u4e2a\u6587\u4ef6\u6839\u636e\u5b9e\u9645\u8bbf\u95ee\u60c5\u51b5\u4f1a\u5728\u672c\u5730\u8fdb\u884c\u7f13\u5b58\uff0c\u63d0\u5347\u8bbf\u95ee\u4e91\u5b58\u50a8\u7684\u6548\u7387\uff0c\u540c\u65f6\u517c\u5bb9\u4e86\u539f\u6709\u4e91\u5b58\u50a8\u6587\u4ef6\u5f62\u5f0f\uff0c\u6570\u636e\u8bbf\u95ee\u4e0a\u80fd\u591f\u4e0e\u5176\u4ed6\u4e91\u5b58\u50a8\u5ba2\u6237\u7aef\u5b8c\u5168\u517c\u5bb9\uff0c\u4f5c\u4e1a\u8bbf\u95ee\u4e91\u5b58\u50a8\u7684\u65b9\u5f0f\u65e0\u9700\u505a\u4efb\u4f55\u4fee\u6539\u3002 \u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301 S3\u3001COS\u3001OBS \u900f\u660e\u7f13\u5b58\u52a0\u901f\u548c\u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002","title":"S3\u3001COS\u3001OBS \u591a\u4e91\u5b58\u50a8\u652f\u6301"},{"location":"backup/jindofsx/multi-cloud/jindofsx_on_multi_cloud/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/multi-cloud/jindofsx_on_multi_cloud/#accesskey","text":"\u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] jindofsx.s3.bucket.XXX.accessKeyId = xxx jindofsx.s3.bucket.XXX.accessKeySecret = xxx jindofsx.s3.bucket.XXX.endpoint = s3.xxx.amazonaws.com jindofsx.cos.bucket.YYY.accessKeyId = xxx jindofsx.cos.bucket.YYY.accessKeySecret = xxx jindofsx.cos.bucket.YYY.endpoint = cos.xxx.myqcloud.com jindofsx.obs.bucket.ZZZ.accessKeyId = xxx jindofsx.obs.bucket.ZZZ.accessKeySecret = xxx jindofsx.obs.bucket.ZZZ.endpoint = obs.xxx.myhuaweicloud.com \u8bf4\u660e: XXX\u3001YYY\u3001ZZZ \u5206\u522b\u4e3a S3 bucket\u3001COS bucket\u3001OBS bucket \u7684\u540d\u79f0\u3002","title":"\u670d\u52a1\u7aef\u914d\u7f6e\u591a\u4e91 AccessKey"},{"location":"backup/jindofsx/multi-cloud/jindofsx_on_multi_cloud/#jindofsx","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/multi-cloud/jindofsx_on_multi_cloud/#jindosdk","text":"\u914d\u7f6e\u591a\u4e91\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1\u591a\u4e91\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> <!-- S3 --> <property> <name>fs.AbstractFileSystem.s3.impl</name> <value>com.aliyun.jindodata.s3.S3</value> </property> <property> <name>fs.s3.impl</name> <value>com.aliyun.jindodata.s3.JindoS3FileSystem</value> </property> <!-- COS --> <property> <name>fs.AbstractFileSystem.cos.impl</name> <value>com.aliyun.jindodata.cos.Cos</value> </property> <property> <name>fs.cos.impl</name> <value>com.aliyun.jindodata.cos.JindoCosFileSystem</value> </property> <!-- OBS --> <property> <name>fs.AbstractFileSystem.obs.impl</name> <value>com.aliyun.jindodata.obs.OBS</value> </property> <property> <name>fs.obs.impl</name> <value>com.aliyun.jindodata.obs.JindoObsFileSystem</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06\u591a\u4e91\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <!-- S3 --> <property> <name>fs.s3.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.s3.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.s3.endpoint</name> <value>xxx</value> </property> <!-- COS --> <property> <name>fs.cos.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.cos.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.cos.endpoint</name> <value>xxx</value> </property> <!-- OBS --> <property> <name>fs.obs.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.obs.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.obs.endpoint</name> <value>xxx</value> </property> </configuration> \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/multi-cloud/jindofsx_on_multi_cloud/#_2","text":"\u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u4ee5\u6302\u8f7d AWS S3 \u76ee\u5f55\u4e3a\u4f8b\uff0ccos\u548cobs\u4f7f\u7528\u65b9\u6cd5\u7c7b\u4f3c\u3002 * \u6302\u8f7d S3 \u76ee\u5f55 jindo admin -mount <path> <realpath> \u4f8b\uff1a jindo admin -mount /jindos3 s3://<Bucket>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindos3 \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f s3://<Bucket>/ [root@emr-header-1 ~]# hdfs dfs -ls jindo://emr-header-1:8101/ Found 1 items ---------- 1 0 1970-01-01 08:00 jindo://emr-header-1:8101/jindos3 \u5373\u8bbf\u95ee jindo://emr-header-1:8101/jindos3/ \u7b49\u4ef7\u4e8e\u8bbf\u95ee s3://<Bucket>/","title":"\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u6302\u8f7d\u591a\u4e91\u5b58\u50a8"},{"location":"backup/jindofsx/multi-cloud/jindofsx_on_multi_cloud/#_3","text":"\u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e","title":"\u914d\u7f6e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/multi-cloud/jindofsx_on_multi_cloud/#aws-s3","text":"\u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u5728\u6570\u636e\u7f13\u5b58\u5f00\u5173\u6253\u5f00\u65f6\u901a\u8fc7 jindo:// \u524d\u7f00\u8bfb\u53d6 S3 \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 jindo:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002","title":"\u8bbf\u95ee AWS S3"},{"location":"backup/jindofsx/multi-cloud/jindofsx_on_multi_cloud/#_4","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/nas/jindofsx_on_nas_jindo/","text":"\u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8 NAS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f \u00a7 \u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8NAS\uff08Apsara File Storage NAS\uff09\u662f\u4e00\u4e2a\u53ef\u5927\u89c4\u6a21\u5171\u4eab\u8bbf\u95ee\uff0c\u5f39\u6027\u6269\u5c55\u7684\u9ad8\u6027\u80fd\u4e91\u539f\u751f\u5206\u5e03\u5f0f\u6587\u4ef6\u7cfb\u7edf\u3002\u652f\u6301\u667a\u80fd\u51b7\u70ed\u6570\u636e\u5206\u5c42\uff0c\u6709\u6548\u964d\u4f4e\u6570\u636e\u5b58\u50a8\u6210\u672c\u3002\u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301\u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8 NAS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u96c6\u7fa4\u5df2\u6302\u8f7d NAS \u6587\u4ef6\u7cfb\u7edf \u6302\u8f7d\u6b65\u9aa4\u8bf7\u53c2\u8003 Linux\u7cfb\u7edf\u6302\u8f7dNFS\u6587\u4ef6\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u6302\u8f7d NAS \u6587\u4ef6\u7cfb\u7edf\u76ee\u5f55 \u00a7 jindo admin -mount <path> <realpath> \u4f8b\uff1a jindo admin -mount /jindonas local:///mnt/nas \u5047\u8bbe NAS \u6587\u4ef6\u7cfb\u7edf\u6302\u8f7d\u5728\u670d\u52a1\u5668\u4e0a\u7684\u672c\u5730\u8def\u5f84\u4e3a /mnt/nas , \u5728\u8fdb\u884c\u6302\u8f7d\u65f6\u9700\u8981\u52a0\u4e0a local:// \u524d\u7f00\u3002\u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindonas \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f /mnt/nas [root@emr-header-1 ~]# hdfs dfs -ls jindo://emr-header-1:8101/ Found 2 items ---------- 1 0 1970-01-01 08:00 jindo://emr-header-1:8101/jindonas \u5373\u8bbf\u95ee jindo://emr-header-1:8101/jindonas/ \u7b49\u4ef7\u4e8e\u8bbf\u95ee /mnt/nas/ \u914d\u7f6e\u7f13\u5b58\u52a0\u901f \u00a7 \u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u8bbf\u95ee NAS \u6587\u4ef6\u7cfb\u7edf \u00a7 \u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u5728\u6570\u636e\u7f13\u5b58\u5f00\u5173\u6253\u5f00\u65f6\u901a\u8fc7 jindo:// \u524d\u7f00\u8bfb\u53d6 NAS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 jindo:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002","title":"\u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8 NAS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/nas/jindofsx_on_nas_jindo/#nas","text":"\u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8NAS\uff08Apsara File Storage NAS\uff09\u662f\u4e00\u4e2a\u53ef\u5927\u89c4\u6a21\u5171\u4eab\u8bbf\u95ee\uff0c\u5f39\u6027\u6269\u5c55\u7684\u9ad8\u6027\u80fd\u4e91\u539f\u751f\u5206\u5e03\u5f0f\u6587\u4ef6\u7cfb\u7edf\u3002\u652f\u6301\u667a\u80fd\u51b7\u70ed\u6570\u636e\u5206\u5c42\uff0c\u6709\u6548\u964d\u4f4e\u6570\u636e\u5b58\u50a8\u6210\u672c\u3002\u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301\u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8 NAS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002","title":"\u963f\u91cc\u4e91\u6587\u4ef6\u5b58\u50a8 NAS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/nas/jindofsx_on_nas_jindo/#_1","text":"\u96c6\u7fa4\u5df2\u6302\u8f7d NAS \u6587\u4ef6\u7cfb\u7edf \u6302\u8f7d\u6b65\u9aa4\u8bf7\u53c2\u8003 Linux\u7cfb\u7edf\u6302\u8f7dNFS\u6587\u4ef6\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/nas/jindofsx_on_nas_jindo/#jindosdk","text":"\u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/nas/jindofsx_on_nas_jindo/#nas_1","text":"jindo admin -mount <path> <realpath> \u4f8b\uff1a jindo admin -mount /jindonas local:///mnt/nas \u5047\u8bbe NAS \u6587\u4ef6\u7cfb\u7edf\u6302\u8f7d\u5728\u670d\u52a1\u5668\u4e0a\u7684\u672c\u5730\u8def\u5f84\u4e3a /mnt/nas , \u5728\u8fdb\u884c\u6302\u8f7d\u65f6\u9700\u8981\u52a0\u4e0a local:// \u524d\u7f00\u3002\u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindonas \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f /mnt/nas [root@emr-header-1 ~]# hdfs dfs -ls jindo://emr-header-1:8101/ Found 2 items ---------- 1 0 1970-01-01 08:00 jindo://emr-header-1:8101/jindonas \u5373\u8bbf\u95ee jindo://emr-header-1:8101/jindonas/ \u7b49\u4ef7\u4e8e\u8bbf\u95ee /mnt/nas/","title":"\u6302\u8f7d NAS \u6587\u4ef6\u7cfb\u7edf\u76ee\u5f55"},{"location":"backup/jindofsx/nas/jindofsx_on_nas_jindo/#_2","text":"\u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e","title":"\u914d\u7f6e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/nas/jindofsx_on_nas_jindo/#nas_2","text":"\u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u5728\u6570\u636e\u7f13\u5b58\u5f00\u5173\u6253\u5f00\u65f6\u901a\u8fc7 jindo:// \u524d\u7f00\u8bfb\u53d6 NAS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 jindo:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002","title":"\u8bbf\u95ee NAS \u6587\u4ef6\u7cfb\u7edf"},{"location":"backup/jindofsx/oss/jindofsx_on_oss/","text":"\u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f \u00a7 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u4e86\u900f\u660e\u7f13\u5b58\u7684\u4f7f\u7528\u65b9\u5f0f\uff0c\u517c\u5bb9\u539f\u751f OSS \u5b58\u50a8\u65b9\u5f0f\uff0c\u6587\u4ef6\u4ee5\u5bf9\u8c61\u7684\u5f62\u5f0f\u5b58\u50a8\u5728 OSS \u4e0a\uff0c\u6bcf\u4e2a\u6587\u4ef6\u6839\u636e\u5b9e\u9645\u8bbf\u95ee\u60c5\u51b5\u4f1a\u5728\u672c\u5730\u8fdb\u884c\u7f13\u5b58\uff0c\u63d0\u5347\u8bbf\u95eeOSS\u7684\u6548\u7387\uff0c\u540c\u65f6\u517c\u5bb9\u4e86\u539f\u6709OSS\u539f\u6709\u6587\u4ef6\u5f62\u5f0f\uff0c\u6570\u636e\u8bbf\u95ee\u4e0a\u80fd\u591f\u4e0e\u5176\u4ed6OSS\u5ba2\u6237\u7aef\u5b8c\u5168\u517c\u5bb9\uff0c\u4f5c\u4e1a\u8bbf\u95ee OSS \u7684\u65b9\u5f0f\u65e0\u9700\u505a\u4efb\u4f55\u4fee\u6539\u3002 \u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u670d\u52a1\u7aef\u914d\u7f6e\u963f\u91cc\u4e91 OSS AccessKey \u00a7 \u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = oss-cn-xxx-internal.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1 OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06 OSS \u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u914d\u7f6e\u7f13\u5b58\u52a0\u901f \u00a7 \u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/oss/jindofsx_on_oss/#oss","text":"JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u63d0\u4f9b\u4e86\u900f\u660e\u7f13\u5b58\u7684\u4f7f\u7528\u65b9\u5f0f\uff0c\u517c\u5bb9\u539f\u751f OSS \u5b58\u50a8\u65b9\u5f0f\uff0c\u6587\u4ef6\u4ee5\u5bf9\u8c61\u7684\u5f62\u5f0f\u5b58\u50a8\u5728 OSS \u4e0a\uff0c\u6bcf\u4e2a\u6587\u4ef6\u6839\u636e\u5b9e\u9645\u8bbf\u95ee\u60c5\u51b5\u4f1a\u5728\u672c\u5730\u8fdb\u884c\u7f13\u5b58\uff0c\u63d0\u5347\u8bbf\u95eeOSS\u7684\u6548\u7387\uff0c\u540c\u65f6\u517c\u5bb9\u4e86\u539f\u6709OSS\u539f\u6709\u6587\u4ef6\u5f62\u5f0f\uff0c\u6570\u636e\u8bbf\u95ee\u4e0a\u80fd\u591f\u4e0e\u5176\u4ed6OSS\u5ba2\u6237\u7aef\u5b8c\u5168\u517c\u5bb9\uff0c\u4f5c\u4e1a\u8bbf\u95ee OSS \u7684\u65b9\u5f0f\u65e0\u9700\u505a\u4efb\u4f55\u4fee\u6539\u3002 \u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002","title":"\u963f\u91cc\u4e91 OSS \u900f\u660e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/oss/jindofsx_on_oss/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/oss/jindofsx_on_oss/#oss-accesskey","text":"\u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = oss-cn-xxx-internal.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002","title":"\u670d\u52a1\u7aef\u914d\u7f6e\u963f\u91cc\u4e91 OSS AccessKey"},{"location":"backup/jindofsx/oss/jindofsx_on_oss/#jindofsx","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/oss/jindofsx_on_oss/#jindosdk","text":"\u914d\u7f6e OSS \u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1 OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06 OSS \u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/oss/jindofsx_on_oss/#_2","text":"\u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e","title":"\u914d\u7f6e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/oss/jindofsx_on_oss/#_3","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/oss/jindofsx_on_oss_jindo/","text":"\u963f\u91cc\u4e91 OSS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f \u00a7 \u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002 \u524d\u63d0\u6761\u4ef6\uff1a \u00a7 \u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u670d\u52a1\u7aef\u914d\u7f6e\u963f\u91cc\u4e91 OSS AccessKey \u00a7 \u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = oss-cn-xxx-internal.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002 \u91cd\u542f JindoFSx \u670d\u52a1 \u00a7 \u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS AccessKey \u5c06 OSS Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u6302\u8f7d\u963f\u91cc\u4e91 OSS \u00a7 \u6302\u8f7d OSS \u76ee\u5f55 jindo admin -mount <path> <realpath> \u4f8b\uff1a jindo admin -mount /jindooss oss://<Bucket>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>/ [root@emr-header-1 ~]# hdfs dfs -ls jindo://emr-header-1:8101/ Found 1 items ---------- 1 0 1970-01-01 08:00 jindo://emr-header-1:8101/jindooss \u5373\u8bbf\u95ee jindo://emr-header-1:8101/jindooss/ \u7b49\u4ef7\u4e8e\u8bbf\u95ee oss://<Bucket>/ \u914d\u7f6e\u7f13\u5b58\u52a0\u901f \u00a7 \u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e \u8bbf\u95ee\u963f\u91cc\u4e91 OSS \u00a7 \u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u5728\u6570\u636e\u7f13\u5b58\u5f00\u5173\u6253\u5f00\u65f6\u901a\u8fc7 jindo:// \u524d\u7f00\u8bfb\u53d6 OSS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 jindo:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002 \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u963f\u91cc\u4e91 OSS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/oss/jindofsx_on_oss_jindo/#oss","text":"\u672c\u6587\u4e3b\u8981\u4ecb\u7ecd JindoFSx \u652f\u6301\u963f\u91cc\u4e91 OSS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f\u7684\u4f7f\u7528\u65b9\u5f0f\u3002","title":"\u963f\u91cc\u4e91 OSS \u7edf\u4e00\u6302\u8f7d\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/oss/jindofsx_on_oss_jindo/#_1","text":"\u5df2\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK","title":"\u524d\u63d0\u6761\u4ef6\uff1a"},{"location":"backup/jindofsx/oss/jindofsx_on_oss_jindo/#oss-accesskey","text":"\u5728 JINDOFSX_CONF_DIR \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e jindofsx.cfg \u6587\u4ef6, \u914d\u7f6e\u7f13\u5b58\u52a0\u901f\u7684 OSS bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \uff0c \u5e76\u66f4\u65b0\u5230\u6240\u9700\u8981\u8282\u70b9\u4e0a\uff08Namespace Service \u548c Storage Service \u6240\u5728\u8282\u70b9\uff09\u3002 [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = oss-cn-xxx-internal.aliyuncs.com \u8bf4\u660e: XXX \u548c YYY \u4e3a OSS bucket \u540d\u79f0\u3002","title":"\u670d\u52a1\u7aef\u914d\u7f6e\u963f\u91cc\u4e91 OSS AccessKey"},{"location":"backup/jindofsx/oss/jindofsx_on_oss_jindo/#jindofsx","text":"\u91cd\u542f JindoFSx \u670d\u52a1\uff0c\u4f7f\u5f97\u914d\u7f6e\u7684 OSS bucket \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u751f\u6548\u3002\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\u3002 cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"\u91cd\u542f JindoFSx \u670d\u52a1"},{"location":"backup/jindofsx/oss/jindofsx_on_oss_jindo/#jindosdk","text":"\u914d\u7f6e\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u670d\u52a1\u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS AccessKey \u5c06 OSS Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528","title":"\u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/oss/jindofsx_on_oss_jindo/#oss_1","text":"\u6302\u8f7d OSS \u76ee\u5f55 jindo admin -mount <path> <realpath> \u4f8b\uff1a jindo admin -mount /jindooss oss://<Bucket>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>/ [root@emr-header-1 ~]# hdfs dfs -ls jindo://emr-header-1:8101/ Found 1 items ---------- 1 0 1970-01-01 08:00 jindo://emr-header-1:8101/jindooss \u5373\u8bbf\u95ee jindo://emr-header-1:8101/jindooss/ \u7b49\u4ef7\u4e8e\u8bbf\u95ee oss://<Bucket>/","title":"\u6302\u8f7d\u963f\u91cc\u4e91 OSS"},{"location":"backup/jindofsx/oss/jindofsx_on_oss_jindo/#_2","text":"\u53ef\u4ee5\u6839\u636e\u9700\u8981\u914d\u7f6e\u5143\u6570\u636e\u7f13\u5b58\u53ca\u6570\u636e\u7f13\u5b58\uff0c\u914d\u7f6e\u65b9\u5f0f\u53c2\u8003\u6587\u6863 JindoFSx \u7f13\u5b58\u4f7f\u7528\u8bf4\u660e","title":"\u914d\u7f6e\u7f13\u5b58\u52a0\u901f"},{"location":"backup/jindofsx/oss/jindofsx_on_oss_jindo/#oss_2","text":"\u5b8c\u6210\u4e0a\u8ff0\u6b65\u9aa4\u540e\u4f5c\u4e1a\u5728\u6570\u636e\u7f13\u5b58\u5f00\u5173\u6253\u5f00\u65f6\u901a\u8fc7 jindo:// \u524d\u7f00\u8bfb\u53d6 OSS \u4e0a\u7684\u6570\u636e\u540e\uff0c\u4f1a\u81ea\u52a8\u7f13\u5b58\u5230 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4e2d\uff0c\u540e\u7eed\u901a\u8fc7 jindo:// \u8bbf\u95ee\u76f8\u540c\u7684\u6570\u636e\u5c31\u80fd\u591f\u547d\u4e2d\u7f13\u5b58\u3002","title":"\u8bbf\u95ee\u963f\u91cc\u4e91 OSS"},{"location":"backup/jindofsx/oss/jindofsx_on_oss_jindo/#_3","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/permission/jindofsx_ranger/","text":"\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u00a7 \u672c\u6587\u4ecb\u7ecd\u5728\u81ea\u5efa\u96c6\u7fa4\u914d\u7f6e\u963f\u91cc\u4e91 OSS \u548c OSS-HDFS \u670d\u52a1\u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\u3002 \u80cc\u666f \u00a7 Apache Ranger \u63d0\u4f9b\u96c6\u4e2d\u5f0f\u7684\u6743\u9650\u7ba1\u7406\u6846\u67b6\uff0c\u53ef\u4ee5\u5bf9 Hadoop \u751f\u6001\u4e2d\u7684\u591a\u4e2a\u7ec4\u4ef6\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u8bbf\u95ee\u63a7\u5236\u3002\u5f53\u7528\u6237\u5c06\u6570\u636e\u5b58\u653e\u5728\u963f\u91cc\u4e91 OSS \u65f6\uff0c\u5219\u662f\u901a\u8fc7\u963f\u91cc\u4e91 RAM \u4ea7\u54c1\u521b\u5efa\u6216\u7ba1\u7406 RAM \u7528\u6237\uff0c\u5bf9RAM\u7528\u6237\u5b9e\u73b0\u5bf9 OSS \u8d44\u6e90\u7684\u8bbf\u95ee\u63a7\u5236\u3002 \u4e3a\u7ef4\u6301\u5927\u6570\u636e\u5ba2\u6237\u7684\u4f7f\u7528\u4e60\u60ef\uff0c\u901a\u8fc7 JindoFSx Namespace \u63a5\u5165 Ranger \u7684\u5ba2\u6237\u7aef\uff0c\u65b9\u4fbf\u7528\u6237\u7edf\u4e00\u7ba1\u7406\u5927\u6570\u636e\u7ec4\u4ef6\u6743\u9650\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u5df2\u90e8\u7f72 JindoFSx Namespace \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx Namespace\uff0c\u8bf7\u53c2\u8003 \u5feb\u901f\u90e8\u7f72\u4e00\u4e2a\u7b80\u5355\u7684 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf(\u4ec5 Namespace) \u5b89\u88c5 Kerberos \u548c Sasl2 \u76f8\u5173\u4f9d\u8d56 \u5982\u679c\u60a8\u7684\u73af\u5883\u6ca1\u6709 Kerberos \u548c Sasl2 \u76f8\u5173\u4f9d\u8d56\uff0c\u8bf7\u5b89\u88c5\u76f8\u5173\u4f9d\u8d56 \u5b89\u88c5\u8bf4\u660e 1. \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u3002 \u00a7 1.1 \u914d\u7f6e Jindo SASL \u63d2\u4ef6 \u00a7 \u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 jindofsx-4.6.12/plugins/ \u8def\u5f84\u4e0b\u3002 \u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\uff0c \u5728 [jindofsx-namespace] section \u4e0b\u6dfb\u52a0\u52a8\u6001\u5e93\u7684\u7edd\u5bf9\u8def\u5f84\u3002 \u53c2\u6570 \u503c namespace.plugin.dir ${JINDDATA_HOME}/plugins/ 1.2 \u542f\u7528 Ranger \u9274\u6743 \u00a7 \u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\uff0c \u5728 [jindofsx-namespace] section \u4e0b\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.oss.authorization.method ranger \u6ce8\u610f\uff1a \u5c06 jindofsx.cfg \u914d\u7f6e\u6587\u4ef6\u90e8\u7f72\u5230 NamespaceService \u6240\u5728\u7684\u6240\u6709\u8282\u70b9\u3002 1.3 \u914d\u7f6e\u4f7f\u7528 Assume Role \u9881\u53d1\u4e34\u65f6 AK \u00a7 \u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 \u5728 [jindofsx-namespace] section \u4e0b\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.oss.credential.provider ASSUME_ROLE \u5728\u963f\u91cc\u4e91 RAM \u521b\u5efa\u7528\u6237\u9881\u53d1\u4e34\u65f6 AK \u7684 Role, \u9009\u62e9\u53ef\u4fe1\u5b9e\u4f53\u7c7b\u578b\u4e3a\"\u963f\u91cc\u4e91\u8d26\u53f7\", \u5e76\u4e3a\u89d2\u8272\u6388\u6743\u3002 \u5728 [jindofsx-common] section \u4e0b\u6dfb\u52a0\u6216\u66f4\u6539\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c default.credential.provider ASSUME_ROLE sts.access.key STS \u7684 accessKey sts.access.secret STS \u7684 accessKeySecret sts.access.endpoint STS \u7684 endpoint, \u5982\uff1a sts.cn-shanghai.aliyuncs.com sts.role.name \u5728 RAM \u4e2d\u521b\u5efa\u7684\u89d2\u8272\uff0c \u5982 JindoFsxTestRole sts.role.arn \u89d2\u8272\u7684ARN\uff0c\u5982\uff1aacs:ram::xxx:role/JindoFsxTestRole \u6ce8\u610f\uff1a \u5c06 jindofsx.cfg \u914d\u7f6e\u6587\u4ef6\u90e8\u7f72\u5230 NamespaceService \u6240\u5728\u7684\u6240\u6709\u8282\u70b9\u3002 1.4 \u914d\u7f6e Principal \u548c Keytab\uff08\u9002\u7528\u4e8e Kerberos \u96c6\u7fa4\uff09 \u00a7 \u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 \u5728 [jindofsx-namespace] section \u4e0b\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.authentication.enable true namespace.authentication.keytab \u914d\u7f6e keytab \u6240\u5728\u7684\u8def\u5f84\uff0c \u5982\uff1a /tmp/jindofsx.keytab namespace.authentication.principal \u914d\u7f6e keytab \u4e2d\u7684 principal, \u5982\uff1a jindofsx/localhost 2. \u91cd\u542f JindoFSx Namespace \u670d\u52a1\u3002 \u00a7 \u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\uff1a cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh 3. \u914d\u7f6e JindoSDK\u3002 \u00a7 3.1 \u914d\u7f6e xengine \u00a7 \u5728 Hadoop \u7684 core-site.xml \u4e2d \u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.xengine jindofsx 3.2 \u914d\u7f6e Jindo SASL \u63d2\u4ef6 \u00a7 \u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 jindosdk-4.6.12/plugins/ \u8def\u5f84\u4e0b\uff0c\u5c06\u8be5\u8def\u5f84\u7684\u7edd\u5bf9\u8def\u5f84\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 \u53c2\u6570 \u503c fs.jdo.plugin.dir ${JINDOSDK_HOME}/plugins/ 3.3 \u542f\u7528 Ranger \u9274\u6743 \u00a7 \u652f\u6301\u7edf\u4e00\u914d\u7f6e\u548c\u6309 bucket \u914d\u7f6e\u4e24\u79cd\u65b9\u5f0f\u3002 a. \u5bf9 OSS \u548c OSS-HDFS \u7684\u5168\u90e8 bucket \u9274\u6743 \u00a7 \u5728Hadoop \u7684 core-site.xml \u4e2d\u6dfb\u52a0\u6216\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.authorization.method ranger fs.oss.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider b. \u5bf9 OSS \u6216 OSS-HDFS \u7684\u90e8\u5206 bucket \u9274\u6743 \u00a7 \u5728Hadoop \u7684 core-site.xml \u4e2d\u6dfb\u52a0\u6216\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.bucket.XXX.authorization.method ranger fs.oss.bucket.XXX.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider \u8bf4\u660e\uff1aXXX \u4e3a \u6216 OSS \u6216 OSS-HDFS \u670d\u52a1 bucket\u7684\u540d\u79f0\u3002 4. \u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 5. \u5b89\u88c5\u652f\u6301\u914d\u7f6e OSS \u6743\u9650\u7684 Ranger \u63d2\u4ef6 \u00a7 Ranger \u7ec4\u4ef6\u9700\u8981\u5b89\u88c5\u65b0\u63d2\u4ef6\u4ee5\u652f\u6301\u5bf9 OSS \u6216 OSS-HDFS \u670d\u52a1\u5bf9\u9274\u6743\u3002 5.1 \u4e0b\u8f7d\u548c\u62f7\u8d1d\u63d2\u4ef6 \u00a7 \u6839\u636e\u96c6\u7fa4\u4e2d\u5b89\u88c5\u7684 Ranger \u7248\u672c\u4e0b\u8f7d\u5bf9\u5e94\u7684\u63d2\u4ef6\u7248\u672c\u3002 1.x \u7248\u672c \u4e0b\u8f7d\u5730\u5740 2.x \u7248\u672c \u4e0b\u8f7d\u5730\u5740 cd /xxx/ranger-admin-current/ews/webapp/WEB-INF/classes/ranger-plugins mkdir oss cp ranger-oss-plugin-1.2.0.jar /xxx/ranger-admin-current/ews/webapp/WEB-INF/classes/ranger-plugins/oss \u6ce8\u610f\uff1a\u9700\u8981\u5b89\u88c5\u5728\u90e8\u7f72 Ranger Admin \u7684\u6240\u6709\u8282\u70b9\u3002 5.2 \u5b89\u88c5\u63d2\u4ef6 \u00a7 \u521b\u5efa\u5982\u4e0branger-oss.json\u6587\u4ef6\uff1a { \"id\": 500, \"name\": \"oss\", \"implClass\": \"org.apache.ranger.services.oss.RangerServiceOSS\", \"label\": \"OSS\", \"description\": \"OSS\", \"guid\": \"0d047247-bafe-4cf8-8e9b-d5d377284b2h\", \"resources\": [ { \"itemId\": 1, \"name\": \"path\", \"type\": \"path\", \"level\": 10, \"parent\": \"\", \"mandatory\": true, \"lookupSupported\": true, \"recursiveSupported\": true, \"excludesSupported\": false, \"matcher\": \"org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher\", \"matcherOptions\": { \"wildCard\": true, \"ignoreCase\": false }, \"validationRegEx\": \"\", \"validationMessage\": \"\", \"uiHint\": \"\", \"label\": \"Path\", \"description\": \"OSS Path, Should Not Start With /, exp. dir/test.txt\" } ], \"accessTypes\": [ { \"itemId\": 1, \"name\": \"read\", \"label\": \"Read\" }, { \"itemId\": 2, \"name\": \"write\", \"label\": \"Write\" }, { \"itemId\": 3, \"name\": \"execute\", \"label\": \"Execute\" } ], \"configs\": [ ], \"enums\": [ ], \"contextEnrichers\": [], \"policyConditions\": [] } \u6267\u884c\u5982\u4e0b\u547d\u4ee4\u88c5\u8f7d ranger-oss\uff0c\u6ce8\u610f\u5176\u4e2d json \u6587\u4ef6\u7684\u8def\u5f84\u548c ranger admin \u7684 url\uff0cadminUser \u9ed8\u8ba4\u4e3a admin\uff0cadminPasswd \u9ed8\u8ba4\u4e3a admin\uff1a curl -v -u${adminUser}:${adminPasswd} -X POST -H \"Accept:application/json\" -H \"Content-Type:application/json\" -d @ranger-oss.json http://emr-header-1:6080/service/plugins/definitions \u63d2\u4ef6\u6210\u529f\u5b89\u88c5\u540e\uff0c\u53ef\u4ee5\u5728 Ranger UI \u4e0a\u770b\u5230\u65b0\u589e\u9879\uff1a \u521b\u5efa\u4e00\u4e2aService\uff0c\u4f8b\u5982emr-oss\uff0c\u521b\u5efa\u5b8c\u6210\u540e\u5982\u4e0b\uff0c\u540e\u7eed\u5c31\u53ef\u4ee5\u5728\u8be5Service\u4e0b\u521b\u5efaPolicy\uff1a 6. \u5728 Ranger WebUI \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1\u6743\u9650 \u00a7 Ranger \u89c4\u5219\u793a\u4f8b \u00a7 \u4f8b\uff1a\u914d\u7f6e\u7528\u6237test\u62e5\u6709\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u6240\u4ee5\u6743\u9650\u7684\u6b65\u9aa4\uff1a a. \u914d\u7f6e test \u7528\u6237\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u8bbf\u95ee\u6743\u9650\u4e3a ALL\u3002 \u00a7 \u8bf4\u660e\uff1a \u00a7 \u89c4\u5219\u914d\u7f6e\u9875\u9762\u4e2d\uff0c\u914d\u7f6e\u7684 path \u6ca1\u6709 oss:// \u7684\u524d\u7f00\u3002 recursive\u6309\u94ae\u4e0d\u53ef\u5173\u95ed\u3002 \u8def\u5f84\u672b\u5c3e\u65e0\u9700\u5e26\u6b63\u659c\u7ebf\uff08/\uff09\u3002 b. \u9700\u8981\u914d\u7f6e\u8bbf\u95ee\u8def\u5f84\u7684\u7236\u76ee\u5f55 oss://bucket-test-hangzhou/user \u7684\u6743\u9650\u4e3a Execute\u3002 \u00a7 7. \u8bbf\u95eeOSS\u3002 \u00a7 \u82e5\u7528\u6237\u8bbf\u95ee Ranger \u6ca1\u6709\u6388\u6743\u7684\u8def\u5f84\uff0c\u5219\u4f1a\u62a5\u5982\u4e0b\u9519\u8bef\uff1a org.apache.hadoop.security.AccessControlException: Permission denied: user=test, access=READ_EXECUTE, resourcePath=\"bucket-test-hangzhou/\" \u6ce8\u610f\u4e8b\u9879 \u00a7 \u9488\u5bf9\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7684\u9274\u6743\uff0c\u76ee\u524d\u4f7f\u7528\u7684\u662f\u5bf9\u8c61\u5b58\u50a8\u7684\u9274\u6743\u65b9\u5f0f\uff08\u901a\u8fc7\u5339\u914d\u8def\u5f84\uff09\uff0c\u5bf9\u8c61\u5b58\u50a8\u6ca1\u6709\u76ee\u5f55\u6216\u6587\u4ef6\u7684 owner \u548c permission \u4fe1\u606f\uff0c\u5982\u9700\u5bf9 OSS-HDFS \u670d\u52a1\u4f7f\u7528 chmod , chown \u7b49\u64cd\u4f5c\uff0c\u53ea\u80fd\u901a\u8fc7 hadoop \u7528\u6237\u6267\u884c\u3002","title":"\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#oss-ranger","text":"\u672c\u6587\u4ecb\u7ecd\u5728\u81ea\u5efa\u96c6\u7fa4\u914d\u7f6e\u963f\u91cc\u4e91 OSS \u548c OSS-HDFS \u670d\u52a1\u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\u3002","title":"\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#_1","text":"Apache Ranger \u63d0\u4f9b\u96c6\u4e2d\u5f0f\u7684\u6743\u9650\u7ba1\u7406\u6846\u67b6\uff0c\u53ef\u4ee5\u5bf9 Hadoop \u751f\u6001\u4e2d\u7684\u591a\u4e2a\u7ec4\u4ef6\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u8bbf\u95ee\u63a7\u5236\u3002\u5f53\u7528\u6237\u5c06\u6570\u636e\u5b58\u653e\u5728\u963f\u91cc\u4e91 OSS \u65f6\uff0c\u5219\u662f\u901a\u8fc7\u963f\u91cc\u4e91 RAM \u4ea7\u54c1\u521b\u5efa\u6216\u7ba1\u7406 RAM \u7528\u6237\uff0c\u5bf9RAM\u7528\u6237\u5b9e\u73b0\u5bf9 OSS \u8d44\u6e90\u7684\u8bbf\u95ee\u63a7\u5236\u3002 \u4e3a\u7ef4\u6301\u5927\u6570\u636e\u5ba2\u6237\u7684\u4f7f\u7528\u4e60\u60ef\uff0c\u901a\u8fc7 JindoFSx Namespace \u63a5\u5165 Ranger \u7684\u5ba2\u6237\u7aef\uff0c\u65b9\u4fbf\u7528\u6237\u7edf\u4e00\u7ba1\u7406\u5927\u6570\u636e\u7ec4\u4ef6\u6743\u9650\u3002","title":"\u80cc\u666f"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#_2","text":"\u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u90e8\u7f72 JindoSDK \u5df2\u90e8\u7f72 JindoFSx Namespace \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoFSx Namespace\uff0c\u8bf7\u53c2\u8003 \u5feb\u901f\u90e8\u7f72\u4e00\u4e2a\u7b80\u5355\u7684 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf(\u4ec5 Namespace) \u5b89\u88c5 Kerberos \u548c Sasl2 \u76f8\u5173\u4f9d\u8d56 \u5982\u679c\u60a8\u7684\u73af\u5883\u6ca1\u6709 Kerberos \u548c Sasl2 \u76f8\u5173\u4f9d\u8d56\uff0c\u8bf7\u5b89\u88c5\u76f8\u5173\u4f9d\u8d56 \u5b89\u88c5\u8bf4\u660e","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#1-jindofsx-namespace","text":"","title":"1. \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#11-jindo-sasl","text":"\u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 jindofsx-4.6.12/plugins/ \u8def\u5f84\u4e0b\u3002 \u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\uff0c \u5728 [jindofsx-namespace] section \u4e0b\u6dfb\u52a0\u52a8\u6001\u5e93\u7684\u7edd\u5bf9\u8def\u5f84\u3002 \u53c2\u6570 \u503c namespace.plugin.dir ${JINDDATA_HOME}/plugins/","title":"1.1 \u914d\u7f6e Jindo SASL \u63d2\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#12-ranger","text":"\u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\uff0c \u5728 [jindofsx-namespace] section \u4e0b\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.oss.authorization.method ranger \u6ce8\u610f\uff1a \u5c06 jindofsx.cfg \u914d\u7f6e\u6587\u4ef6\u90e8\u7f72\u5230 NamespaceService \u6240\u5728\u7684\u6240\u6709\u8282\u70b9\u3002","title":"1.2 \u542f\u7528 Ranger \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#13-assume-role-ak","text":"\u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 \u5728 [jindofsx-namespace] section \u4e0b\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.oss.credential.provider ASSUME_ROLE \u5728\u963f\u91cc\u4e91 RAM \u521b\u5efa\u7528\u6237\u9881\u53d1\u4e34\u65f6 AK \u7684 Role, \u9009\u62e9\u53ef\u4fe1\u5b9e\u4f53\u7c7b\u578b\u4e3a\"\u963f\u91cc\u4e91\u8d26\u53f7\", \u5e76\u4e3a\u89d2\u8272\u6388\u6743\u3002 \u5728 [jindofsx-common] section \u4e0b\u6dfb\u52a0\u6216\u66f4\u6539\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c default.credential.provider ASSUME_ROLE sts.access.key STS \u7684 accessKey sts.access.secret STS \u7684 accessKeySecret sts.access.endpoint STS \u7684 endpoint, \u5982\uff1a sts.cn-shanghai.aliyuncs.com sts.role.name \u5728 RAM \u4e2d\u521b\u5efa\u7684\u89d2\u8272\uff0c \u5982 JindoFsxTestRole sts.role.arn \u89d2\u8272\u7684ARN\uff0c\u5982\uff1aacs:ram::xxx:role/JindoFsxTestRole \u6ce8\u610f\uff1a \u5c06 jindofsx.cfg \u914d\u7f6e\u6587\u4ef6\u90e8\u7f72\u5230 NamespaceService \u6240\u5728\u7684\u6240\u6709\u8282\u70b9\u3002","title":"1.3 \u914d\u7f6e\u4f7f\u7528 Assume Role \u9881\u53d1\u4e34\u65f6 AK"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#14-principal-keytab-kerberos","text":"\u5728 jindofsx-x.x.x/conf \u6587\u4ef6\u5939\u4e0b\u4fee\u6539\u914d\u7f6e\u6587\u4ef6 jindofsx.cfg\u3002 \u5728 [jindofsx-namespace] section \u4e0b\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.authentication.enable true namespace.authentication.keytab \u914d\u7f6e keytab \u6240\u5728\u7684\u8def\u5f84\uff0c \u5982\uff1a /tmp/jindofsx.keytab namespace.authentication.principal \u914d\u7f6e keytab \u4e2d\u7684 principal, \u5982\uff1a jindofsx/localhost","title":"1.4 \u914d\u7f6e Principal \u548c Keytab\uff08\u9002\u7528\u4e8e Kerberos \u96c6\u7fa4\uff09"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#2-jindofsx-namespace","text":"\u5728 master \u8282\u70b9\u6267\u884c\u4ee5\u4e0b\u811a\u672c\uff1a cd jindofsx-x.x.x sh sbin/stop-service.sh sh sbin/start-service.sh","title":"2. \u91cd\u542f JindoFSx Namespace \u670d\u52a1\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#31-xengine","text":"\u5728 Hadoop \u7684 core-site.xml \u4e2d \u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.xengine jindofsx","title":"3.1 \u914d\u7f6e xengine"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#32-jindo-sasl","text":"\u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 jindosdk-4.6.12/plugins/ \u8def\u5f84\u4e0b\uff0c\u5c06\u8be5\u8def\u5f84\u7684\u7edd\u5bf9\u8def\u5f84\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 \u53c2\u6570 \u503c fs.jdo.plugin.dir ${JINDOSDK_HOME}/plugins/","title":"3.2 \u914d\u7f6e Jindo SASL \u63d2\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#33-ranger","text":"\u652f\u6301\u7edf\u4e00\u914d\u7f6e\u548c\u6309 bucket \u914d\u7f6e\u4e24\u79cd\u65b9\u5f0f\u3002","title":"3.3 \u542f\u7528 Ranger \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#a-oss-oss-hdfs-bucket","text":"\u5728Hadoop \u7684 core-site.xml \u4e2d\u6dfb\u52a0\u6216\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.authorization.method ranger fs.oss.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider","title":"a. \u5bf9 OSS \u548c OSS-HDFS \u7684\u5168\u90e8 bucket \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#b-oss-oss-hdfs-bucket","text":"\u5728Hadoop \u7684 core-site.xml \u4e2d\u6dfb\u52a0\u6216\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.bucket.XXX.authorization.method ranger fs.oss.bucket.XXX.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider \u8bf4\u660e\uff1aXXX \u4e3a \u6216 OSS \u6216 OSS-HDFS \u670d\u52a1 bucket\u7684\u540d\u79f0\u3002","title":"b. \u5bf9 OSS \u6216 OSS-HDFS \u7684\u90e8\u5206 bucket \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#4-hive","text":"","title":"4. \u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#5-oss-ranger","text":"Ranger \u7ec4\u4ef6\u9700\u8981\u5b89\u88c5\u65b0\u63d2\u4ef6\u4ee5\u652f\u6301\u5bf9 OSS \u6216 OSS-HDFS \u670d\u52a1\u5bf9\u9274\u6743\u3002","title":"5. \u5b89\u88c5\u652f\u6301\u914d\u7f6e OSS \u6743\u9650\u7684 Ranger \u63d2\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#51","text":"\u6839\u636e\u96c6\u7fa4\u4e2d\u5b89\u88c5\u7684 Ranger \u7248\u672c\u4e0b\u8f7d\u5bf9\u5e94\u7684\u63d2\u4ef6\u7248\u672c\u3002 1.x \u7248\u672c \u4e0b\u8f7d\u5730\u5740 2.x \u7248\u672c \u4e0b\u8f7d\u5730\u5740 cd /xxx/ranger-admin-current/ews/webapp/WEB-INF/classes/ranger-plugins mkdir oss cp ranger-oss-plugin-1.2.0.jar /xxx/ranger-admin-current/ews/webapp/WEB-INF/classes/ranger-plugins/oss \u6ce8\u610f\uff1a\u9700\u8981\u5b89\u88c5\u5728\u90e8\u7f72 Ranger Admin \u7684\u6240\u6709\u8282\u70b9\u3002","title":"5.1 \u4e0b\u8f7d\u548c\u62f7\u8d1d\u63d2\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#52","text":"\u521b\u5efa\u5982\u4e0branger-oss.json\u6587\u4ef6\uff1a { \"id\": 500, \"name\": \"oss\", \"implClass\": \"org.apache.ranger.services.oss.RangerServiceOSS\", \"label\": \"OSS\", \"description\": \"OSS\", \"guid\": \"0d047247-bafe-4cf8-8e9b-d5d377284b2h\", \"resources\": [ { \"itemId\": 1, \"name\": \"path\", \"type\": \"path\", \"level\": 10, \"parent\": \"\", \"mandatory\": true, \"lookupSupported\": true, \"recursiveSupported\": true, \"excludesSupported\": false, \"matcher\": \"org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher\", \"matcherOptions\": { \"wildCard\": true, \"ignoreCase\": false }, \"validationRegEx\": \"\", \"validationMessage\": \"\", \"uiHint\": \"\", \"label\": \"Path\", \"description\": \"OSS Path, Should Not Start With /, exp. dir/test.txt\" } ], \"accessTypes\": [ { \"itemId\": 1, \"name\": \"read\", \"label\": \"Read\" }, { \"itemId\": 2, \"name\": \"write\", \"label\": \"Write\" }, { \"itemId\": 3, \"name\": \"execute\", \"label\": \"Execute\" } ], \"configs\": [ ], \"enums\": [ ], \"contextEnrichers\": [], \"policyConditions\": [] } \u6267\u884c\u5982\u4e0b\u547d\u4ee4\u88c5\u8f7d ranger-oss\uff0c\u6ce8\u610f\u5176\u4e2d json \u6587\u4ef6\u7684\u8def\u5f84\u548c ranger admin \u7684 url\uff0cadminUser \u9ed8\u8ba4\u4e3a admin\uff0cadminPasswd \u9ed8\u8ba4\u4e3a admin\uff1a curl -v -u${adminUser}:${adminPasswd} -X POST -H \"Accept:application/json\" -H \"Content-Type:application/json\" -d @ranger-oss.json http://emr-header-1:6080/service/plugins/definitions \u63d2\u4ef6\u6210\u529f\u5b89\u88c5\u540e\uff0c\u53ef\u4ee5\u5728 Ranger UI \u4e0a\u770b\u5230\u65b0\u589e\u9879\uff1a \u521b\u5efa\u4e00\u4e2aService\uff0c\u4f8b\u5982emr-oss\uff0c\u521b\u5efa\u5b8c\u6210\u540e\u5982\u4e0b\uff0c\u540e\u7eed\u5c31\u53ef\u4ee5\u5728\u8be5Service\u4e0b\u521b\u5efaPolicy\uff1a","title":"5.2 \u5b89\u88c5\u63d2\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#6-ranger-webui-oss-oss-hdfs","text":"","title":"6. \u5728 Ranger WebUI \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1\u6743\u9650"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#ranger","text":"\u4f8b\uff1a\u914d\u7f6e\u7528\u6237test\u62e5\u6709\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u6240\u4ee5\u6743\u9650\u7684\u6b65\u9aa4\uff1a","title":"Ranger \u89c4\u5219\u793a\u4f8b"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#a-test-ossbucket-test-hangzhouusertest-all","text":"","title":"a. \u914d\u7f6e test \u7528\u6237\u8bbf\u95eeoss://bucket-test-hangzhou/user/test\u76ee\u5f55\u7684\u8bbf\u95ee\u6743\u9650\u4e3a ALL\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#_3","text":"\u89c4\u5219\u914d\u7f6e\u9875\u9762\u4e2d\uff0c\u914d\u7f6e\u7684 path \u6ca1\u6709 oss:// \u7684\u524d\u7f00\u3002 recursive\u6309\u94ae\u4e0d\u53ef\u5173\u95ed\u3002 \u8def\u5f84\u672b\u5c3e\u65e0\u9700\u5e26\u6b63\u659c\u7ebf\uff08/\uff09\u3002","title":"\u8bf4\u660e\uff1a"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#b-ossbucket-test-hangzhouuser-execute","text":"","title":"b. \u9700\u8981\u914d\u7f6e\u8bbf\u95ee\u8def\u5f84\u7684\u7236\u76ee\u5f55oss://bucket-test-hangzhou/user\u7684\u6743\u9650\u4e3a Execute\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#7-oss","text":"\u82e5\u7528\u6237\u8bbf\u95ee Ranger \u6ca1\u6709\u6388\u6743\u7684\u8def\u5f84\uff0c\u5219\u4f1a\u62a5\u5982\u4e0b\u9519\u8bef\uff1a org.apache.hadoop.security.AccessControlException: Permission denied: user=test, access=READ_EXECUTE, resourcePath=\"bucket-test-hangzhou/\"","title":"7. \u8bbf\u95eeOSS\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger/#_4","text":"\u9488\u5bf9\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u7684\u9274\u6743\uff0c\u76ee\u524d\u4f7f\u7528\u7684\u662f\u5bf9\u8c61\u5b58\u50a8\u7684\u9274\u6743\u65b9\u5f0f\uff08\u901a\u8fc7\u5339\u914d\u8def\u5f84\uff09\uff0c\u5bf9\u8c61\u5b58\u50a8\u6ca1\u6709\u76ee\u5f55\u6216\u6587\u4ef6\u7684 owner \u548c permission \u4fe1\u606f\uff0c\u5982\u9700\u5bf9 OSS-HDFS \u670d\u52a1\u4f7f\u7528 chmod , chown \u7b49\u64cd\u4f5c\uff0c\u53ea\u80fd\u901a\u8fc7 hadoop \u7528\u6237\u6267\u884c\u3002","title":"\u6ce8\u610f\u4e8b\u9879"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/","text":"\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u00a7 \u672c\u6587\u4ecb\u7ecd\u5728\u963f\u91cc\u4e91 E-MapReduce \u96c6\u7fa4\u914d\u7f6e\u963f\u91cc\u4e91 OSS \u548c OSS-HDFS \u670d\u52a1\u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\u3002 \u80cc\u666f \u00a7 Apache Ranger \u63d0\u4f9b\u96c6\u4e2d\u5f0f\u7684\u6743\u9650\u7ba1\u7406\u6846\u67b6\uff0c\u53ef\u4ee5\u5bf9 Hadoop \u751f\u6001\u4e2d\u7684\u591a\u4e2a\u7ec4\u4ef6\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u8bbf\u95ee\u63a7\u5236\u3002\u5f53\u7528\u6237\u5c06\u6570\u636e\u5b58\u653e\u5728\u963f\u91cc\u4e91 OSS \u65f6\uff0c\u5219\u662f\u901a\u8fc7\u963f\u91cc\u4e91 RAM \u4ea7\u54c1\u521b\u5efa\u6216\u7ba1\u7406 RAM \u7528\u6237\uff0c\u5bf9RAM\u7528\u6237\u5b9e\u73b0\u5bf9 OSS \u8d44\u6e90\u7684\u8bbf\u95ee\u63a7\u5236\u3002 \u4e3a\u7ef4\u6301\u5927\u6570\u636e\u5ba2\u6237\u7684\u4f7f\u7528\u4e60\u60ef\uff0c\u901a\u8fc7 JindoFSx Namespace \u63a5\u5165 Ranger \u7684\u5ba2\u6237\u7aef\uff0c\u65b9\u4fbf\u7528\u6237\u7edf\u4e00\u7ba1\u7406\u5927\u6570\u636e\u7ec4\u4ef6\u6743\u9650\u3002 \u8bbf\u95ee OSS \u9274\u6743\u6d41\u7a0b \u00a7 OSS \u7684\u8bbf\u95ee\u5bc6\u94a5 AccessKey\uff08AK\uff09\u7edf\u4e00\u5728 JindoFSx Namespace \u4e2d\u8bbe\u7f6e\uff0c\u907f\u514d\u7528\u6237\u5728\u5ba2\u6237\u7aef\u914d\u7f6e\u660e\u6587\u5bc6\u94a5\uff0c\u5efa\u8bae\u53ea\u5141\u8bb8\u7ba1\u7406\u5458\u64cd\u4f5c\u548c\u7ba1\u7406 JindoFSx Namespace \u670d\u52a1\uff1b JindoSDK \u63d0\u4f9b\u6807\u51c6 Hadoop Filesystem \u5ba2\u6237\u7aef\uff0c\u5c06\u8bbf\u95ee OSS \u7684\u8bf7\u6c42\u4f1a\u53d1\u9001\u81f3 Namespace \u670d\u52a1\uff1b JindoFSx Namespace Service \u8d1f\u8d23\u96c6\u6210 Ranger \u5ba2\u6237\u7aef\uff0c\u5468\u671f\u6027\u5c06\u6743\u9650\u7b56\u7565\u4ece Ranger \u670d\u52a1\u7aef\u540c\u6b65\u5230\u672c\u5730\uff1b Namespace \u670d\u52a1\u5728\u6536\u5230 JindoSDK \u7684\u9274\u6743\u8bf7\u6c42\u540e\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u6821\u9a8c\uff1b \u901a\u8fc7\u6743\u9650\u6821\u9a8c\u540e JindoSDK \u5219\u53ef\u4ee5\u4f7f\u7528 JindoFSx Namespace \u670d\u52a1\u9881\u53d1\u7684\u4e34\u65f6 AK \u8bbf\u95ee OSS\u3002 \u6ce8\uff1a\u82e5 Namespace \u90e8\u7f72\u7684\u8282\u70b9\u4e3a\u963f\u91cc\u4e91 ECS \u8282\u70b9\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u5b89\u5168\u7ec4\uff0c\u9650\u5236\u8bbf\u95ee Namespace \u7684\u5ba2\u6237\u7aef\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u521b\u5efa E-MapReduce EMR-5.6.0/EMR-3.40.0 \u6216\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\uff0c\u5e76\u4e14\u9009\u62e9\u4e86 Ranger \u670d\u52a1\u3002 \u542f\u7528 Kerberos \u8ba4\u8bc1\u6ee1\u8db3\u8ba4\u8bc1\u9700\u6c42\u3002 \u82e5\u4f7f\u7528\u96c6\u7fa4\u90fd\u662f\u53ef\u4fe1\u7528\u6237\uff0c\u5219\u53ef\u4e0d\u542f\u7528 Kerberos \u8ba4\u8bc1\u3002 \u6ce8\u610f\uff1aEMR \u96c6\u7fa4\u5f00\u542f Kerberos \u540e\uff0c\u5c06\u65e0\u6cd5\u5173\u95ed Kerberos \u8ba4\u8bc1\uff0c\u8bf7\u8c28\u614e\u9009\u62e9\u3002\u5f00\u542f Kerberos \u540e\uff0c\u96c6\u7fa4\u6240\u6709\u5927\u6570\u636e\u7ec4\u4ef6\u7684\u670d\u52a1\u5747\u9700\u8981\u7ecf\u8fc7 Kerberos \u8ba4\u8bc1\uff0c\u5927\u6570\u636e\u4f5c\u4e1a\u63d0\u4ea4\u5230\u96c6\u7fa4\u4f1a\u5148\u7ecf\u8fc7\u8eab\u4efd\u8ba4\u8bc1\u3002Kerberos \u8be6\u60c5\u4fe1\u606f\uff0c\u8bf7\u53c2\u89c1 \u9ad8\u5b89\u5168\u7c7b\u578b\u96c6\u7fa4 \u521b\u5efa\u9ad8\u5b89\u5168\u96c6\u7fa4 \u00a7 \u521b\u5efa\u96c6\u7fa4\u65f6\uff0c\u5728\u8f6f\u4ef6\u914d\u7f6e\u9875\u9762\u7684\u9ad8\u7ea7\u8bbe\u7f6e\u533a\u57df\u4e2d\uff0c\u6253\u5f00Kerberos\u96c6\u7fa4\u6a21\u5f0f\u5f00\u5173\u3002 \u82e5\u6240\u5728\u96c6\u7fa4\u90fd\u662f\u53ef\u4fe1\u7528\u6237\uff0c\u5219\u53ef\u4e0d\u542f\u7528 Kerberos \u8ba4\u8bc1\u3002 \u3010\u6ce8\u610f\u3011\uff1a\u82e5\u4e3a\u975e\u9ad8\u5b89\u5168\u96c6\u7fa4\uff0c\u5219\u4e0d\u9700\u8981\u6b65\u9aa4 5 \u548c\u6b65\u9aa4 6\u3002 1. \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u3002 \u00a7 1.1 \u914d\u7f6e Jindo SASL \u63d2\u4ef6 \u00a7 \u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 jindodata-4.6.12/plugins/ \u8def\u5f84\u4e0b\uff0c\u5c06\u8be5\u8def\u5f84\u7684\u7edd\u5bf9\u8def\u5f84\u6dfb\u52a0\u5230 JindoData \u7684 namespace \u4e2d\u3002 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.plugin.dir ${JINDDATA_HOME}/plugins/ 1.2 \u542f\u7528 Ranger \u9274\u6743 \u00a7 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.oss.authorization.method ranger 2. \u91cd\u542f JindoFSx Namespace \u670d\u52a1\u3002 \u00a7 3. \u914d\u7f6e JindoSDK\u3002 \u00a7 3.1 \u914d\u7f6e xengine \u00a7 \u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.xengine jindofsx 3.2 \u914d\u7f6e Jindo SASL \u63d2\u4ef6 \u00a7 \u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 jindosdk-4.6.12/plugins/ \u8def\u5f84\u4e0b\uff0c\u5c06\u8be5\u8def\u5f84\u7684\u7edd\u5bf9\u8def\u5f84\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 \u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.jdo.plugin.dir ${JINDOSDK_HOME}/plugins/ 3.3 \u542f\u7528 Ranger \u9274\u6743 \u00a7 \u652f\u6301\u7edf\u4e00\u914d\u7f6e\u548c\u6309 bucket \u914d\u7f6e\u4e24\u79cd\u65b9\u5f0f\u3002 a. \u5bf9 OSS \u548c OSS-HDFS \u7684\u5168\u90e8 bucket \u9274\u6743 \u00a7 \u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.authorization.method ranger \u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site \uff0c\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider b. \u5bf9 OSS \u6216 OSS-HDFS \u7684\u90e8\u5206 bucket \u9274\u6743 \u00a7 \u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.bucket.XXX.authorization.method ranger \u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site \uff0c\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.bucket.XXX.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider \u8bf4\u660e\uff1aXXX \u4e3a \u6216 OSS \u6216 OSS-HDFS \u670d\u52a1 bucket\u7684\u540d\u79f0\u3002 4. \u91cd\u542fHiveServer2\u3002 \u00a7 5. \u521b\u5efa\u7528\u6237 Principal\u3002\uff08\u9002\u7528\u4e8e\u9ad8\u5b89\u5168\u96c6\u7fa4\uff09 \u00a7 a. \u901a\u8fc7SSH\u65b9\u5f0f\u8fde\u63a5\u96c6\u7fa4\u7684emr-header-1\u8282\u70b9\u3002 \u00a7 b. \u6267\u884c\u5982\u4e0b\u547d\u4ee4\uff0c\u8fdb\u5165Kerberos\u7684admin\u5de5\u5177\u3002 \u00a7 sh /usr/lib/has-current/bin/admin-local.sh /etc/ecm/has-conf -k /etc/ecm/has-conf/admin.keytab \u672c\u793a\u4f8b\u5bc6\u7801\u8bbe\u7f6e\u4e3a 123456\u3002 addprinc -pw 123456 test \u8bf4\u660e: \u9700\u8981\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5728\u521b\u5efaTGT\u65f6\u4f1a\u7528\u5230\u3002\u5982\u679c\u60a8\u4e0d\u60f3\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5219\u53ef\u4ee5\u6267\u884c\u4e0b\u4e00\u6b65\uff0c\u628aPrincipal\u7684\u7528\u6237\u540d\u548c\u5bc6\u7801\u5bfc\u5165\u5230keytab\u6587\u4ef6\u4e2d\u3002 \u00a7 c. \u53ef\u9009\uff1a\u6267\u884c\u5982\u4e0b\u547d\u4ee4\uff0c\u751f\u6210keytab\u6587\u4ef6\u3002 \u00a7 ktadd -k /root/test.keytab test \u6267\u884c quit \u547d\u4ee4\uff0c\u53ef\u4ee5\u9000\u51fa Kerberos \u7684 admin \u5de5\u5177\u3002 6. \u521b\u5efa TGT\u3002\uff08\u9002\u7528\u4e8e\u9ad8\u5b89\u5168\u96c6\u7fa4\uff09 \u00a7 \u521b\u5efa TGT \u7684\u673a\u5668\uff0c\u53ef\u4ee5\u662f\u4efb\u610f\u4e00\u53f0\u9700\u8981\u8bbf\u95ee OSS \u7684\u673a\u5668\u3002 a. \u4f7f\u7528 root \u7528\u6237\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u521b\u5efa test \u7528\u6237\u3002 \u00a7 useradd test b. \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u5207\u6362\u4e3a test \u7528\u6237\u3002 \u00a7 su test c. \u751f\u6210TGT\u3002 \u00a7 \u65b9\u5f0f\u4e00\uff1a\u4f7f\u7528\u7528\u6237\u540d\u548c\u5bc6\u7801\u65b9\u5f0f\uff0c\u521b\u5efa TGT\u3002 \u6267\u884c kinit \u547d\u4ee4\uff0c\u56de\u8f66\u540e\u8f93\u5165 test \u7684\u5bc6\u7801 123456\u3002 \u65b9\u5f0f\u4e8c\uff1a\u4f7f\u7528 keytab \u6587\u4ef6\uff0c\u521b\u5efa TGT\u3002 \u5728\u6b65\u9aa4 4 \u4e2d\u7684 test.keytab \u6587\u4ef6\uff0c\u5df2\u7ecf\u4fdd\u5b58\u5728 emr-header-1 \u673a\u5668\u7684 /root/ \u76ee\u5f55\u4e0b\uff0c\u9700\u8981\u4f7f\u7528 scp \u547d\u4ee4\u62f7\u8d1d\u5230\u5f53\u524d\u673a\u5668\u7684 /home/test/ \u76ee\u5f55\u4e0b\u3002 kinit -kt /home/test/test.keytab test d. \u67e5\u770bTGT\u3002 \u00a7 \u4f7f\u7528 klist \u547d\u4ee4\uff0c\u5982\u679c\u51fa\u73b0\u5982\u4e0b\u4fe1\u606f\uff0c\u5219\u8bf4\u660eTGT\u521b\u5efa\u6210\u529f\uff0c\u5373\u53ef\u4ee5\u8bbf\u95eeOSS\u4e86\u3002 Ticket cache: FILE:/tmp/krb5cc_1012 Default principal: test@EMR.23****.COM Valid starting Expires Service principal 03/27/2022 23:20:44 03/28/2022 23:20:44 krbtgt/EMR.238075.COM@EMR.238075.COM renew until 03/28/2022 23:20:44 \u6ce8\u610f: \u9700\u8981\u8bb0\u5f55\u4e0b\u56de\u663e\u4fe1\u606f EMR.23****.COM \u4e2d\u7684\u6570\u5b57 23**** \uff0c\u5373\u4e3a cluster_id \u7684\u503c\uff0c\u540e\u9762\u8bbf\u95ee OSS \u65f6\u9700\u8981\u3002 7. \u5728 Ranger WebUI \u914d\u7f6e OSS \u6743\u9650 \u00a7 Ranger \u89c4\u5219\u793a\u4f8b \u00a7 \u4f8b\uff1a\u914d\u7f6e\u7528\u6237test\u62e5\u6709\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u6240\u4ee5\u6743\u9650\u7684\u6b65\u9aa4\uff1a a. \u914d\u7f6e test \u7528\u6237\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u8bbf\u95ee\u6743\u9650\u4e3a ALL\u3002 \u00a7 \u8bf4\u660e\uff1a \u00a7 \u89c4\u5219\u914d\u7f6e\u9875\u9762\u4e2d\uff0c\u914d\u7f6e\u7684 path \u6ca1\u6709 oss:// \u7684\u524d\u7f00\u3002 recursive\u6309\u94ae\u4e0d\u53ef\u5173\u95ed\u3002 \u8def\u5f84\u672b\u5c3e\u65e0\u9700\u5e26\u6b63\u659c\u7ebf\uff08/\uff09\u3002 b. \u9700\u8981\u914d\u7f6e\u8bbf\u95ee\u8def\u5f84\u7684\u7236\u76ee\u5f55 oss://bucket-test-hangzhou/user \u7684\u6743\u9650\u4e3a Execute\u3002 \u00a7 8. \u8bbf\u95eeOSS\u3002 \u00a7 \u82e5\u7528\u6237\u8bbf\u95ee Ranger \u6ca1\u6709\u6388\u6743\u7684\u8def\u5f84\uff0c\u5219\u4f1a\u62a5\u5982\u4e0b\u9519\u8bef\uff1a org.apache.hadoop.security.AccessControlException: Permission denied: user=test, access=READ_EXECUTE, resourcePath=\"bucket-test-hangzhou/\"","title":"\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#oss-ranger","text":"\u672c\u6587\u4ecb\u7ecd\u5728\u963f\u91cc\u4e91 E-MapReduce \u96c6\u7fa4\u914d\u7f6e\u963f\u91cc\u4e91 OSS \u548c OSS-HDFS \u670d\u52a1\u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\u3002","title":"\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#_1","text":"Apache Ranger \u63d0\u4f9b\u96c6\u4e2d\u5f0f\u7684\u6743\u9650\u7ba1\u7406\u6846\u67b6\uff0c\u53ef\u4ee5\u5bf9 Hadoop \u751f\u6001\u4e2d\u7684\u591a\u4e2a\u7ec4\u4ef6\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u8bbf\u95ee\u63a7\u5236\u3002\u5f53\u7528\u6237\u5c06\u6570\u636e\u5b58\u653e\u5728\u963f\u91cc\u4e91 OSS \u65f6\uff0c\u5219\u662f\u901a\u8fc7\u963f\u91cc\u4e91 RAM \u4ea7\u54c1\u521b\u5efa\u6216\u7ba1\u7406 RAM \u7528\u6237\uff0c\u5bf9RAM\u7528\u6237\u5b9e\u73b0\u5bf9 OSS \u8d44\u6e90\u7684\u8bbf\u95ee\u63a7\u5236\u3002 \u4e3a\u7ef4\u6301\u5927\u6570\u636e\u5ba2\u6237\u7684\u4f7f\u7528\u4e60\u60ef\uff0c\u901a\u8fc7 JindoFSx Namespace \u63a5\u5165 Ranger \u7684\u5ba2\u6237\u7aef\uff0c\u65b9\u4fbf\u7528\u6237\u7edf\u4e00\u7ba1\u7406\u5927\u6570\u636e\u7ec4\u4ef6\u6743\u9650\u3002","title":"\u80cc\u666f"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#oss","text":"OSS \u7684\u8bbf\u95ee\u5bc6\u94a5 AccessKey\uff08AK\uff09\u7edf\u4e00\u5728 JindoFSx Namespace \u4e2d\u8bbe\u7f6e\uff0c\u907f\u514d\u7528\u6237\u5728\u5ba2\u6237\u7aef\u914d\u7f6e\u660e\u6587\u5bc6\u94a5\uff0c\u5efa\u8bae\u53ea\u5141\u8bb8\u7ba1\u7406\u5458\u64cd\u4f5c\u548c\u7ba1\u7406 JindoFSx Namespace \u670d\u52a1\uff1b JindoSDK \u63d0\u4f9b\u6807\u51c6 Hadoop Filesystem \u5ba2\u6237\u7aef\uff0c\u5c06\u8bbf\u95ee OSS \u7684\u8bf7\u6c42\u4f1a\u53d1\u9001\u81f3 Namespace \u670d\u52a1\uff1b JindoFSx Namespace Service \u8d1f\u8d23\u96c6\u6210 Ranger \u5ba2\u6237\u7aef\uff0c\u5468\u671f\u6027\u5c06\u6743\u9650\u7b56\u7565\u4ece Ranger \u670d\u52a1\u7aef\u540c\u6b65\u5230\u672c\u5730\uff1b Namespace \u670d\u52a1\u5728\u6536\u5230 JindoSDK \u7684\u9274\u6743\u8bf7\u6c42\u540e\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u6821\u9a8c\uff1b \u901a\u8fc7\u6743\u9650\u6821\u9a8c\u540e JindoSDK \u5219\u53ef\u4ee5\u4f7f\u7528 JindoFSx Namespace \u670d\u52a1\u9881\u53d1\u7684\u4e34\u65f6 AK \u8bbf\u95ee OSS\u3002 \u6ce8\uff1a\u82e5 Namespace \u90e8\u7f72\u7684\u8282\u70b9\u4e3a\u963f\u91cc\u4e91 ECS \u8282\u70b9\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u5b89\u5168\u7ec4\uff0c\u9650\u5236\u8bbf\u95ee Namespace \u7684\u5ba2\u6237\u7aef\u3002","title":"\u8bbf\u95ee OSS \u9274\u6743\u6d41\u7a0b"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#_2","text":"\u5df2\u521b\u5efa E-MapReduce EMR-5.6.0/EMR-3.40.0 \u6216\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\uff0c\u5e76\u4e14\u9009\u62e9\u4e86 Ranger \u670d\u52a1\u3002 \u542f\u7528 Kerberos \u8ba4\u8bc1\u6ee1\u8db3\u8ba4\u8bc1\u9700\u6c42\u3002 \u82e5\u4f7f\u7528\u96c6\u7fa4\u90fd\u662f\u53ef\u4fe1\u7528\u6237\uff0c\u5219\u53ef\u4e0d\u542f\u7528 Kerberos \u8ba4\u8bc1\u3002 \u6ce8\u610f\uff1aEMR \u96c6\u7fa4\u5f00\u542f Kerberos \u540e\uff0c\u5c06\u65e0\u6cd5\u5173\u95ed Kerberos \u8ba4\u8bc1\uff0c\u8bf7\u8c28\u614e\u9009\u62e9\u3002\u5f00\u542f Kerberos \u540e\uff0c\u96c6\u7fa4\u6240\u6709\u5927\u6570\u636e\u7ec4\u4ef6\u7684\u670d\u52a1\u5747\u9700\u8981\u7ecf\u8fc7 Kerberos \u8ba4\u8bc1\uff0c\u5927\u6570\u636e\u4f5c\u4e1a\u63d0\u4ea4\u5230\u96c6\u7fa4\u4f1a\u5148\u7ecf\u8fc7\u8eab\u4efd\u8ba4\u8bc1\u3002Kerberos \u8be6\u60c5\u4fe1\u606f\uff0c\u8bf7\u53c2\u89c1 \u9ad8\u5b89\u5168\u7c7b\u578b\u96c6\u7fa4","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#_3","text":"\u521b\u5efa\u96c6\u7fa4\u65f6\uff0c\u5728\u8f6f\u4ef6\u914d\u7f6e\u9875\u9762\u7684\u9ad8\u7ea7\u8bbe\u7f6e\u533a\u57df\u4e2d\uff0c\u6253\u5f00Kerberos\u96c6\u7fa4\u6a21\u5f0f\u5f00\u5173\u3002 \u82e5\u6240\u5728\u96c6\u7fa4\u90fd\u662f\u53ef\u4fe1\u7528\u6237\uff0c\u5219\u53ef\u4e0d\u542f\u7528 Kerberos \u8ba4\u8bc1\u3002 \u3010\u6ce8\u610f\u3011\uff1a\u82e5\u4e3a\u975e\u9ad8\u5b89\u5168\u96c6\u7fa4\uff0c\u5219\u4e0d\u9700\u8981\u6b65\u9aa4 5 \u548c\u6b65\u9aa4 6\u3002","title":"\u521b\u5efa\u9ad8\u5b89\u5168\u96c6\u7fa4"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#1-jindofsx-namespace","text":"","title":"1. \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#11-jindo-sasl","text":"\u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 jindodata-4.6.12/plugins/ \u8def\u5f84\u4e0b\uff0c\u5c06\u8be5\u8def\u5f84\u7684\u7edd\u5bf9\u8def\u5f84\u6dfb\u52a0\u5230 JindoData \u7684 namespace \u4e2d\u3002 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.plugin.dir ${JINDDATA_HOME}/plugins/","title":"1.1 \u914d\u7f6e Jindo SASL \u63d2\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#12-ranger","text":"\u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.oss.authorization.method ranger","title":"1.2 \u542f\u7528 Ranger \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#2-jindofsx-namespace","text":"","title":"2. \u91cd\u542f JindoFSx Namespace \u670d\u52a1\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#31-xengine","text":"\u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.xengine jindofsx","title":"3.1 \u914d\u7f6e xengine"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#32-jindo-sasl","text":"\u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 jindosdk-4.6.12/plugins/ \u8def\u5f84\u4e0b\uff0c\u5c06\u8be5\u8def\u5f84\u7684\u7edd\u5bf9\u8def\u5f84\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 \u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.jdo.plugin.dir ${JINDOSDK_HOME}/plugins/","title":"3.2 \u914d\u7f6e Jindo SASL \u63d2\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#33-ranger","text":"\u652f\u6301\u7edf\u4e00\u914d\u7f6e\u548c\u6309 bucket \u914d\u7f6e\u4e24\u79cd\u65b9\u5f0f\u3002","title":"3.3 \u542f\u7528 Ranger \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#a-oss-oss-hdfs-bucket","text":"\u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.authorization.method ranger \u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site \uff0c\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider","title":"a. \u5bf9 OSS \u548c OSS-HDFS \u7684\u5168\u90e8 bucket \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#b-oss-oss-hdfs-bucket","text":"\u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site > \u81ea\u5b9a\u4e49\u914d\u7f6e \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.bucket.XXX.authorization.method ranger \u5728 HDFS \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site \uff0c\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.bucket.XXX.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider \u8bf4\u660e\uff1aXXX \u4e3a \u6216 OSS \u6216 OSS-HDFS \u670d\u52a1 bucket\u7684\u540d\u79f0\u3002","title":"b. \u5bf9 OSS \u6216 OSS-HDFS \u7684\u90e8\u5206 bucket \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#4-hiveserver2","text":"","title":"4. \u91cd\u542fHiveServer2\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#5-principal","text":"","title":"5. \u521b\u5efa\u7528\u6237 Principal\u3002\uff08\u9002\u7528\u4e8e\u9ad8\u5b89\u5168\u96c6\u7fa4\uff09"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#a-sshemr-header-1","text":"","title":"a. \u901a\u8fc7SSH\u65b9\u5f0f\u8fde\u63a5\u96c6\u7fa4\u7684emr-header-1\u8282\u70b9\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#b-kerberosadmin","text":"sh /usr/lib/has-current/bin/admin-local.sh /etc/ecm/has-conf -k /etc/ecm/has-conf/admin.keytab \u672c\u793a\u4f8b\u5bc6\u7801\u8bbe\u7f6e\u4e3a 123456\u3002 addprinc -pw 123456 test","title":"b. \u6267\u884c\u5982\u4e0b\u547d\u4ee4\uff0c\u8fdb\u5165Kerberos\u7684admin\u5de5\u5177\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#tgtprincipalkeytab","text":"","title":"\u8bf4\u660e: \u9700\u8981\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5728\u521b\u5efaTGT\u65f6\u4f1a\u7528\u5230\u3002\u5982\u679c\u60a8\u4e0d\u60f3\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5219\u53ef\u4ee5\u6267\u884c\u4e0b\u4e00\u6b65\uff0c\u628aPrincipal\u7684\u7528\u6237\u540d\u548c\u5bc6\u7801\u5bfc\u5165\u5230keytab\u6587\u4ef6\u4e2d\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#c-keytab","text":"ktadd -k /root/test.keytab test \u6267\u884c quit \u547d\u4ee4\uff0c\u53ef\u4ee5\u9000\u51fa Kerberos \u7684 admin \u5de5\u5177\u3002","title":"c. \u53ef\u9009\uff1a\u6267\u884c\u5982\u4e0b\u547d\u4ee4\uff0c\u751f\u6210keytab\u6587\u4ef6\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#6-tgt","text":"\u521b\u5efa TGT \u7684\u673a\u5668\uff0c\u53ef\u4ee5\u662f\u4efb\u610f\u4e00\u53f0\u9700\u8981\u8bbf\u95ee OSS \u7684\u673a\u5668\u3002","title":"6. \u521b\u5efa TGT\u3002\uff08\u9002\u7528\u4e8e\u9ad8\u5b89\u5168\u96c6\u7fa4\uff09"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#a-root-test","text":"useradd test","title":"a. \u4f7f\u7528 root \u7528\u6237\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u521b\u5efa test \u7528\u6237\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#b-test","text":"su test","title":"b. \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u5207\u6362\u4e3a test \u7528\u6237\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#c-tgt","text":"\u65b9\u5f0f\u4e00\uff1a\u4f7f\u7528\u7528\u6237\u540d\u548c\u5bc6\u7801\u65b9\u5f0f\uff0c\u521b\u5efa TGT\u3002 \u6267\u884c kinit \u547d\u4ee4\uff0c\u56de\u8f66\u540e\u8f93\u5165 test \u7684\u5bc6\u7801 123456\u3002 \u65b9\u5f0f\u4e8c\uff1a\u4f7f\u7528 keytab \u6587\u4ef6\uff0c\u521b\u5efa TGT\u3002 \u5728\u6b65\u9aa4 4 \u4e2d\u7684 test.keytab \u6587\u4ef6\uff0c\u5df2\u7ecf\u4fdd\u5b58\u5728 emr-header-1 \u673a\u5668\u7684 /root/ \u76ee\u5f55\u4e0b\uff0c\u9700\u8981\u4f7f\u7528 scp \u547d\u4ee4\u62f7\u8d1d\u5230\u5f53\u524d\u673a\u5668\u7684 /home/test/ \u76ee\u5f55\u4e0b\u3002 kinit -kt /home/test/test.keytab test","title":"c. \u751f\u6210TGT\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#d-tgt","text":"\u4f7f\u7528 klist \u547d\u4ee4\uff0c\u5982\u679c\u51fa\u73b0\u5982\u4e0b\u4fe1\u606f\uff0c\u5219\u8bf4\u660eTGT\u521b\u5efa\u6210\u529f\uff0c\u5373\u53ef\u4ee5\u8bbf\u95eeOSS\u4e86\u3002 Ticket cache: FILE:/tmp/krb5cc_1012 Default principal: test@EMR.23****.COM Valid starting Expires Service principal 03/27/2022 23:20:44 03/28/2022 23:20:44 krbtgt/EMR.238075.COM@EMR.238075.COM renew until 03/28/2022 23:20:44 \u6ce8\u610f: \u9700\u8981\u8bb0\u5f55\u4e0b\u56de\u663e\u4fe1\u606f EMR.23****.COM \u4e2d\u7684\u6570\u5b57 23**** \uff0c\u5373\u4e3a cluster_id \u7684\u503c\uff0c\u540e\u9762\u8bbf\u95ee OSS \u65f6\u9700\u8981\u3002","title":"d. \u67e5\u770bTGT\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#7-ranger-webui-oss","text":"","title":"7. \u5728 Ranger WebUI \u914d\u7f6e OSS \u6743\u9650"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#ranger","text":"\u4f8b\uff1a\u914d\u7f6e\u7528\u6237test\u62e5\u6709\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u6240\u4ee5\u6743\u9650\u7684\u6b65\u9aa4\uff1a","title":"Ranger \u89c4\u5219\u793a\u4f8b"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#a-test-ossbucket-test-hangzhouusertest-all","text":"","title":"a. \u914d\u7f6e test \u7528\u6237\u8bbf\u95eeoss://bucket-test-hangzhou/user/test\u76ee\u5f55\u7684\u8bbf\u95ee\u6743\u9650\u4e3a ALL\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#_4","text":"\u89c4\u5219\u914d\u7f6e\u9875\u9762\u4e2d\uff0c\u914d\u7f6e\u7684 path \u6ca1\u6709 oss:// \u7684\u524d\u7f00\u3002 recursive\u6309\u94ae\u4e0d\u53ef\u5173\u95ed\u3002 \u8def\u5f84\u672b\u5c3e\u65e0\u9700\u5e26\u6b63\u659c\u7ebf\uff08/\uff09\u3002","title":"\u8bf4\u660e\uff1a"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#b-ossbucket-test-hangzhouuser-execute","text":"","title":"b. \u9700\u8981\u914d\u7f6e\u8bbf\u95ee\u8def\u5f84\u7684\u7236\u76ee\u5f55oss://bucket-test-hangzhou/user\u7684\u6743\u9650\u4e3a Execute\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_config_emr_upgrade/#8-oss","text":"\u82e5\u7528\u6237\u8bbf\u95ee Ranger \u6ca1\u6709\u6388\u6743\u7684\u8def\u5f84\uff0c\u5219\u4f1a\u62a5\u5982\u4e0b\u9519\u8bef\uff1a org.apache.hadoop.security.AccessControlException: Permission denied: user=test, access=READ_EXECUTE, resourcePath=\"bucket-test-hangzhou/\"","title":"8. \u8bbf\u95eeOSS\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/","text":"EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u00a7 \u672c\u6587\u4ecb\u7ecd\u5728 EMR \u96c6\u7fa4\u914d\u7f6e\u963f\u91cc\u4e91 OSS \u548c OSS-HDFS \u670d\u52a1\u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\u3002 \u80cc\u666f \u00a7 Apache Ranger \u63d0\u4f9b\u96c6\u4e2d\u5f0f\u7684\u6743\u9650\u7ba1\u7406\u6846\u67b6\uff0c\u53ef\u4ee5\u5bf9 Hadoop \u751f\u6001\u4e2d\u7684\u591a\u4e2a\u7ec4\u4ef6\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u8bbf\u95ee\u63a7\u5236\u3002\u5f53\u7528\u6237\u5c06\u6570\u636e\u5b58\u653e\u5728\u963f\u91cc\u4e91 OSS \u65f6\uff0c\u5219\u662f\u901a\u8fc7\u963f\u91cc\u4e91 RAM \u4ea7\u54c1\u521b\u5efa\u6216\u7ba1\u7406 RAM \u7528\u6237\uff0c\u5bf9RAM\u7528\u6237\u5b9e\u73b0\u5bf9 OSS \u8d44\u6e90\u7684\u8bbf\u95ee\u63a7\u5236\u3002 \u4e3a\u7ef4\u6301\u5927\u6570\u636e\u5ba2\u6237\u7684\u4f7f\u7528\u4e60\u60ef\uff0c\u901a\u8fc7 JindoFSx Namespace \u63a5\u5165 Ranger \u7684\u5ba2\u6237\u7aef\uff0c\u65b9\u4fbf\u7528\u6237\u7edf\u4e00\u7ba1\u7406\u5927\u6570\u636e\u7ec4\u4ef6\u6743\u9650\u3002 \u8bbf\u95ee OSS \u9274\u6743\u6d41\u7a0b \u00a7 OSS \u7684\u8bbf\u95ee\u5bc6\u94a5 AccessKey\uff08AK\uff09\u7edf\u4e00\u5728 JindoFSx Namespace \u4e2d\u8bbe\u7f6e\uff0c\u907f\u514d\u7528\u6237\u5728\u5ba2\u6237\u7aef\u914d\u7f6e\u660e\u6587\u5bc6\u94a5\uff0c\u5efa\u8bae\u53ea\u5141\u8bb8\u7ba1\u7406\u5458\u64cd\u4f5c\u548c\u7ba1\u7406 JindoFSx Namespace \u670d\u52a1\uff1b JindoSDK \u63d0\u4f9b\u6807\u51c6 Hadoop Filesystem \u5ba2\u6237\u7aef\uff0c\u5c06\u8bbf\u95ee OSS \u7684\u8bf7\u6c42\u4f1a\u53d1\u9001\u81f3 Namespace \u670d\u52a1\uff1b JindoFSx Namespace Service \u8d1f\u8d23\u96c6\u6210 Ranger \u5ba2\u6237\u7aef\uff0c\u5468\u671f\u6027\u5c06\u6743\u9650\u7b56\u7565\u4ece Ranger \u670d\u52a1\u7aef\u540c\u6b65\u5230\u672c\u5730\uff1b Namespace \u670d\u52a1\u5728\u6536\u5230 JindoSDK \u7684\u9274\u6743\u8bf7\u6c42\u540e\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u6821\u9a8c\uff1b \u901a\u8fc7\u6743\u9650\u6821\u9a8c\u540e JindoSDK \u5219\u53ef\u4ee5\u4f7f\u7528 JindoFSx Namespace \u670d\u52a1\u9881\u53d1\u7684\u4e34\u65f6 AK \u8bbf\u95ee OSS\u3002 \u6ce8\uff1a\u82e5 Namespace \u90e8\u7f72\u7684\u8282\u70b9\u4e3a\u963f\u91cc\u4e91 ECS \u8282\u70b9\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u5b89\u5168\u7ec4\uff0c\u9650\u5236\u8bbf\u95ee Namespace \u7684\u5ba2\u6237\u7aef\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u521b\u5efa E-MapReduce EMR-5.9.1/EMR-3.43.1 \u6216\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\uff0c\u5e76\u4e14\u9009\u62e9\u4e86 Ranger \u670d\u52a1\u3002 \u542f\u7528 Kerberos \u8ba4\u8bc1\u6ee1\u8db3\u8ba4\u8bc1\u9700\u6c42\u3002 \u82e5\u4f7f\u7528\u96c6\u7fa4\u90fd\u662f\u53ef\u4fe1\u7528\u6237\uff0c\u5219\u53ef\u4e0d\u542f\u7528 Kerberos \u8ba4\u8bc1\u3002 \u6ce8\u610f\uff1aEMR \u96c6\u7fa4\u5f00\u542f Kerberos \u540e\uff0c\u5c06\u65e0\u6cd5\u5173\u95ed Kerberos \u8ba4\u8bc1\uff0c\u8bf7\u8c28\u614e\u9009\u62e9\u3002\u5f00\u542f Kerberos \u540e\uff0c\u96c6\u7fa4\u6240\u6709\u5927\u6570\u636e\u7ec4\u4ef6\u7684\u670d\u52a1\u5747\u9700\u8981\u7ecf\u8fc7 Kerberos \u8ba4\u8bc1\uff0c\u5927\u6570\u636e\u4f5c\u4e1a\u63d0\u4ea4\u5230\u96c6\u7fa4\u4f1a\u5148\u7ecf\u8fc7\u8eab\u4efd\u8ba4\u8bc1\u3002Kerberos \u8be6\u60c5\u4fe1\u606f\uff0c\u8bf7\u53c2\u89c1 \u9ad8\u5b89\u5168\u7c7b\u578b\u96c6\u7fa4 1. \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u3002 \u00a7 1.1 \u914d\u7f6e Jindo SASL \u63d2\u4ef6 \u00a7 \u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 /opt/apps/JINDODATA/jindodata-current/plugins \u8def\u5f84\u4e0b\u3002 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.plugin.dir /opt/apps/JINDODATA/jindodata-current/plugins 1.2 \u542f\u7528 Ranger \u9274\u6743 \u00a7 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.oss.authorization.method ranger 1.3 \u914d\u7f6e\u4f7f\u7528 Assume Role \u9881\u53d1\u4e34\u65f6 AK\uff08\u53ef\u9009\uff0c\u9ed8\u8ba4\u4f7f\u7528 ECS_ROLE \u514d\u5bc6\uff09 \u00a7 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.oss.credential.provider ASSUME_ROLE \u5728\u963f\u91cc\u4e91 RAM \u521b\u5efa\u7528\u6237\u9881\u53d1\u4e34\u65f6 AK \u7684 Role, \u9009\u62e9\u53ef\u4fe1\u5b9e\u4f53\u7c7b\u578b\u4e3a\"\u963f\u91cc\u4e91\u8d26\u53f7\", \u5e76\u4e3a\u89d2\u8272\u6388\u6743\u3002 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > common > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c default.credential.provider ASSUME_ROLE sts.access.key STS \u7684 accessKey sts.access.secret STS \u7684 accessKeySecret sts.access.endpoint STS \u7684 endpoint, \u5982\uff1a sts.cn-shanghai.aliyuncs.com sts.role.name \u5728 RAM \u4e2d\u521b\u5efa\u7684\u89d2\u8272\uff0c \u5982 JindoFsxTestRole sts.role.arn \u89d2\u8272\u7684ARN\uff0c\u5982\uff1aacs:ram::xxx:role/JindoFsxTestRole 1.4 \u914d\u7f6e Principal \u548c Keytab\uff08\u9002\u7528\u4e8e Kerberos \u96c6\u7fa4\uff09 \u00a7 a. \u521b\u5efa principal \u548c keytab \u00a7 \u5982\u679c\u60a8\u662f\u4f7f\u7528root\u7528\u6237\uff0c\u767b\u5f55KDC\uff08Kerberos\u7684\u670d\u52a1\u7aef\u7a0b\u5e8f\uff09\u6240\u5728\u7684master-1-1\u8282\u70b9\uff0c\u5219\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u76f4\u63a5\u8fdb\u5165admin\u5de5\u5177\u3002 kadmin.local \u5f53\u8fd4\u56de\u4fe1\u606f\u4e2d\u5305\u542b\u5982\u4e0b\u4fe1\u606f\u65f6\uff0c\u8868\u793a\u5df2\u8fdb\u5165admin.local\u547d\u4ee4\u884c\u3002 Authenticating as principal hadoop/admin@EMR.C-85D4B8D74296****.COM with password. kadmin.local: \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u521b\u5efa\u7528\u6237\u540d\u4e3a jindodata \u7684 principal, \u683c\u5f0f\u4e3a jindodata/ . .emr.aliyuncs.com\u3002 addprinc -randkey jindodata/master-1-1.c-9e4fc2dcc****.cn-shanghai.emr.aliyuncs.com \u5bfc\u51fa keytab \u6587\u4ef6\u3002 ktadd -k jindodata.keytab jindodata/master-1-1.c-9e4fc2dcc****.cn-shanghai.emr.aliyuncs.com b. \u914d\u7f6e \u00a7 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.authentication.enable true namespace.authentication.keytab \u914d\u7f6e keytab \u6240\u5728\u7684\u8def\u5f84\uff0c \u5982\uff1a /opt/apps/JINDODATA/jindodata-current/conf/jindodata.keytab namespace.authentication.principal \u914d\u7f6e keytab \u4e2d\u7684 principal, \u5982\uff1a jindodata/master-1-1.c-9e4fc2dcc *.cn-shanghai.emr.aliyuncs.com@EMR.C-85D4B8D74296 *.COM 1.5 \u914d\u7f6e Ranger Admin Service URL \u00a7 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c ranger.plugin.jindo-auth.policy.rest.url \u914d\u7f6e\u83b7\u53d6 Ranger Policy \u7684 URL, \u5982\uff1ahttp://master-1-1:6080\uff0c \u9ed8\u8ba4\u4e3a http://emr-header-1:6080 1.6 \u91cd\u542f JindoFSx Namespace \u670d\u52a1\u3002 \u00a7 2. \u914d\u7f6e JindoSDK\u3002 \u00a7 2.1 \u914d\u7f6e xengine \u00a7 \u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \uff0c \u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.xengine jindofsx 2.2 \u914d\u7f6e Jindo SASL \u63d2\u4ef6 \u00a7 \u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 /opt/apps/JINDOSDK/jindosdk-current/plugins/ \u8def\u5f84\u4e0b\uff0c \u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c \u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.jdo.plugin.dir /opt/apps/JINDOSDK/jindosdk-current/plugins 2.3 \u542f\u7528 Ranger \u9274\u6743 \u00a7 \u652f\u6301\u7edf\u4e00\u914d\u7f6e\u548c\u6309 bucket \u914d\u7f6e\u4e24\u79cd\u65b9\u5f0f\u3002 a. \u5bf9 OSS \u548c OSS-HDFS \u7684\u5168\u90e8 bucket \u9274\u6743 \u00a7 \u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \uff0c \u65b0\u589e\u6216\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.authorization.method ranger fs.oss.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider b. \u5bf9 OSS \u6216 OSS-HDFS \u7684\u90e8\u5206 bucket \u9274\u6743 \u00a7 \u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \uff0c \u65b0\u589e\u6216\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.bucket.XXX.authorization.method ranger fs.oss.bucket.XXX.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider \u8bf4\u660e\uff1aXXX \u4e3a \u6216 OSS \u6216 OSS-HDFS \u670d\u52a1 bucket\u7684\u540d\u79f0\u3002 2.4. \u91cd\u542f HiveServer2 \u7b49\u6240\u6709\u5e38\u9a7b\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 3. \u521b\u5efa\u7528\u6237 Principal\u3002 \u00a7 \u5982\u679c\u60a8\u662f\u4f7f\u7528root\u7528\u6237\uff0c\u767b\u5f55KDC\uff08Kerberos\u7684\u670d\u52a1\u7aef\u7a0b\u5e8f\uff09\u6240\u5728\u7684 master-1-1 \u8282\u70b9\uff0c\u5219\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u76f4\u63a5\u8fdb\u5165admin\u5de5\u5177\u3002 kadmin.local \u5f53\u8fd4\u56de\u4fe1\u606f\u4e2d\u5305\u542b\u5982\u4e0b\u4fe1\u606f\u65f6\uff0c\u8868\u793a\u5df2\u8fdb\u5165admin.local\u547d\u4ee4\u884c\u3002 Authenticating as principal hadoop/admin@EMR.C-85D4B8D74296****.COM with password. kadmin.local: \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u521b\u5efa\u7528\u6237\u540d\u4e3a test \u7684 principal\u3002 addprinc -pw 123456 test 4. \u521b\u5efa TGT\u3002 \u00a7 \u521b\u5efa TGT \u7684\u673a\u5668\uff0c\u53ef\u4ee5\u662f\u4efb\u610f\u4e00\u53f0\u9700\u8981\u8bbf\u95ee OSS \u7684\u673a\u5668\u3002 a. \u4f7f\u7528 root \u7528\u6237\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u521b\u5efa test \u7528\u6237\u3002 \u00a7 useradd test \u8bf4\u660e: \u9700\u8981\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5728\u521b\u5efaTGT\u65f6\u4f1a\u7528\u5230\u3002\u5982\u679c\u60a8\u4e0d\u60f3\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5219\u53ef\u4ee5\u6267\u884c\u4e0b\u4e00\u6b65\uff0c\u628a principal \u7684\u7528\u6237\u540d\u548c\u5bc6\u7801\u5bfc\u5165\u5230 keytab \u6587\u4ef6\u4e2d\u3002 \u00a7 b. \u53ef\u9009\uff1a\u6267\u884c\u5982\u4e0b\u547d\u4ee4\uff0c\u751f\u6210keytab\u6587\u4ef6\u3002 \u00a7 ktadd -k /root/test.keytab test c. \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u5207\u6362\u4e3a test \u7528\u6237\u3002 \u00a7 su test d. \u751f\u6210TGT\u3002 \u00a7 \u65b9\u5f0f\u4e00\uff1a\u4f7f\u7528\u7528\u6237\u540d\u548c\u5bc6\u7801\u65b9\u5f0f\uff0c\u521b\u5efa TGT\u3002 \u6267\u884c kinit \u547d\u4ee4\uff0c\u56de\u8f66\u540e\u8f93\u5165 test \u7684\u5bc6\u7801 123456\u3002 \u65b9\u5f0f\u4e8c\uff1a\u4f7f\u7528 keytab \u6587\u4ef6\uff0c\u521b\u5efa TGT\u3002 \u5728\u6b65\u9aa4 4 \u4e2d\u7684 test.keytab \u6587\u4ef6\uff0c\u5df2\u7ecf\u4fdd\u5b58\u5728 emr-header-1 \u673a\u5668\u7684 /root/ \u76ee\u5f55\u4e0b\uff0c\u9700\u8981\u4f7f\u7528 scp \u547d\u4ee4\u62f7\u8d1d\u5230\u5f53\u524d\u673a\u5668\u7684 /home/test/ \u76ee\u5f55\u4e0b\u3002 kinit -kt /home/test/test.keytab test e. \u67e5\u770bTGT\u3002 \u00a7 \u4f7f\u7528 klist \u547d\u4ee4\uff0c\u5982\u679c\u51fa\u73b0\u5982\u4e0b\u4fe1\u606f\uff0c\u5219\u8bf4\u660eTGT\u521b\u5efa\u6210\u529f\uff0c\u5373\u53ef\u4ee5\u8bbf\u95eeOSS\u4e86\u3002 Ticket cache: FILE:/tmp/krb5cc_1012 Default principal: test@EMR.C-85D4B8D74296****.COM Valid starting Expires Service principal 11/15/22 15:22:57 11/16/22 15:22:57 krbtgt/EMR.EMR.C-85D4B8D74296****.COM@EMR.C-85D4B8D74296****.COM renew until 11/22/22 15:22:57 5. \u5728 Ranger WebUI \u914d\u7f6e OSS \u6743\u9650 \u00a7 a. \u521b\u5efa\u4e00\u4e2a\u540d\u4e3a emr-oss \u7684 service\uff0c \u5e76\u6dfb\u52a0\u4e0b\u5217\u914d\u7f6e\u3002 \u00a7 \u53c2\u6570 \u503c policy.download.auth.users jindodata \u521b\u5efa\u5b8c\u6210\u540e\u5982\u4e0b\uff0c\u540e\u7eed\u5c31\u53ef\u4ee5\u5728\u8be5Service\u4e0b\u521b\u5efaPolicy\u3002 b. Ranger \u89c4\u5219\u793a\u4f8b \u00a7 \u4f8b\uff1a\u914d\u7f6e\u7528\u6237test\u62e5\u6709\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u6240\u4ee5\u6743\u9650\u7684\u6b65\u9aa4\uff1a \u914d\u7f6e test \u7528\u6237\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u8bbf\u95ee\u6743\u9650\u4e3a ALL\u3002 \u00a7 \u8bf4\u660e\uff1a \u00a7 \u89c4\u5219\u914d\u7f6e\u9875\u9762\u4e2d\uff0c\u914d\u7f6e\u7684 path \u6ca1\u6709 oss:// \u7684\u524d\u7f00\u3002 recursive\u6309\u94ae\u4e0d\u53ef\u5173\u95ed\u3002 \u8def\u5f84\u672b\u5c3e\u65e0\u9700\u5e26\u6b63\u659c\u7ebf\uff08/\uff09\u3002 6. \u8bbf\u95eeOSS\u3002 \u00a7 \u82e5\u7528\u6237\u8bbf\u95ee Ranger \u6ca1\u6709\u6388\u6743\u7684\u8def\u5f84\uff0c\u5219\u4f1a\u62a5\u5982\u4e0b\u9519\u8bef\uff1a org.apache.hadoop.security.AccessControlException: Permission denied: user=test, access=READ_EXECUTE, resourcePath=\"bucket-test-hangzhou/\"","title":"EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#emr-oss-ranger","text":"\u672c\u6587\u4ecb\u7ecd\u5728 EMR \u96c6\u7fa4\u914d\u7f6e\u963f\u91cc\u4e91 OSS \u548c OSS-HDFS \u670d\u52a1\u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\u3002","title":"EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#_1","text":"Apache Ranger \u63d0\u4f9b\u96c6\u4e2d\u5f0f\u7684\u6743\u9650\u7ba1\u7406\u6846\u67b6\uff0c\u53ef\u4ee5\u5bf9 Hadoop \u751f\u6001\u4e2d\u7684\u591a\u4e2a\u7ec4\u4ef6\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u8bbf\u95ee\u63a7\u5236\u3002\u5f53\u7528\u6237\u5c06\u6570\u636e\u5b58\u653e\u5728\u963f\u91cc\u4e91 OSS \u65f6\uff0c\u5219\u662f\u901a\u8fc7\u963f\u91cc\u4e91 RAM \u4ea7\u54c1\u521b\u5efa\u6216\u7ba1\u7406 RAM \u7528\u6237\uff0c\u5bf9RAM\u7528\u6237\u5b9e\u73b0\u5bf9 OSS \u8d44\u6e90\u7684\u8bbf\u95ee\u63a7\u5236\u3002 \u4e3a\u7ef4\u6301\u5927\u6570\u636e\u5ba2\u6237\u7684\u4f7f\u7528\u4e60\u60ef\uff0c\u901a\u8fc7 JindoFSx Namespace \u63a5\u5165 Ranger \u7684\u5ba2\u6237\u7aef\uff0c\u65b9\u4fbf\u7528\u6237\u7edf\u4e00\u7ba1\u7406\u5927\u6570\u636e\u7ec4\u4ef6\u6743\u9650\u3002","title":"\u80cc\u666f"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#oss","text":"OSS \u7684\u8bbf\u95ee\u5bc6\u94a5 AccessKey\uff08AK\uff09\u7edf\u4e00\u5728 JindoFSx Namespace \u4e2d\u8bbe\u7f6e\uff0c\u907f\u514d\u7528\u6237\u5728\u5ba2\u6237\u7aef\u914d\u7f6e\u660e\u6587\u5bc6\u94a5\uff0c\u5efa\u8bae\u53ea\u5141\u8bb8\u7ba1\u7406\u5458\u64cd\u4f5c\u548c\u7ba1\u7406 JindoFSx Namespace \u670d\u52a1\uff1b JindoSDK \u63d0\u4f9b\u6807\u51c6 Hadoop Filesystem \u5ba2\u6237\u7aef\uff0c\u5c06\u8bbf\u95ee OSS \u7684\u8bf7\u6c42\u4f1a\u53d1\u9001\u81f3 Namespace \u670d\u52a1\uff1b JindoFSx Namespace Service \u8d1f\u8d23\u96c6\u6210 Ranger \u5ba2\u6237\u7aef\uff0c\u5468\u671f\u6027\u5c06\u6743\u9650\u7b56\u7565\u4ece Ranger \u670d\u52a1\u7aef\u540c\u6b65\u5230\u672c\u5730\uff1b Namespace \u670d\u52a1\u5728\u6536\u5230 JindoSDK \u7684\u9274\u6743\u8bf7\u6c42\u540e\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u6821\u9a8c\uff1b \u901a\u8fc7\u6743\u9650\u6821\u9a8c\u540e JindoSDK \u5219\u53ef\u4ee5\u4f7f\u7528 JindoFSx Namespace \u670d\u52a1\u9881\u53d1\u7684\u4e34\u65f6 AK \u8bbf\u95ee OSS\u3002 \u6ce8\uff1a\u82e5 Namespace \u90e8\u7f72\u7684\u8282\u70b9\u4e3a\u963f\u91cc\u4e91 ECS \u8282\u70b9\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u5b89\u5168\u7ec4\uff0c\u9650\u5236\u8bbf\u95ee Namespace \u7684\u5ba2\u6237\u7aef\u3002","title":"\u8bbf\u95ee OSS \u9274\u6743\u6d41\u7a0b"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#_2","text":"\u5df2\u521b\u5efa E-MapReduce EMR-5.9.1/EMR-3.43.1 \u6216\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\uff0c\u5e76\u4e14\u9009\u62e9\u4e86 Ranger \u670d\u52a1\u3002 \u542f\u7528 Kerberos \u8ba4\u8bc1\u6ee1\u8db3\u8ba4\u8bc1\u9700\u6c42\u3002 \u82e5\u4f7f\u7528\u96c6\u7fa4\u90fd\u662f\u53ef\u4fe1\u7528\u6237\uff0c\u5219\u53ef\u4e0d\u542f\u7528 Kerberos \u8ba4\u8bc1\u3002 \u6ce8\u610f\uff1aEMR \u96c6\u7fa4\u5f00\u542f Kerberos \u540e\uff0c\u5c06\u65e0\u6cd5\u5173\u95ed Kerberos \u8ba4\u8bc1\uff0c\u8bf7\u8c28\u614e\u9009\u62e9\u3002\u5f00\u542f Kerberos \u540e\uff0c\u96c6\u7fa4\u6240\u6709\u5927\u6570\u636e\u7ec4\u4ef6\u7684\u670d\u52a1\u5747\u9700\u8981\u7ecf\u8fc7 Kerberos \u8ba4\u8bc1\uff0c\u5927\u6570\u636e\u4f5c\u4e1a\u63d0\u4ea4\u5230\u96c6\u7fa4\u4f1a\u5148\u7ecf\u8fc7\u8eab\u4efd\u8ba4\u8bc1\u3002Kerberos \u8be6\u60c5\u4fe1\u606f\uff0c\u8bf7\u53c2\u89c1 \u9ad8\u5b89\u5168\u7c7b\u578b\u96c6\u7fa4","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#1-jindofsx-namespace","text":"","title":"1. \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#11-jindo-sasl","text":"\u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 /opt/apps/JINDODATA/jindodata-current/plugins \u8def\u5f84\u4e0b\u3002 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.plugin.dir /opt/apps/JINDODATA/jindodata-current/plugins","title":"1.1 \u914d\u7f6e Jindo SASL \u63d2\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#12-ranger","text":"\u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.oss.authorization.method ranger","title":"1.2 \u542f\u7528 Ranger \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#13-assume-role-ak-ecs_role","text":"\u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.oss.credential.provider ASSUME_ROLE \u5728\u963f\u91cc\u4e91 RAM \u521b\u5efa\u7528\u6237\u9881\u53d1\u4e34\u65f6 AK \u7684 Role, \u9009\u62e9\u53ef\u4fe1\u5b9e\u4f53\u7c7b\u578b\u4e3a\"\u963f\u91cc\u4e91\u8d26\u53f7\", \u5e76\u4e3a\u89d2\u8272\u6388\u6743\u3002 \u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > common > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c default.credential.provider ASSUME_ROLE sts.access.key STS \u7684 accessKey sts.access.secret STS \u7684 accessKeySecret sts.access.endpoint STS \u7684 endpoint, \u5982\uff1a sts.cn-shanghai.aliyuncs.com sts.role.name \u5728 RAM \u4e2d\u521b\u5efa\u7684\u89d2\u8272\uff0c \u5982 JindoFsxTestRole sts.role.arn \u89d2\u8272\u7684ARN\uff0c\u5982\uff1aacs:ram::xxx:role/JindoFsxTestRole","title":"1.3 \u914d\u7f6e\u4f7f\u7528 Assume Role \u9881\u53d1\u4e34\u65f6 AK\uff08\u53ef\u9009\uff0c\u9ed8\u8ba4\u4f7f\u7528 ECS_ROLE \u514d\u5bc6\uff09"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#14-principal-keytab-kerberos","text":"","title":"1.4 \u914d\u7f6e Principal \u548c Keytab\uff08\u9002\u7528\u4e8e Kerberos \u96c6\u7fa4\uff09"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#a-principal-keytab","text":"\u5982\u679c\u60a8\u662f\u4f7f\u7528root\u7528\u6237\uff0c\u767b\u5f55KDC\uff08Kerberos\u7684\u670d\u52a1\u7aef\u7a0b\u5e8f\uff09\u6240\u5728\u7684master-1-1\u8282\u70b9\uff0c\u5219\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u76f4\u63a5\u8fdb\u5165admin\u5de5\u5177\u3002 kadmin.local \u5f53\u8fd4\u56de\u4fe1\u606f\u4e2d\u5305\u542b\u5982\u4e0b\u4fe1\u606f\u65f6\uff0c\u8868\u793a\u5df2\u8fdb\u5165admin.local\u547d\u4ee4\u884c\u3002 Authenticating as principal hadoop/admin@EMR.C-85D4B8D74296****.COM with password. kadmin.local: \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u521b\u5efa\u7528\u6237\u540d\u4e3a jindodata \u7684 principal, \u683c\u5f0f\u4e3a jindodata/ . .emr.aliyuncs.com\u3002 addprinc -randkey jindodata/master-1-1.c-9e4fc2dcc****.cn-shanghai.emr.aliyuncs.com \u5bfc\u51fa keytab \u6587\u4ef6\u3002 ktadd -k jindodata.keytab jindodata/master-1-1.c-9e4fc2dcc****.cn-shanghai.emr.aliyuncs.com","title":"a. \u521b\u5efa principal \u548c keytab"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#b","text":"\u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c namespace.authentication.enable true namespace.authentication.keytab \u914d\u7f6e keytab \u6240\u5728\u7684\u8def\u5f84\uff0c \u5982\uff1a /opt/apps/JINDODATA/jindodata-current/conf/jindodata.keytab namespace.authentication.principal \u914d\u7f6e keytab \u4e2d\u7684 principal, \u5982\uff1a jindodata/master-1-1.c-9e4fc2dcc *.cn-shanghai.emr.aliyuncs.com@EMR.C-85D4B8D74296 *.COM","title":"b. \u914d\u7f6e"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#15-ranger-admin-service-url","text":"\u5728 JindoData \u670d\u52a1\u9875\u9762\u9009\u62e9 \u914d\u7f6e > namespace > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c\u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c ranger.plugin.jindo-auth.policy.rest.url \u914d\u7f6e\u83b7\u53d6 Ranger Policy \u7684 URL, \u5982\uff1ahttp://master-1-1:6080\uff0c \u9ed8\u8ba4\u4e3a http://emr-header-1:6080","title":"1.5 \u914d\u7f6e Ranger Admin Service URL"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#16-jindofsx-namespace","text":"","title":"1.6 \u91cd\u542f JindoFSx Namespace \u670d\u52a1\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#2-jindosdk","text":"","title":"2. \u914d\u7f6e JindoSDK\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#21-xengine","text":"\u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \uff0c \u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.xengine jindofsx","title":"2.1 \u914d\u7f6e xengine"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#22-jindo-sasl","text":"\u8be5\u529f\u80fd\u4f9d\u8d56 Jindo SASL \u52a8\u6001\u5e93\uff0c\u9700\u8981\u914d\u7f6e\u8def\u5f84\u4ee5\u52a0\u8f7d\u52a8\u6001\u5e93\uff0c\u52a8\u6001\u5e93\u5728 /opt/apps/JINDOSDK/jindosdk-current/plugins/ \u8def\u5f84\u4e0b\uff0c \u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml > \u65b0\u589e\u914d\u7f6e\u9879 \uff0c \u6dfb\u52a0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.jdo.plugin.dir /opt/apps/JINDOSDK/jindosdk-current/plugins","title":"2.2 \u914d\u7f6e Jindo SASL \u63d2\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#23-ranger","text":"\u652f\u6301\u7edf\u4e00\u914d\u7f6e\u548c\u6309 bucket \u914d\u7f6e\u4e24\u79cd\u65b9\u5f0f\u3002","title":"2.3 \u542f\u7528 Ranger \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#a-oss-oss-hdfs-bucket","text":"\u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \uff0c \u65b0\u589e\u6216\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.authorization.method ranger fs.oss.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider","title":"a. \u5bf9 OSS \u548c OSS-HDFS \u7684\u5168\u90e8 bucket \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#b-oss-oss-hdfs-bucket","text":"\u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \uff0c \u65b0\u589e\u6216\u66f4\u65b0\u5982\u4e0b\u53c2\u6570\u3002 \u53c2\u6570 \u503c fs.oss.bucket.XXX.authorization.method ranger fs.oss.bucket.XXX.credentials.provider com.aliyun.jindodata.oss.auth.RangerCredentialsProvider \u8bf4\u660e\uff1aXXX \u4e3a \u6216 OSS \u6216 OSS-HDFS \u670d\u52a1 bucket\u7684\u540d\u79f0\u3002","title":"b. \u5bf9 OSS \u6216 OSS-HDFS \u7684\u90e8\u5206 bucket \u9274\u6743"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#24-hiveserver2","text":"","title":"2.4. \u91cd\u542f HiveServer2 \u7b49\u6240\u6709\u5e38\u9a7b\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#3-principal","text":"\u5982\u679c\u60a8\u662f\u4f7f\u7528root\u7528\u6237\uff0c\u767b\u5f55KDC\uff08Kerberos\u7684\u670d\u52a1\u7aef\u7a0b\u5e8f\uff09\u6240\u5728\u7684 master-1-1 \u8282\u70b9\uff0c\u5219\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u76f4\u63a5\u8fdb\u5165admin\u5de5\u5177\u3002 kadmin.local \u5f53\u8fd4\u56de\u4fe1\u606f\u4e2d\u5305\u542b\u5982\u4e0b\u4fe1\u606f\u65f6\uff0c\u8868\u793a\u5df2\u8fdb\u5165admin.local\u547d\u4ee4\u884c\u3002 Authenticating as principal hadoop/admin@EMR.C-85D4B8D74296****.COM with password. kadmin.local: \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u521b\u5efa\u7528\u6237\u540d\u4e3a test \u7684 principal\u3002 addprinc -pw 123456 test","title":"3. \u521b\u5efa\u7528\u6237 Principal\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#4-tgt","text":"\u521b\u5efa TGT \u7684\u673a\u5668\uff0c\u53ef\u4ee5\u662f\u4efb\u610f\u4e00\u53f0\u9700\u8981\u8bbf\u95ee OSS \u7684\u673a\u5668\u3002","title":"4. \u521b\u5efa TGT\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#a-root-test","text":"useradd test","title":"a. \u4f7f\u7528 root \u7528\u6237\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u521b\u5efa test \u7528\u6237\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#tgt-principal-keytab","text":"","title":"\u8bf4\u660e: \u9700\u8981\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5728\u521b\u5efaTGT\u65f6\u4f1a\u7528\u5230\u3002\u5982\u679c\u60a8\u4e0d\u60f3\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5219\u53ef\u4ee5\u6267\u884c\u4e0b\u4e00\u6b65\uff0c\u628a principal \u7684\u7528\u6237\u540d\u548c\u5bc6\u7801\u5bfc\u5165\u5230 keytab \u6587\u4ef6\u4e2d\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#b-keytab","text":"ktadd -k /root/test.keytab test","title":"b. \u53ef\u9009\uff1a\u6267\u884c\u5982\u4e0b\u547d\u4ee4\uff0c\u751f\u6210keytab\u6587\u4ef6\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#c-test","text":"su test","title":"c. \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u5207\u6362\u4e3a test \u7528\u6237\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#d-tgt","text":"\u65b9\u5f0f\u4e00\uff1a\u4f7f\u7528\u7528\u6237\u540d\u548c\u5bc6\u7801\u65b9\u5f0f\uff0c\u521b\u5efa TGT\u3002 \u6267\u884c kinit \u547d\u4ee4\uff0c\u56de\u8f66\u540e\u8f93\u5165 test \u7684\u5bc6\u7801 123456\u3002 \u65b9\u5f0f\u4e8c\uff1a\u4f7f\u7528 keytab \u6587\u4ef6\uff0c\u521b\u5efa TGT\u3002 \u5728\u6b65\u9aa4 4 \u4e2d\u7684 test.keytab \u6587\u4ef6\uff0c\u5df2\u7ecf\u4fdd\u5b58\u5728 emr-header-1 \u673a\u5668\u7684 /root/ \u76ee\u5f55\u4e0b\uff0c\u9700\u8981\u4f7f\u7528 scp \u547d\u4ee4\u62f7\u8d1d\u5230\u5f53\u524d\u673a\u5668\u7684 /home/test/ \u76ee\u5f55\u4e0b\u3002 kinit -kt /home/test/test.keytab test","title":"d. \u751f\u6210TGT\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#e-tgt","text":"\u4f7f\u7528 klist \u547d\u4ee4\uff0c\u5982\u679c\u51fa\u73b0\u5982\u4e0b\u4fe1\u606f\uff0c\u5219\u8bf4\u660eTGT\u521b\u5efa\u6210\u529f\uff0c\u5373\u53ef\u4ee5\u8bbf\u95eeOSS\u4e86\u3002 Ticket cache: FILE:/tmp/krb5cc_1012 Default principal: test@EMR.C-85D4B8D74296****.COM Valid starting Expires Service principal 11/15/22 15:22:57 11/16/22 15:22:57 krbtgt/EMR.EMR.C-85D4B8D74296****.COM@EMR.C-85D4B8D74296****.COM renew until 11/22/22 15:22:57","title":"e. \u67e5\u770bTGT\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#5-ranger-webui-oss","text":"","title":"5. \u5728 Ranger WebUI \u914d\u7f6e OSS \u6743\u9650"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#a-emr-oss-service","text":"\u53c2\u6570 \u503c policy.download.auth.users jindodata \u521b\u5efa\u5b8c\u6210\u540e\u5982\u4e0b\uff0c\u540e\u7eed\u5c31\u53ef\u4ee5\u5728\u8be5Service\u4e0b\u521b\u5efaPolicy\u3002","title":"a. \u521b\u5efa\u4e00\u4e2a\u540d\u4e3a emr-oss \u7684 service\uff0c \u5e76\u6dfb\u52a0\u4e0b\u5217\u914d\u7f6e\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#b-ranger","text":"\u4f8b\uff1a\u914d\u7f6e\u7528\u6237test\u62e5\u6709\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u6240\u4ee5\u6743\u9650\u7684\u6b65\u9aa4\uff1a","title":"b. Ranger \u89c4\u5219\u793a\u4f8b"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#test-ossbucket-test-hangzhouusertest-all","text":"","title":"\u914d\u7f6e test \u7528\u6237\u8bbf\u95eeoss://bucket-test-hangzhou/user/test\u76ee\u5f55\u7684\u8bbf\u95ee\u6743\u9650\u4e3a ALL\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#_3","text":"\u89c4\u5219\u914d\u7f6e\u9875\u9762\u4e2d\uff0c\u914d\u7f6e\u7684 path \u6ca1\u6709 oss:// \u7684\u524d\u7f00\u3002 recursive\u6309\u94ae\u4e0d\u53ef\u5173\u95ed\u3002 \u8def\u5f84\u672b\u5c3e\u65e0\u9700\u5e26\u6b63\u659c\u7ebf\uff08/\uff09\u3002","title":"\u8bf4\u660e\uff1a"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr-next/#6-oss","text":"\u82e5\u7528\u6237\u8bbf\u95ee Ranger \u6ca1\u6709\u6388\u6743\u7684\u8def\u5f84\uff0c\u5219\u4f1a\u62a5\u5982\u4e0b\u9519\u8bef\uff1a org.apache.hadoop.security.AccessControlException: Permission denied: user=test, access=READ_EXECUTE, resourcePath=\"bucket-test-hangzhou/\"","title":"6. \u8bbf\u95eeOSS\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/","text":"EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u00a7 \u672c\u6587\u4ecb\u7ecd\u5728 EMR \u96c6\u7fa4\u914d\u7f6e\u963f\u91cc\u4e91 OSS \u548c OSS-HDFS \u670d\u52a1\u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\u3002 \u80cc\u666f \u00a7 Apache Ranger \u63d0\u4f9b\u96c6\u4e2d\u5f0f\u7684\u6743\u9650\u7ba1\u7406\u6846\u67b6\uff0c\u53ef\u4ee5\u5bf9 Hadoop \u751f\u6001\u4e2d\u7684\u591a\u4e2a\u7ec4\u4ef6\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u8bbf\u95ee\u63a7\u5236\u3002\u5f53\u7528\u6237\u5c06\u6570\u636e\u5b58\u653e\u5728\u963f\u91cc\u4e91 OSS \u65f6\uff0c\u5219\u662f\u901a\u8fc7\u963f\u91cc\u4e91 RAM \u4ea7\u54c1\u521b\u5efa\u6216\u7ba1\u7406 RAM \u7528\u6237\uff0c\u5bf9RAM\u7528\u6237\u5b9e\u73b0\u5bf9 OSS \u8d44\u6e90\u7684\u8bbf\u95ee\u63a7\u5236\u3002 \u4e3a\u7ef4\u6301\u5927\u6570\u636e\u5ba2\u6237\u7684\u4f7f\u7528\u4e60\u60ef\uff0c\u901a\u8fc7 JindoFSx Namespace \u63a5\u5165 Ranger \u7684\u5ba2\u6237\u7aef\uff0c\u65b9\u4fbf\u7528\u6237\u7edf\u4e00\u7ba1\u7406\u5927\u6570\u636e\u7ec4\u4ef6\u6743\u9650\u3002 \u8bbf\u95ee OSS \u9274\u6743\u6d41\u7a0b \u00a7 OSS \u7684\u8bbf\u95ee\u5bc6\u94a5 AccessKey\uff08AK\uff09\u7edf\u4e00\u5728 JindoFSx Namespace \u4e2d\u8bbe\u7f6e\uff0c\u907f\u514d\u7528\u6237\u5728\u5ba2\u6237\u7aef\u914d\u7f6e\u660e\u6587\u5bc6\u94a5\uff0c\u5efa\u8bae\u53ea\u5141\u8bb8\u7ba1\u7406\u5458\u64cd\u4f5c\u548c\u7ba1\u7406 JindoFSx Namespace \u670d\u52a1\uff1b JindoSDK \u63d0\u4f9b\u6807\u51c6 Hadoop Filesystem \u5ba2\u6237\u7aef\uff0c\u5c06\u8bbf\u95ee OSS \u7684\u8bf7\u6c42\u4f1a\u53d1\u9001\u81f3 Namespace \u670d\u52a1\uff1b JindoFSx Namespace Service \u8d1f\u8d23\u96c6\u6210 Ranger \u5ba2\u6237\u7aef\uff0c\u5468\u671f\u6027\u5c06\u6743\u9650\u7b56\u7565\u4ece Ranger \u670d\u52a1\u7aef\u540c\u6b65\u5230\u672c\u5730\uff1b Namespace \u670d\u52a1\u5728\u6536\u5230 JindoSDK \u7684\u9274\u6743\u8bf7\u6c42\u540e\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u6821\u9a8c\uff1b \u901a\u8fc7\u6743\u9650\u6821\u9a8c\u540e JindoSDK \u5219\u53ef\u4ee5\u4f7f\u7528 JindoFSx Namespace \u670d\u52a1\u9881\u53d1\u7684\u4e34\u65f6 AK \u8bbf\u95ee OSS\u3002 \u6ce8\uff1a\u82e5 Namespace \u90e8\u7f72\u7684\u8282\u70b9\u4e3a\u963f\u91cc\u4e91 ECS \u8282\u70b9\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u5b89\u5168\u7ec4\uff0c\u9650\u5236\u8bbf\u95ee Namespace \u7684\u5ba2\u6237\u7aef\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u521b\u5efa E-MapReduce EMR-5.6.0/EMR-3.40.0 \u6216\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\uff0c\u5e76\u4e14\u9009\u62e9\u4e86 Ranger \u670d\u52a1\u3002 \u542f\u7528 Kerberos \u8ba4\u8bc1\u6ee1\u8db3\u8ba4\u8bc1\u9700\u6c42\u3002 \u82e5\u4f7f\u7528\u96c6\u7fa4\u90fd\u662f\u53ef\u4fe1\u7528\u6237\uff0c\u5219\u53ef\u4e0d\u542f\u7528 Kerberos \u8ba4\u8bc1\u3002 \u6ce8\u610f\uff1aEMR \u96c6\u7fa4\u5f00\u542f Kerberos \u540e\uff0c\u5c06\u65e0\u6cd5\u5173\u95ed Kerberos \u8ba4\u8bc1\uff0c\u8bf7\u8c28\u614e\u9009\u62e9\u3002\u5f00\u542f Kerberos \u540e\uff0c\u96c6\u7fa4\u6240\u6709\u5927\u6570\u636e\u7ec4\u4ef6\u7684\u670d\u52a1\u5747\u9700\u8981\u7ecf\u8fc7 Kerberos \u8ba4\u8bc1\uff0c\u5927\u6570\u636e\u4f5c\u4e1a\u63d0\u4ea4\u5230\u96c6\u7fa4\u4f1a\u5148\u7ecf\u8fc7\u8eab\u4efd\u8ba4\u8bc1\u3002Kerberos \u8be6\u60c5\u4fe1\u606f\uff0c\u8bf7\u53c2\u89c1 \u9ad8\u5b89\u5168\u7c7b\u578b\u96c6\u7fa4 \u521b\u5efa\u9ad8\u5b89\u5168\u96c6\u7fa4 \u00a7 \u521b\u5efa\u96c6\u7fa4\u65f6\uff0c\u5728\u8f6f\u4ef6\u914d\u7f6e\u9875\u9762\u7684\u9ad8\u7ea7\u8bbe\u7f6e\u533a\u57df\u4e2d\uff0c\u6253\u5f00Kerberos\u96c6\u7fa4\u6a21\u5f0f\u5f00\u5173\u3002 1. Ranger \u542f\u7528 OSS\u3002 \u00a7 2. \u90e8\u7f72\u5ba2\u6237\u7aef\u914d\u7f6e\u3002 \u00a7 \u5728 HDFS \u670d\u52a1\u9875\u9762\uff0c\u5355\u51fb\u53f3\u4e0a\u89d2\u7684\u90e8\u7f72\u5ba2\u6237\u7aef\u914d\u7f6e\u3002 3. \u91cd\u542f JindoFSx Namespace \u670d\u52a1\u3002 \u00a7 4. \u91cd\u542fHiveServer2\u3002 \u00a7 5. \u521b\u5efa\u7528\u6237 Principal\u3002 \u00a7 a. \u901a\u8fc7SSH\u65b9\u5f0f\u8fde\u63a5\u96c6\u7fa4\u7684emr-header-1\u8282\u70b9\u3002 \u00a7 b. \u6267\u884c\u5982\u4e0b\u547d\u4ee4\uff0c\u8fdb\u5165Kerberos\u7684admin\u5de5\u5177\u3002 \u00a7 sh /usr/lib/has-current/bin/admin-local.sh /etc/ecm/has-conf -k /etc/ecm/has-conf/admin.keytab \u672c\u793a\u4f8b\u5bc6\u7801\u8bbe\u7f6e\u4e3a 123456\u3002 addprinc -pw 123456 test \u8bf4\u660e: \u9700\u8981\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5728\u521b\u5efaTGT\u65f6\u4f1a\u7528\u5230\u3002\u5982\u679c\u60a8\u4e0d\u60f3\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5219\u53ef\u4ee5\u6267\u884c\u4e0b\u4e00\u6b65\uff0c\u628aPrincipal\u7684\u7528\u6237\u540d\u548c\u5bc6\u7801\u5bfc\u5165\u5230keytab\u6587\u4ef6\u4e2d\u3002 \u00a7 c. \u53ef\u9009\uff1a\u6267\u884c\u5982\u4e0b\u547d\u4ee4\uff0c\u751f\u6210keytab\u6587\u4ef6\u3002 \u00a7 ktadd -k /root/test.keytab test \u6267\u884c quit \u547d\u4ee4\uff0c\u53ef\u4ee5\u9000\u51fa Kerberos \u7684 admin \u5de5\u5177\u3002 6. \u521b\u5efa TGT\u3002 \u00a7 \u521b\u5efa TGT \u7684\u673a\u5668\uff0c\u53ef\u4ee5\u662f\u4efb\u610f\u4e00\u53f0\u9700\u8981\u8bbf\u95ee OSS \u7684\u673a\u5668\u3002 a. \u4f7f\u7528 root \u7528\u6237\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u521b\u5efa test \u7528\u6237\u3002 \u00a7 useradd test b. \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u5207\u6362\u4e3a test \u7528\u6237\u3002 \u00a7 su test c. \u751f\u6210TGT\u3002 \u00a7 \u65b9\u5f0f\u4e00\uff1a\u4f7f\u7528\u7528\u6237\u540d\u548c\u5bc6\u7801\u65b9\u5f0f\uff0c\u521b\u5efa TGT\u3002 \u6267\u884c kinit \u547d\u4ee4\uff0c\u56de\u8f66\u540e\u8f93\u5165 test \u7684\u5bc6\u7801 123456\u3002 \u65b9\u5f0f\u4e8c\uff1a\u4f7f\u7528 keytab \u6587\u4ef6\uff0c\u521b\u5efa TGT\u3002 \u5728\u6b65\u9aa4 4 \u4e2d\u7684 test.keytab \u6587\u4ef6\uff0c\u5df2\u7ecf\u4fdd\u5b58\u5728 emr-header-1 \u673a\u5668\u7684 /root/ \u76ee\u5f55\u4e0b\uff0c\u9700\u8981\u4f7f\u7528 scp \u547d\u4ee4\u62f7\u8d1d\u5230\u5f53\u524d\u673a\u5668\u7684 /home/test/ \u76ee\u5f55\u4e0b\u3002 kinit -kt /home/test/test.keytab test d. \u67e5\u770bTGT\u3002 \u00a7 \u4f7f\u7528 klist \u547d\u4ee4\uff0c\u5982\u679c\u51fa\u73b0\u5982\u4e0b\u4fe1\u606f\uff0c\u5219\u8bf4\u660eTGT\u521b\u5efa\u6210\u529f\uff0c\u5373\u53ef\u4ee5\u8bbf\u95eeOSS\u4e86\u3002 Ticket cache: FILE:/tmp/krb5cc_1012 Default principal: test@EMR.23****.COM Valid starting Expires Service principal 03/27/2022 23:20:44 03/28/2022 23:20:44 krbtgt/EMR.238075.COM@EMR.238075.COM renew until 03/28/2022 23:20:44 \u6ce8\u610f: \u9700\u8981\u8bb0\u5f55\u4e0b\u56de\u663e\u4fe1\u606f EMR.23****.COM \u4e2d\u7684\u6570\u5b57 23**** \uff0c\u5373\u4e3a cluster_id \u7684\u503c\uff0c\u540e\u9762\u8bbf\u95ee OSS \u65f6\u9700\u8981\u3002 7. \u5728 Ranger WebUI \u914d\u7f6e OSS \u6743\u9650 \u00a7 Ranger \u89c4\u5219\u793a\u4f8b \u00a7 \u4f8b\uff1a\u914d\u7f6e\u7528\u6237test\u62e5\u6709\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u6240\u4ee5\u6743\u9650\u7684\u6b65\u9aa4\uff1a a. \u914d\u7f6e test \u7528\u6237\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u8bbf\u95ee\u6743\u9650\u4e3a ALL\u3002 \u00a7 \u8bf4\u660e\uff1a \u00a7 \u89c4\u5219\u914d\u7f6e\u9875\u9762\u4e2d\uff0c\u914d\u7f6e\u7684 path \u6ca1\u6709 oss:// \u7684\u524d\u7f00\u3002 recursive\u6309\u94ae\u4e0d\u53ef\u5173\u95ed\u3002 \u8def\u5f84\u672b\u5c3e\u65e0\u9700\u5e26\u6b63\u659c\u7ebf\uff08/\uff09\u3002 b. \u9700\u8981\u914d\u7f6e\u8bbf\u95ee\u8def\u5f84\u7684\u7236\u76ee\u5f55 oss://bucket-test-hangzhou/user \u7684\u6743\u9650\u4e3a Execute\u3002 \u00a7 8. \u8bbf\u95eeOSS\u3002 \u00a7 \u82e5\u7528\u6237\u8bbf\u95ee Ranger \u6ca1\u6709\u6388\u6743\u7684\u8def\u5f84\uff0c\u5219\u4f1a\u62a5\u5982\u4e0b\u9519\u8bef\uff1a org.apache.hadoop.security.AccessControlException: Permission denied: user=test, access=READ_EXECUTE, resourcePath=\"bucket-test-hangzhou/\"","title":"EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#emr-oss-ranger","text":"\u672c\u6587\u4ecb\u7ecd\u5728 EMR \u96c6\u7fa4\u914d\u7f6e\u963f\u91cc\u4e91 OSS \u548c OSS-HDFS \u670d\u52a1\u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\u3002","title":"EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#_1","text":"Apache Ranger \u63d0\u4f9b\u96c6\u4e2d\u5f0f\u7684\u6743\u9650\u7ba1\u7406\u6846\u67b6\uff0c\u53ef\u4ee5\u5bf9 Hadoop \u751f\u6001\u4e2d\u7684\u591a\u4e2a\u7ec4\u4ef6\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u8bbf\u95ee\u63a7\u5236\u3002\u5f53\u7528\u6237\u5c06\u6570\u636e\u5b58\u653e\u5728\u963f\u91cc\u4e91 OSS \u65f6\uff0c\u5219\u662f\u901a\u8fc7\u963f\u91cc\u4e91 RAM \u4ea7\u54c1\u521b\u5efa\u6216\u7ba1\u7406 RAM \u7528\u6237\uff0c\u5bf9RAM\u7528\u6237\u5b9e\u73b0\u5bf9 OSS \u8d44\u6e90\u7684\u8bbf\u95ee\u63a7\u5236\u3002 \u4e3a\u7ef4\u6301\u5927\u6570\u636e\u5ba2\u6237\u7684\u4f7f\u7528\u4e60\u60ef\uff0c\u901a\u8fc7 JindoFSx Namespace \u63a5\u5165 Ranger \u7684\u5ba2\u6237\u7aef\uff0c\u65b9\u4fbf\u7528\u6237\u7edf\u4e00\u7ba1\u7406\u5927\u6570\u636e\u7ec4\u4ef6\u6743\u9650\u3002","title":"\u80cc\u666f"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#oss","text":"OSS \u7684\u8bbf\u95ee\u5bc6\u94a5 AccessKey\uff08AK\uff09\u7edf\u4e00\u5728 JindoFSx Namespace \u4e2d\u8bbe\u7f6e\uff0c\u907f\u514d\u7528\u6237\u5728\u5ba2\u6237\u7aef\u914d\u7f6e\u660e\u6587\u5bc6\u94a5\uff0c\u5efa\u8bae\u53ea\u5141\u8bb8\u7ba1\u7406\u5458\u64cd\u4f5c\u548c\u7ba1\u7406 JindoFSx Namespace \u670d\u52a1\uff1b JindoSDK \u63d0\u4f9b\u6807\u51c6 Hadoop Filesystem \u5ba2\u6237\u7aef\uff0c\u5c06\u8bbf\u95ee OSS \u7684\u8bf7\u6c42\u4f1a\u53d1\u9001\u81f3 Namespace \u670d\u52a1\uff1b JindoFSx Namespace Service \u8d1f\u8d23\u96c6\u6210 Ranger \u5ba2\u6237\u7aef\uff0c\u5468\u671f\u6027\u5c06\u6743\u9650\u7b56\u7565\u4ece Ranger \u670d\u52a1\u7aef\u540c\u6b65\u5230\u672c\u5730\uff1b Namespace \u670d\u52a1\u5728\u6536\u5230 JindoSDK \u7684\u9274\u6743\u8bf7\u6c42\u540e\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u6821\u9a8c\uff1b \u901a\u8fc7\u6743\u9650\u6821\u9a8c\u540e JindoSDK \u5219\u53ef\u4ee5\u4f7f\u7528 JindoFSx Namespace \u670d\u52a1\u9881\u53d1\u7684\u4e34\u65f6 AK \u8bbf\u95ee OSS\u3002 \u6ce8\uff1a\u82e5 Namespace \u90e8\u7f72\u7684\u8282\u70b9\u4e3a\u963f\u91cc\u4e91 ECS \u8282\u70b9\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u5b89\u5168\u7ec4\uff0c\u9650\u5236\u8bbf\u95ee Namespace \u7684\u5ba2\u6237\u7aef\u3002","title":"\u8bbf\u95ee OSS \u9274\u6743\u6d41\u7a0b"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#_2","text":"\u5df2\u521b\u5efa E-MapReduce EMR-5.6.0/EMR-3.40.0 \u6216\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\uff0c\u5e76\u4e14\u9009\u62e9\u4e86 Ranger \u670d\u52a1\u3002 \u542f\u7528 Kerberos \u8ba4\u8bc1\u6ee1\u8db3\u8ba4\u8bc1\u9700\u6c42\u3002 \u82e5\u4f7f\u7528\u96c6\u7fa4\u90fd\u662f\u53ef\u4fe1\u7528\u6237\uff0c\u5219\u53ef\u4e0d\u542f\u7528 Kerberos \u8ba4\u8bc1\u3002 \u6ce8\u610f\uff1aEMR \u96c6\u7fa4\u5f00\u542f Kerberos \u540e\uff0c\u5c06\u65e0\u6cd5\u5173\u95ed Kerberos \u8ba4\u8bc1\uff0c\u8bf7\u8c28\u614e\u9009\u62e9\u3002\u5f00\u542f Kerberos \u540e\uff0c\u96c6\u7fa4\u6240\u6709\u5927\u6570\u636e\u7ec4\u4ef6\u7684\u670d\u52a1\u5747\u9700\u8981\u7ecf\u8fc7 Kerberos \u8ba4\u8bc1\uff0c\u5927\u6570\u636e\u4f5c\u4e1a\u63d0\u4ea4\u5230\u96c6\u7fa4\u4f1a\u5148\u7ecf\u8fc7\u8eab\u4efd\u8ba4\u8bc1\u3002Kerberos \u8be6\u60c5\u4fe1\u606f\uff0c\u8bf7\u53c2\u89c1 \u9ad8\u5b89\u5168\u7c7b\u578b\u96c6\u7fa4","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#_3","text":"\u521b\u5efa\u96c6\u7fa4\u65f6\uff0c\u5728\u8f6f\u4ef6\u914d\u7f6e\u9875\u9762\u7684\u9ad8\u7ea7\u8bbe\u7f6e\u533a\u57df\u4e2d\uff0c\u6253\u5f00Kerberos\u96c6\u7fa4\u6a21\u5f0f\u5f00\u5173\u3002","title":"\u521b\u5efa\u9ad8\u5b89\u5168\u96c6\u7fa4"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#1-ranger-oss","text":"","title":"1. Ranger \u542f\u7528 OSS\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#2","text":"\u5728 HDFS \u670d\u52a1\u9875\u9762\uff0c\u5355\u51fb\u53f3\u4e0a\u89d2\u7684\u90e8\u7f72\u5ba2\u6237\u7aef\u914d\u7f6e\u3002","title":"2. \u90e8\u7f72\u5ba2\u6237\u7aef\u914d\u7f6e\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#3-jindofsx-namespace","text":"","title":"3. \u91cd\u542f JindoFSx Namespace \u670d\u52a1\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#4-hiveserver2","text":"","title":"4. \u91cd\u542fHiveServer2\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#5-principal","text":"","title":"5. \u521b\u5efa\u7528\u6237 Principal\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#a-sshemr-header-1","text":"","title":"a. \u901a\u8fc7SSH\u65b9\u5f0f\u8fde\u63a5\u96c6\u7fa4\u7684emr-header-1\u8282\u70b9\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#b-kerberosadmin","text":"sh /usr/lib/has-current/bin/admin-local.sh /etc/ecm/has-conf -k /etc/ecm/has-conf/admin.keytab \u672c\u793a\u4f8b\u5bc6\u7801\u8bbe\u7f6e\u4e3a 123456\u3002 addprinc -pw 123456 test","title":"b. \u6267\u884c\u5982\u4e0b\u547d\u4ee4\uff0c\u8fdb\u5165Kerberos\u7684admin\u5de5\u5177\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#tgtprincipalkeytab","text":"","title":"\u8bf4\u660e: \u9700\u8981\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5728\u521b\u5efaTGT\u65f6\u4f1a\u7528\u5230\u3002\u5982\u679c\u60a8\u4e0d\u60f3\u8bb0\u5f55\u7528\u6237\u540d\u548c\u5bc6\u7801\uff0c\u5219\u53ef\u4ee5\u6267\u884c\u4e0b\u4e00\u6b65\uff0c\u628aPrincipal\u7684\u7528\u6237\u540d\u548c\u5bc6\u7801\u5bfc\u5165\u5230keytab\u6587\u4ef6\u4e2d\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#c-keytab","text":"ktadd -k /root/test.keytab test \u6267\u884c quit \u547d\u4ee4\uff0c\u53ef\u4ee5\u9000\u51fa Kerberos \u7684 admin \u5de5\u5177\u3002","title":"c. \u53ef\u9009\uff1a\u6267\u884c\u5982\u4e0b\u547d\u4ee4\uff0c\u751f\u6210keytab\u6587\u4ef6\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#6-tgt","text":"\u521b\u5efa TGT \u7684\u673a\u5668\uff0c\u53ef\u4ee5\u662f\u4efb\u610f\u4e00\u53f0\u9700\u8981\u8bbf\u95ee OSS \u7684\u673a\u5668\u3002","title":"6. \u521b\u5efa TGT\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#a-root-test","text":"useradd test","title":"a. \u4f7f\u7528 root \u7528\u6237\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u521b\u5efa test \u7528\u6237\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#b-test","text":"su test","title":"b. \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u5207\u6362\u4e3a test \u7528\u6237\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#c-tgt","text":"\u65b9\u5f0f\u4e00\uff1a\u4f7f\u7528\u7528\u6237\u540d\u548c\u5bc6\u7801\u65b9\u5f0f\uff0c\u521b\u5efa TGT\u3002 \u6267\u884c kinit \u547d\u4ee4\uff0c\u56de\u8f66\u540e\u8f93\u5165 test \u7684\u5bc6\u7801 123456\u3002 \u65b9\u5f0f\u4e8c\uff1a\u4f7f\u7528 keytab \u6587\u4ef6\uff0c\u521b\u5efa TGT\u3002 \u5728\u6b65\u9aa4 4 \u4e2d\u7684 test.keytab \u6587\u4ef6\uff0c\u5df2\u7ecf\u4fdd\u5b58\u5728 emr-header-1 \u673a\u5668\u7684 /root/ \u76ee\u5f55\u4e0b\uff0c\u9700\u8981\u4f7f\u7528 scp \u547d\u4ee4\u62f7\u8d1d\u5230\u5f53\u524d\u673a\u5668\u7684 /home/test/ \u76ee\u5f55\u4e0b\u3002 kinit -kt /home/test/test.keytab test","title":"c. \u751f\u6210TGT\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#d-tgt","text":"\u4f7f\u7528 klist \u547d\u4ee4\uff0c\u5982\u679c\u51fa\u73b0\u5982\u4e0b\u4fe1\u606f\uff0c\u5219\u8bf4\u660eTGT\u521b\u5efa\u6210\u529f\uff0c\u5373\u53ef\u4ee5\u8bbf\u95eeOSS\u4e86\u3002 Ticket cache: FILE:/tmp/krb5cc_1012 Default principal: test@EMR.23****.COM Valid starting Expires Service principal 03/27/2022 23:20:44 03/28/2022 23:20:44 krbtgt/EMR.238075.COM@EMR.238075.COM renew until 03/28/2022 23:20:44 \u6ce8\u610f: \u9700\u8981\u8bb0\u5f55\u4e0b\u56de\u663e\u4fe1\u606f EMR.23****.COM \u4e2d\u7684\u6570\u5b57 23**** \uff0c\u5373\u4e3a cluster_id \u7684\u503c\uff0c\u540e\u9762\u8bbf\u95ee OSS \u65f6\u9700\u8981\u3002","title":"d. \u67e5\u770bTGT\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#7-ranger-webui-oss","text":"","title":"7. \u5728 Ranger WebUI \u914d\u7f6e OSS \u6743\u9650"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#ranger","text":"\u4f8b\uff1a\u914d\u7f6e\u7528\u6237test\u62e5\u6709\u8bbf\u95ee oss://bucket-test-hangzhou/user/test \u76ee\u5f55\u7684\u6240\u4ee5\u6743\u9650\u7684\u6b65\u9aa4\uff1a","title":"Ranger \u89c4\u5219\u793a\u4f8b"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#a-test-ossbucket-test-hangzhouusertest-all","text":"","title":"a. \u914d\u7f6e test \u7528\u6237\u8bbf\u95eeoss://bucket-test-hangzhou/user/test\u76ee\u5f55\u7684\u8bbf\u95ee\u6743\u9650\u4e3a ALL\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#_4","text":"\u89c4\u5219\u914d\u7f6e\u9875\u9762\u4e2d\uff0c\u914d\u7f6e\u7684 path \u6ca1\u6709 oss:// \u7684\u524d\u7f00\u3002 recursive\u6309\u94ae\u4e0d\u53ef\u5173\u95ed\u3002 \u8def\u5f84\u672b\u5c3e\u65e0\u9700\u5e26\u6b63\u659c\u7ebf\uff08/\uff09\u3002","title":"\u8bf4\u660e\uff1a"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#b-ossbucket-test-hangzhouuser-execute","text":"","title":"b. \u9700\u8981\u914d\u7f6e\u8bbf\u95ee\u8def\u5f84\u7684\u7236\u76ee\u5f55oss://bucket-test-hangzhou/user\u7684\u6743\u9650\u4e3a Execute\u3002"},{"location":"backup/jindofsx/permission/jindofsx_ranger_emr/#8-oss","text":"\u82e5\u7528\u6237\u8bbf\u95ee Ranger \u6ca1\u6709\u6388\u6743\u7684\u8def\u5f84\uff0c\u5219\u4f1a\u62a5\u5982\u4e0b\u9519\u8bef\uff1a org.apache.hadoop.security.AccessControlException: Permission denied: user=test, access=READ_EXECUTE, resourcePath=\"bucket-test-hangzhou/\"","title":"8. \u8bbf\u95eeOSS\u3002"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_dls/","text":"Presto \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002 \u6b65\u9aa4 \u00a7 1. \u5b89\u88c5 jar \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/ 2. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e Access Key \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 3. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 \u4f7f\u7528\u793a\u4f8b \u00a7 \u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236\u53f0 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='oss://<Bucket>.<Endpoint>/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"Presto \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_dls/#presto-oss-hdfs-jindofs-jindofsx","text":"Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002","title":"Presto \u67e5\u8be2\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_dls/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_dls/#1-jar","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/","title":"1. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_dls/#2-jindosdk","text":"\u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e Access Key \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1 \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"2. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_dls/#3-presto","text":"","title":"3. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_dls/#_2","text":"\u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236\u53f0 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='oss://<Bucket>.<Endpoint>/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"\u4f7f\u7528\u793a\u4f8b"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_jindo/","text":"Presto \u67e5\u8be2 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e \u00a7 Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002 \u6b65\u9aa4 \u00a7 1. \u5b89\u88c5 jar \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/ 2. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e Access Key \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 3. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 4. \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55 \u00a7 \u6302\u8f7d\u547d\u4ee4 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/ \u4f7f\u7528\u793a\u4f8b \u00a7 \u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236\u53f0 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='jindo://emr-header-1:8101/jindooss/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"Presto \u67e5\u8be2 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_jindo/#presto-jindofsx","text":"Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002","title":"Presto \u67e5\u8be2 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_jindo/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_jindo/#1-jar","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/","title":"1. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_jindo/#2-jindosdk","text":"\u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e Access Key \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"2. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_jindo/#3-presto","text":"","title":"3. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_jindo/#4-oss-oss-hdfs","text":"\u6302\u8f7d\u547d\u4ee4 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/","title":"4. \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_jindo/#_2","text":"\u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236\u53f0 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='jindo://emr-header-1:8101/jindooss/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"\u4f7f\u7528\u793a\u4f8b"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_oss/","text":"Presto \u67e5\u8be2\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002 \u6b65\u9aa4 \u00a7 1. \u5b89\u88c5 jar \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/ 2. \u914d\u7f6e JindoSDK \u00a7 *\u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key \u5c06 OSS \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 3. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 \u4f7f\u7528\u793a\u4f8b \u00a7 \u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236\u53f0 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='oss://<bucket>/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"Presto \u67e5\u8be2\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_oss/#presto-oss-jindofsx","text":"Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002","title":"Presto \u67e5\u8be2\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_oss/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_oss/#1-jar","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/","title":"1. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_oss/#2-jindosdk","text":"*\u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key \u5c06 OSS \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 \u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"2. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_oss/#3-presto","text":"","title":"3. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"backup/jindofsx/presto/jindosdk_on_presto_oss/#_2","text":"\u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236\u53f0 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='oss://<bucket>/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"\u4f7f\u7528\u793a\u4f8b"},{"location":"backup/jindofsx/security/jindofsx_credential_provider/","text":"JindoSDK OSS Credential Provider \u914d\u7f6e \u00a7 \u57fa\u672c\u914d\u7f6e\u65b9\u5f0f \u00a7 \u5728\u7f13\u5b58\u6570\u636e\u65f6\uff0cJindoFSx \u670d\u52a1\u7aef\u9700\u8981\u80fd\u8bbf\u95ee\u88ab\u7f13\u5b58\u7684\u540e\u7aef\u5b58\u50a8\u7cfb\u7edf\uff0c\u83b7\u53d6\u6240\u9700\u6570\u636e\u3002\u6839\u636e\u60a8\u53ef\u4ee5\u6839\u636e\u8bbf\u95ee\u7684\u5b58\u50a8\u7c7b\u578b\uff0c\u60a8\u53ef\u4ee5\u53c2\u8003\u76f8\u5e94\u7684\u6587\u6863\u5bf9 JindoFSx \u670d\u52a1\u8bbf\u95ee\u540e\u7aef\u5b58\u50a8\u7684 Credential \u548c Endpoint \u8fdb\u884c\u914d\u7f6e\uff0c\u914d\u7f6e\u652f\u6301\u660e\u786e\u5230 bucket \u7ea7\u522b\u3002 \u670d\u52a1\u7aef\u8bbf\u95ee\u963f\u91cc\u4e91 OSS \u00a7 JindoFSx \u670d\u52a1\u7aef\u652f\u6301\u5bf9 OSS \u534f\u8bae\u7684\u6240\u6709\u8bbf\u95ee\u914d\u7f6e\u7edf\u4e00\u7684 Credential \u4fe1\u606f\uff0c\u5982\u4e0b\u6240\u793a\uff1a [jindofsx-common] jindofsx.oss.accessKeyId = xxx jindofsx.oss.accessKeySecret = xxx jindofsx.oss.endpoint = oss-cn-xxx-internal.aliyuncs.com \u60a8\u4e5f\u53ef\u4ee5\u4e3a\u6bcf\u4e2a bucket \u914d\u7f6e\u72ec\u7acb\u7684 Credential \u4fe1\u606f\u548c Endpoint\uff1a jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = oss-cn-xxx-internal.aliyuncs.com \u670d\u52a1\u7aef\u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 JindoFSx \u670d\u52a1\u7aef\u652f\u6301\u5bf9 OSS \u534f\u8bae\u7684\u6240\u6709\u8bbf\u95ee\u914d\u7f6e\u7edf\u4e00\u7684 Credential \u4fe1\u606f\uff0c\u5982\u4e0b\u6240\u793a\uff1a [jindofsx-common] jindofsx.oss.accessKeyId = xxx jindofsx.oss.accessKeySecret = xxx jindofsx.oss.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.user = user #Storage Service \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u4f7f\u7528\u7684\u7528\u6237\u540d \u60a8\u4e5f\u53ef\u4ee5\u4e3a\u6bcf\u4e2a bucket \u914d\u7f6e\u72ec\u7acb\u7684 Credential \u4fe1\u606f\u548c Endpoint\uff0c\u5c24\u5176\u662f\u5728\u540c\u65f6\u4f7f\u7528 OSS \u670d\u52a1\u4e0e OSS-HDFS \u670d\u52a1\u65f6\uff0c\u9700\u8981\u5206\u522b\u914d\u7f6e\u4e0d\u540c\u7684 bucket\uff1a [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = cn-xxx.oss-dls.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = cn-xxx.oss-dls.aliyuncs.com jindofsx.oss.bucket.YYY.user = user #Storage Service \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u4f7f\u7528\u7684\u7528\u6237\u540d \u670d\u52a1\u7aef\u8bbf\u95ee Apache HDFS \u00a7 JindoFSx \u670d\u52a1\u7aef\u9700\u8981\u4e0e HDFS \u96c6\u7fa4\u8fdb\u884c\u7f51\u7edc\u901a\u4fe1\uff0c\u5e76\u6839\u636e\u76f8\u5e94 HDFS \u96c6\u7fa4\u7684\u914d\u7f6e\u8bbe\u7f6e HDFS \u96c6\u7fa4\u7684\u8bbf\u95ee\u4fe1\u606f\u3002\u5982\u679c\u8bbf\u95ee\u5355 master \u7684 HDFS \u96c6\u7fa4\uff0c\u9700\u8981\u914d\u7f6e\u8bbf\u95ee\u7528\u6237\u540d\uff1a [jindofsx-common] jindofsx.hdfs.user = user \u5982\u9700\u8bbf\u95ee nameservice \u914d\u7f6e\u4e3a mycluster \u7684 HDFS HA \u96c6\u7fa4\uff0c\u5219\u914d\u7f6e\u5982\u4e0b\u6240\u793a\uff1a [jindofsx-common] jindofsx.hdfs.mycluster.dfs.ha.namenodes = nn1,nn2 jindofsx.hdfs.mycluster.dfs.namenode.rpc-address.nn1 = <nn1-host>:8998 jindofsx.hdfs.mycluster.dfs.namenode.rpc-address.nn2 = <nn2-host>:8999 jindofsx.hdfs.user = user \u670d\u52a1\u7aef\u8bbf\u95ee NAS \u00a7 JindoFSx \u4e0e JindoSDK \u8981\u6c42 NAS \u670d\u52a1\u5df2\u7ecf mount \u5230\u6240\u6709\u8282\u70b9\u7684\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\uff0c\u5e76\u4fdd\u8bc1\u670d\u52a1\u548c\u5ba2\u6237\u7aef\u90fd\u5bf9\u6302\u8f7d\u76ee\u5f55\u6709\u8bfb\u6743\u9650\u3002 \u670d\u52a1\u7aef\u8bbf\u95ee\u5176\u4ed6\u4e91\u5b58\u50a8\uff08S3/COS/OBS\u7b49\uff09 \u00a7 JindoFSx \u670d\u52a1\u7aef\u652f\u6301\u5bf9\u67d0\u79cd\u534f\u8bae\u7684\u6240\u6709\u8bbf\u95ee\u914d\u7f6e\u7edf\u4e00\u7684 Credential \u4fe1\u606f\uff0c\u5982\u4e0b\u6240\u793a\uff1a [jindofsx-common] jindofsx.s3.accessKeyId = xxx jindofsx.s3.accessKeySecret = xxx jindofsx.s3.endpoint = s3.us-west-1.amazonaws.com jindofsx.cos.accessKeyId = xxx jindofsx.cos.accessKeySecret = xxx jindofsx.cos.endpoint = cos.ap-shanghai.myqcloud.com jindofsx.obs.accessKeyId = xxx jindofsx.obs.accessKeySecret = xxx jindofsx.obs.endpoint = obs.cn-east-3.myhuaweicloud.com \u60a8\u4e5f\u53ef\u4ee5\u4e3a\u6bcf\u4e2a bucket \u914d\u7f6e\u72ec\u7acb\u7684 Credential \u4fe1\u606f\u548c Endpoint\uff0c\u4ee5\u4e0b\u4ee5 S3 \u4e3a\u4f8b\uff1a jindofsx.s3.bucket.XXX.accessKeyId = xxx jindofsx.s3.bucket.XXX.accessKeySecret = xxx jindofsx.s3.bucket.XXX.endpoint = s3.us-west-1.amazonaws.com jindofsx.s3.bucket.YYY.accessKeyId = xxx jindofsx.s3.bucket.YYY.accessKeySecret = xxx jindofsx.s3.bucket.YYY.endpoint = s3.ap-east-1.amazonaws.com \u670d\u52a1\u7aef\u8bbf\u95ee\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u4e0b\u7684\u5b58\u50a8 \u00a7 \u6839\u636e\u5bf9\u5e94\u7684\u5b9e\u9645\u5b58\u50a8\u7cfb\u7edf\uff0c\u8fdb\u884c\u914d\u7f6e\u5373\u53ef\u3002 \u670d\u52a1\u7aef\u4f7f\u7528 JindoSDK OSS Credential Provider \u00a7 \u5f53\u524d\u7248\u672c\u6682\u4e0d\u652f\u6301\u5728 JindoFSx \u670d\u52a1\u7aef\u4f7f\u7528 JindoSDK OSS Credential Provider\u3002","title":"JindoSDK OSS Credential Provider \u914d\u7f6e"},{"location":"backup/jindofsx/security/jindofsx_credential_provider/#jindosdk-oss-credential-provider","text":"","title":"JindoSDK OSS Credential Provider \u914d\u7f6e"},{"location":"backup/jindofsx/security/jindofsx_credential_provider/#_1","text":"\u5728\u7f13\u5b58\u6570\u636e\u65f6\uff0cJindoFSx \u670d\u52a1\u7aef\u9700\u8981\u80fd\u8bbf\u95ee\u88ab\u7f13\u5b58\u7684\u540e\u7aef\u5b58\u50a8\u7cfb\u7edf\uff0c\u83b7\u53d6\u6240\u9700\u6570\u636e\u3002\u6839\u636e\u60a8\u53ef\u4ee5\u6839\u636e\u8bbf\u95ee\u7684\u5b58\u50a8\u7c7b\u578b\uff0c\u60a8\u53ef\u4ee5\u53c2\u8003\u76f8\u5e94\u7684\u6587\u6863\u5bf9 JindoFSx \u670d\u52a1\u8bbf\u95ee\u540e\u7aef\u5b58\u50a8\u7684 Credential \u548c Endpoint \u8fdb\u884c\u914d\u7f6e\uff0c\u914d\u7f6e\u652f\u6301\u660e\u786e\u5230 bucket \u7ea7\u522b\u3002","title":"\u57fa\u672c\u914d\u7f6e\u65b9\u5f0f"},{"location":"backup/jindofsx/security/jindofsx_credential_provider/#oss","text":"JindoFSx \u670d\u52a1\u7aef\u652f\u6301\u5bf9 OSS \u534f\u8bae\u7684\u6240\u6709\u8bbf\u95ee\u914d\u7f6e\u7edf\u4e00\u7684 Credential \u4fe1\u606f\uff0c\u5982\u4e0b\u6240\u793a\uff1a [jindofsx-common] jindofsx.oss.accessKeyId = xxx jindofsx.oss.accessKeySecret = xxx jindofsx.oss.endpoint = oss-cn-xxx-internal.aliyuncs.com \u60a8\u4e5f\u53ef\u4ee5\u4e3a\u6bcf\u4e2a bucket \u914d\u7f6e\u72ec\u7acb\u7684 Credential \u4fe1\u606f\u548c Endpoint\uff1a jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = oss-cn-xxx-internal.aliyuncs.com","title":"\u670d\u52a1\u7aef\u8bbf\u95ee\u963f\u91cc\u4e91 OSS"},{"location":"backup/jindofsx/security/jindofsx_credential_provider/#oss-hdfs-jindofs","text":"JindoFSx \u670d\u52a1\u7aef\u652f\u6301\u5bf9 OSS \u534f\u8bae\u7684\u6240\u6709\u8bbf\u95ee\u914d\u7f6e\u7edf\u4e00\u7684 Credential \u4fe1\u606f\uff0c\u5982\u4e0b\u6240\u793a\uff1a [jindofsx-common] jindofsx.oss.accessKeyId = xxx jindofsx.oss.accessKeySecret = xxx jindofsx.oss.endpoint = oss-cn-xxx-internal.aliyuncs.com jindofsx.oss.user = user #Storage Service \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u4f7f\u7528\u7684\u7528\u6237\u540d \u60a8\u4e5f\u53ef\u4ee5\u4e3a\u6bcf\u4e2a bucket \u914d\u7f6e\u72ec\u7acb\u7684 Credential \u4fe1\u606f\u548c Endpoint\uff0c\u5c24\u5176\u662f\u5728\u540c\u65f6\u4f7f\u7528 OSS \u670d\u52a1\u4e0e OSS-HDFS \u670d\u52a1\u65f6\uff0c\u9700\u8981\u5206\u522b\u914d\u7f6e\u4e0d\u540c\u7684 bucket\uff1a [jindofsx-common] jindofsx.oss.bucket.XXX.accessKeyId = xxx jindofsx.oss.bucket.XXX.accessKeySecret = xxx jindofsx.oss.bucket.XXX.endpoint = cn-xxx.oss-dls.aliyuncs.com jindofsx.oss.bucket.YYY.accessKeyId = xxx jindofsx.oss.bucket.YYY.accessKeySecret = xxx jindofsx.oss.bucket.YYY.endpoint = cn-xxx.oss-dls.aliyuncs.com jindofsx.oss.bucket.YYY.user = user #Storage Service \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u4f7f\u7528\u7684\u7528\u6237\u540d","title":"\u670d\u52a1\u7aef\u8bbf\u95ee\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"backup/jindofsx/security/jindofsx_credential_provider/#apache-hdfs","text":"JindoFSx \u670d\u52a1\u7aef\u9700\u8981\u4e0e HDFS \u96c6\u7fa4\u8fdb\u884c\u7f51\u7edc\u901a\u4fe1\uff0c\u5e76\u6839\u636e\u76f8\u5e94 HDFS \u96c6\u7fa4\u7684\u914d\u7f6e\u8bbe\u7f6e HDFS \u96c6\u7fa4\u7684\u8bbf\u95ee\u4fe1\u606f\u3002\u5982\u679c\u8bbf\u95ee\u5355 master \u7684 HDFS \u96c6\u7fa4\uff0c\u9700\u8981\u914d\u7f6e\u8bbf\u95ee\u7528\u6237\u540d\uff1a [jindofsx-common] jindofsx.hdfs.user = user \u5982\u9700\u8bbf\u95ee nameservice \u914d\u7f6e\u4e3a mycluster \u7684 HDFS HA \u96c6\u7fa4\uff0c\u5219\u914d\u7f6e\u5982\u4e0b\u6240\u793a\uff1a [jindofsx-common] jindofsx.hdfs.mycluster.dfs.ha.namenodes = nn1,nn2 jindofsx.hdfs.mycluster.dfs.namenode.rpc-address.nn1 = <nn1-host>:8998 jindofsx.hdfs.mycluster.dfs.namenode.rpc-address.nn2 = <nn2-host>:8999 jindofsx.hdfs.user = user","title":"\u670d\u52a1\u7aef\u8bbf\u95ee Apache HDFS"},{"location":"backup/jindofsx/security/jindofsx_credential_provider/#nas","text":"JindoFSx \u4e0e JindoSDK \u8981\u6c42 NAS \u670d\u52a1\u5df2\u7ecf mount \u5230\u6240\u6709\u8282\u70b9\u7684\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\uff0c\u5e76\u4fdd\u8bc1\u670d\u52a1\u548c\u5ba2\u6237\u7aef\u90fd\u5bf9\u6302\u8f7d\u76ee\u5f55\u6709\u8bfb\u6743\u9650\u3002","title":"\u670d\u52a1\u7aef\u8bbf\u95ee NAS"},{"location":"backup/jindofsx/security/jindofsx_credential_provider/#s3cosobs","text":"JindoFSx \u670d\u52a1\u7aef\u652f\u6301\u5bf9\u67d0\u79cd\u534f\u8bae\u7684\u6240\u6709\u8bbf\u95ee\u914d\u7f6e\u7edf\u4e00\u7684 Credential \u4fe1\u606f\uff0c\u5982\u4e0b\u6240\u793a\uff1a [jindofsx-common] jindofsx.s3.accessKeyId = xxx jindofsx.s3.accessKeySecret = xxx jindofsx.s3.endpoint = s3.us-west-1.amazonaws.com jindofsx.cos.accessKeyId = xxx jindofsx.cos.accessKeySecret = xxx jindofsx.cos.endpoint = cos.ap-shanghai.myqcloud.com jindofsx.obs.accessKeyId = xxx jindofsx.obs.accessKeySecret = xxx jindofsx.obs.endpoint = obs.cn-east-3.myhuaweicloud.com \u60a8\u4e5f\u53ef\u4ee5\u4e3a\u6bcf\u4e2a bucket \u914d\u7f6e\u72ec\u7acb\u7684 Credential \u4fe1\u606f\u548c Endpoint\uff0c\u4ee5\u4e0b\u4ee5 S3 \u4e3a\u4f8b\uff1a jindofsx.s3.bucket.XXX.accessKeyId = xxx jindofsx.s3.bucket.XXX.accessKeySecret = xxx jindofsx.s3.bucket.XXX.endpoint = s3.us-west-1.amazonaws.com jindofsx.s3.bucket.YYY.accessKeyId = xxx jindofsx.s3.bucket.YYY.accessKeySecret = xxx jindofsx.s3.bucket.YYY.endpoint = s3.ap-east-1.amazonaws.com","title":"\u670d\u52a1\u7aef\u8bbf\u95ee\u5176\u4ed6\u4e91\u5b58\u50a8\uff08S3/COS/OBS\u7b49\uff09"},{"location":"backup/jindofsx/security/jindofsx_credential_provider/#_2","text":"\u6839\u636e\u5bf9\u5e94\u7684\u5b9e\u9645\u5b58\u50a8\u7cfb\u7edf\uff0c\u8fdb\u884c\u914d\u7f6e\u5373\u53ef\u3002","title":"\u670d\u52a1\u7aef\u8bbf\u95ee\u7edf\u4e00\u547d\u540d\u7a7a\u95f4\u4e0b\u7684\u5b58\u50a8"},{"location":"backup/jindofsx/security/jindofsx_credential_provider/#jindosdk-oss-credential-provider_1","text":"\u5f53\u524d\u7248\u672c\u6682\u4e0d\u652f\u6301\u5728 JindoFSx \u670d\u52a1\u7aef\u4f7f\u7528 JindoSDK OSS Credential Provider\u3002","title":"\u670d\u52a1\u7aef\u4f7f\u7528 JindoSDK OSS Credential Provider"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_dls/","text":"Spark \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\uff0c\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fdd Spark \u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/ 3. \u914d\u7f6e JindoSDK \u00a7 \u5168\u5c40\u914d\u7f6e \u00a7 \u4fee\u6539 Spark \u4f7f\u7528\u7684 core-site.xml \u3002 * \u914d\u7f6e JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u4efb\u52a1\u7ea7\u522b\u914d\u7f6e \u00a7 \u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx --conf spark.hadoop.fs.jindofsx.namespace.rpc.address=hostname:port --conf spark.hadoop.fs.jindofsx.data.cache.enable=true 4. \u4f7f\u7528 Spark \u8bbf\u95ee OSS \u00a7 \u521b\u5efa\u8868 create table test_oss (c1 string) location \"oss://test-bucket.<Endpoint>/<path>\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss; 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Spark \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_dls/#spark-oss-hdfs-jindofs-jindofsx","text":"JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\uff0c\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Spark \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_dls/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_dls/#1-spark-hadoop","text":"\u786e\u4fdd Spark \u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_dls/#2-spark-classpath-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/","title":"2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_dls/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_dls/#_2","text":"\u4fee\u6539 Spark \u4f7f\u7528\u7684 core-site.xml \u3002 * \u914d\u7f6e JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 <configuration> <property> <!-- \u6570\u636e\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"\u5168\u5c40\u914d\u7f6e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_dls/#_3","text":"\u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx --conf spark.hadoop.fs.jindofsx.namespace.rpc.address=hostname:port --conf spark.hadoop.fs.jindofsx.data.cache.enable=true","title":"\u4efb\u52a1\u7ea7\u522b\u914d\u7f6e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_dls/#4-spark-oss","text":"\u521b\u5efa\u8868 create table test_oss (c1 string) location \"oss://test-bucket.<Endpoint>/<path>\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss;","title":"4. \u4f7f\u7528 Spark \u8bbf\u95ee OSS"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_dls/#5","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/","text":"Spark \u5904\u7406 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e \u00a7 JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411Hadoop/Spark\u751f\u6001\u7684OSS\u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684Hadoop FileSystem\u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fddSpark\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/ 3. \u914d\u7f6e JindoSDK \u00a7 \u5168\u5c40\u914d\u7f6e \u00a7 \u4fee\u6539 Spark \u4f7f\u7528\u7684 core-site.xml \u3002 \u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06 OSS \u6216 OSS-HDFS \u670d\u52a1 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 <configuration> <property> <!-- \u5185\u5b58\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u4efb\u52a1\u7ea7\u522b\u914d\u7f6e \u00a7 \u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx --conf spark.hadoop.fs.oss.endpoint=oss-cn-xxx-internal.aliyuncs.com --conf spark.hadoop.fs.jindofsx.namespace.rpc.address=hostname:port --conf spark.hadoop.fs.jindofsx.data.cache.enable=true 4. \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55 \u00a7 \u6302\u8f7d\u547d\u4ee4 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/ 5. \u4f7f\u7528 Spark \u8bbf\u95ee OSS \u00a7 \u521b\u5efa\u8868 create table test_oss (c1 string) location \"jindo://emr-header-1:8101/jindooss/<path>\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss; \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Spark \u5904\u7406 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/#spark-jindofsx","text":"JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411Hadoop/Spark\u751f\u6001\u7684OSS\u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684Hadoop FileSystem\u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Spark \u5904\u7406 JindoFSx \u7edf\u4e00\u6302\u8f7d\u7684\u6570\u636e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/#1-spark-hadoop","text":"\u786e\u4fddSpark\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/#2-spark-classpath-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/","title":"2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/#_2","text":"\u4fee\u6539 Spark \u4f7f\u7528\u7684 core-site.xml \u3002 \u914d\u7f6e\u5b9e\u73b0\u7c7b \u5c06 JindoFSx \u7edf\u4e00\u540d\u5b57\u7a7a\u95f4\u4f7f\u7528\u7684\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JINDO</value> </property> <property> <name>fs.jindo.impl</name> <value>com.aliyun.jindodata.jindo.JindoFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e AccessKey \u5c06 OSS \u6216 OSS-HDFS \u670d\u52a1 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\u3002 OSS Bucket \u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 OSS-HDFS \u670d\u52a1 Bucket \u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS \u6216 OSS-HDFS \u670d\u52a1 Endpoint <configuration> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> \u5982\u679c\u7edf\u4e00\u6302\u8f7d\u7684\u4e3a OSS-HDFS \u670d\u52a1\u76ee\u5f55\uff0c\u914d\u7f6e Endpoint \u8bf7\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 <configuration> <property> <!-- \u5185\u5b58\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"\u5168\u5c40\u914d\u7f6e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/#_3","text":"\u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx --conf spark.hadoop.fs.oss.endpoint=oss-cn-xxx-internal.aliyuncs.com --conf spark.hadoop.fs.jindofsx.namespace.rpc.address=hostname:port --conf spark.hadoop.fs.jindofsx.data.cache.enable=true","title":"\u4efb\u52a1\u7ea7\u522b\u914d\u7f6e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/#4-oss-oss-hdfs","text":"\u6302\u8f7d\u547d\u4ee4 jindo fsxadmin -mount <path> <realpath> \u4f8b\uff1a jindo fsxadmin -mount /jindooss oss://<Bucket>.<Endpoint>/ \u6267\u884c\u5982\u4e0a\u547d\u4ee4\u540e\uff0c\u5219 /jindooss \u76ee\u5f55\u4e0b\u771f\u6b63\u6302\u8f7d\u7684\u6587\u4ef6\u8def\u5f84\u662f oss://<Bucket>.<Endpoint>/","title":"4. \u6302\u8f7d OSS \u6216 OSS-HDFS \u670d\u52a1\u76ee\u5f55"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/#5-spark-oss","text":"\u521b\u5efa\u8868 create table test_oss (c1 string) location \"jindo://emr-header-1:8101/jindooss/<path>\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss;","title":"5. \u4f7f\u7528 Spark \u8bbf\u95ee OSS"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_jindo/#_4","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_oss/","text":"Spark \u5904\u7406\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f \u00a7 JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411Hadoop/Spark\u751f\u6001\u7684OSS\u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684Hadoop FileSystem\u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fddSpark\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/ 3. \u914d\u7f6e JindoSDK \u00a7 \u5168\u5c40\u914d\u7f6e \u00a7 \u4fee\u6539 Spark \u4f7f\u7528\u7684 core-site.xml \u3002 \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 <configuration> <property> <!-- \u5185\u5b58\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570 \u4efb\u52a1\u7ea7\u522b\u914d\u7f6e \u00a7 \u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx --conf spark.hadoop.fs.oss.endpoint=oss-cn-xxx-internal.aliyuncs.com --conf spark.hadoop.fs.jindofsx.namespace.rpc.address=hostname:port --conf spark.hadoop.fs.jindofsx.data.cache.enable=true 4. \u4f7f\u7528 Spark \u8bbf\u95ee OSS \u00a7 \u521b\u5efa\u8868 create table test_oss (c1 string) location \"oss://<Bucket>/<path>\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss; 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Spark \u5904\u7406\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_oss/#spark-oss-jindofsx","text":"JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411Hadoop/Spark\u751f\u6001\u7684OSS\u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684Hadoop FileSystem\u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Spark \u5904\u7406\u963f\u91cc\u4e91 OSS \u4e0a\u7684\u6570\u636e + JindoFSx \u900f\u660e\u52a0\u901f"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_oss/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_oss/#1-spark-hadoop","text":"\u786e\u4fddSpark\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_oss/#2-spark-classpath-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/","title":"2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_oss/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_oss/#_2","text":"\u4fee\u6539 Spark \u4f7f\u7528\u7684 core-site.xml \u3002 \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> <property> <name>fs.xengine</name> <value>jindofsx</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx-internal.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e JindoFSx Namespace \u670d\u52a1\u5730\u5740\u3002 <configuration> <property> <!-- Namespace service \u5730\u5740 --> <name>fs.jindofsx.namespace.rpc.address</name> <value>headerhost:8101</value> </property> </configuration> \u82e5\u4f7f\u7528\u9ad8\u53ef\u7528 Namespace, \u8bf7\u53c2\u8003 \u9ad8\u53ef\u7528 JindoFSx Namespace \u914d\u7f6e\u548c\u4f7f\u7528 \u5f00\u542f\u7f13\u5b58\u3002 <configuration> <property> <!-- \u5185\u5b58\u7f13\u5b58\u5f00\u5173 --> <name>fs.jindofsx.data.cache.enable</name> <value>true</value> </property> </configuration> \u66f4\u591a\u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570\uff0c\u8bf7\u53c2\u8003 \u7f13\u5b58\u4f18\u5316\u76f8\u5173\u53c2\u6570","title":"\u5168\u5c40\u914d\u7f6e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_oss/#_3","text":"\u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx --conf spark.hadoop.fs.oss.endpoint=oss-cn-xxx-internal.aliyuncs.com --conf spark.hadoop.fs.jindofsx.namespace.rpc.address=hostname:port --conf spark.hadoop.fs.jindofsx.data.cache.enable=true","title":"\u4efb\u52a1\u7ea7\u522b\u914d\u7f6e"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_oss/#4-spark-oss","text":"\u521b\u5efa\u8868 create table test_oss (c1 string) location \"oss://<Bucket>/<path>\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss;","title":"4. \u4f7f\u7528 Spark \u8bbf\u95ee OSS"},{"location":"backup/jindofsx/spark/jindosdk_on_spark_oss/#5","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/jindosdk/jindosdk_memory_best_practice/","text":"JindoSDk \u901a\u8fc7 jemalloc \u63a7\u5236\u5185\u5b58\u4e0a\u6da8\u6700\u4f73\u5b9e\u8df5 \u00a7 \u80cc\u666f \u00a7 jindosdk hadoop sdk \u4f7f\u7528 jni \u8c03\u7528 native \u5b9e\u73b0\uff0c\u56e0\u6b64 jvm \u76f8\u5173 gc \u53c2\u6570\u65e0\u6cd5\u6709\u6548\u63a7\u5236 native \u5185\u5b58\u4e0a\u6da8\uff0c\u53ef\u4ee5\u901a\u8fc7 jemalloc \u914d\u7f6e\u6765\u8fdb\u884c\u63a7\u5236\u3002 \u90e8\u5c5e\u65b9\u6cd5 \u00a7 jindosdk 4.6.8 \u53ca\u4ee5\u4e0a\u7248\u672c\u5728 tar \u5305\u4e2d\u9ed8\u8ba4\u643a\u5e26\u4e86 jemalloc\uff1a . \u251c\u2500\u2500 bin \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 conf \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 lib \u2502 \u251c\u2500\u2500 jindo-core-4.6.12.jar \u2502 \u251c\u2500\u2500 jindo-sdk-4.6.12.jar \u2502 \u2514\u2500\u2500 native \u2502 \u251c\u2500\u2500 libjemalloc.so \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 plugins \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 tools \u2502 \u251c\u2500\u2500 ... \u2514\u2500\u2500 versions \u2514\u2500\u2500 JINDOSDK_VERSION \u5047\u8bbe\u5b89\u88c5\u76ee\u5f55\u4e3a/opt/apps/JINDOSDK/jindosdk-current/\uff0c\u8981\u4fdd\u8bc1\u6240\u6709\u8c03\u7528jindosdk\u7684\u73af\u5883\uff0c\u90fd\u914d\u7f6e\u4ee5\u4e0b\u7684\u73af\u5883\u53d8\u91cf\uff1a LD_PRELOAD=/opt/apps/JINDOSDK/jindosdk-current/lib/native/libjemalloc.so MALLOC_CONF=narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0,background_thread:true,abort_conf:true \u6548\u679c \u00a7 wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/memcheck/jindo-sdk-memleak-check-1.0.jar \u6267\u884c\u4ee5\u4e0bmemleak-check\u6d4b\u8bd5 HADOOP_HEAPSIZE=2560 HADOOP_ROOT_LOGGER=ERROR,console hadoop jar jindo-sdk-memleak-check-1.0.jar MemLeakTestProgram -baseDir oss://yanbin-hd2-test/ -duration 8899 \u4f7f\u7528\u9ed8\u8ba4glibc\uff0c\u6700\u7ec8\u5360\u75283627M \u4f7f\u7528jemalloc\u4e0d\u8bbe\u7f6eMALLOC_CONF\uff0c\u6700\u7ec8\u5360\u75282929M \u4f7f\u7528jemalloc\u5e76\u8bbe\u7f6eMALLOC_CONF\uff0c\u6700\u7ec8\u5360\u75282289M","title":"JindoSDk \u901a\u8fc7 jemalloc \u63a7\u5236\u5185\u5b58\u4e0a\u6da8\u6700\u4f73\u5b9e\u8df5"},{"location":"backup/jindosdk/jindosdk_memory_best_practice/#jindosdk-jemalloc","text":"","title":"JindoSDk \u901a\u8fc7 jemalloc \u63a7\u5236\u5185\u5b58\u4e0a\u6da8\u6700\u4f73\u5b9e\u8df5"},{"location":"backup/jindosdk/jindosdk_memory_best_practice/#_1","text":"jindosdk hadoop sdk \u4f7f\u7528 jni \u8c03\u7528 native \u5b9e\u73b0\uff0c\u56e0\u6b64 jvm \u76f8\u5173 gc \u53c2\u6570\u65e0\u6cd5\u6709\u6548\u63a7\u5236 native \u5185\u5b58\u4e0a\u6da8\uff0c\u53ef\u4ee5\u901a\u8fc7 jemalloc \u914d\u7f6e\u6765\u8fdb\u884c\u63a7\u5236\u3002","title":"\u80cc\u666f"},{"location":"backup/jindosdk/jindosdk_memory_best_practice/#_2","text":"jindosdk 4.6.8 \u53ca\u4ee5\u4e0a\u7248\u672c\u5728 tar \u5305\u4e2d\u9ed8\u8ba4\u643a\u5e26\u4e86 jemalloc\uff1a . \u251c\u2500\u2500 bin \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 conf \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 lib \u2502 \u251c\u2500\u2500 jindo-core-4.6.12.jar \u2502 \u251c\u2500\u2500 jindo-sdk-4.6.12.jar \u2502 \u2514\u2500\u2500 native \u2502 \u251c\u2500\u2500 libjemalloc.so \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 plugins \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 tools \u2502 \u251c\u2500\u2500 ... \u2514\u2500\u2500 versions \u2514\u2500\u2500 JINDOSDK_VERSION \u5047\u8bbe\u5b89\u88c5\u76ee\u5f55\u4e3a/opt/apps/JINDOSDK/jindosdk-current/\uff0c\u8981\u4fdd\u8bc1\u6240\u6709\u8c03\u7528jindosdk\u7684\u73af\u5883\uff0c\u90fd\u914d\u7f6e\u4ee5\u4e0b\u7684\u73af\u5883\u53d8\u91cf\uff1a LD_PRELOAD=/opt/apps/JINDOSDK/jindosdk-current/lib/native/libjemalloc.so MALLOC_CONF=narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0,background_thread:true,abort_conf:true","title":"\u90e8\u5c5e\u65b9\u6cd5"},{"location":"backup/jindosdk/jindosdk_memory_best_practice/#_3","text":"wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/memcheck/jindo-sdk-memleak-check-1.0.jar \u6267\u884c\u4ee5\u4e0bmemleak-check\u6d4b\u8bd5 HADOOP_HEAPSIZE=2560 HADOOP_ROOT_LOGGER=ERROR,console hadoop jar jindo-sdk-memleak-check-1.0.jar MemLeakTestProgram -baseDir oss://yanbin-hd2-test/ -duration 8899 \u4f7f\u7528\u9ed8\u8ba4glibc\uff0c\u6700\u7ec8\u5360\u75283627M \u4f7f\u7528jemalloc\u4e0d\u8bbe\u7f6eMALLOC_CONF\uff0c\u6700\u7ec8\u5360\u75282929M \u4f7f\u7528jemalloc\u5e76\u8bbe\u7f6eMALLOC_CONF\uff0c\u6700\u7ec8\u5360\u75282289M","title":"\u6548\u679c"},{"location":"backup/jindosdk/outline/","text":"JindoSDK \u591a\u4e91\u5b58\u50a8 \u00a7 JindoSDK \u652f\u6301 S3\u3001COS\u3001OBS \u591a\u4e91\u5b58\u50a8\u3002 \u5feb\u901f\u5165\u95e8 \u00a7 JindoSDK + S3 \u5feb\u901f\u5165\u95e8 JindoSDK + COS \u5feb\u901f\u5165\u95e8 JindoSDK + OBS \u5feb\u901f\u5165\u95e8 \u5386\u53f2\u7248\u672c \u00a7 JindoSDK \u591a\u4e91\u5b58\u50a8 4.6.0 \u7528\u6237\u6587\u6863 JindoSDK \u591a\u4e91\u5b58\u50a8 4.5.1 \u7528\u6237\u6587\u6863 JindoSDK \u591a\u4e91\u5b58\u50a8 4.5.0 \u7528\u6237\u6587\u6863 JindoSDK \u591a\u4e91\u5b58\u50a8 4.4.0 \u7528\u6237\u6587\u6863 JindoSDK \u591a\u4e91\u5b58\u50a8 4.3.0 \u7528\u6237\u6587\u6863","title":"JindoSDK \u591a\u4e91\u5b58\u50a8"},{"location":"backup/jindosdk/outline/#jindosdk","text":"JindoSDK \u652f\u6301 S3\u3001COS\u3001OBS \u591a\u4e91\u5b58\u50a8\u3002","title":"JindoSDK \u591a\u4e91\u5b58\u50a8"},{"location":"backup/jindosdk/outline/#_1","text":"JindoSDK + S3 \u5feb\u901f\u5165\u95e8 JindoSDK + COS \u5feb\u901f\u5165\u95e8 JindoSDK + OBS \u5feb\u901f\u5165\u95e8","title":"\u5feb\u901f\u5165\u95e8"},{"location":"backup/jindosdk/outline/#_2","text":"JindoSDK \u591a\u4e91\u5b58\u50a8 4.6.0 \u7528\u6237\u6587\u6863 JindoSDK \u591a\u4e91\u5b58\u50a8 4.5.1 \u7528\u6237\u6587\u6863 JindoSDK \u591a\u4e91\u5b58\u50a8 4.5.0 \u7528\u6237\u6587\u6863 JindoSDK \u591a\u4e91\u5b58\u50a8 4.4.0 \u7528\u6237\u6587\u6863 JindoSDK \u591a\u4e91\u5b58\u50a8 4.3.0 \u7528\u6237\u6587\u6863","title":"\u5386\u53f2\u7248\u672c"},{"location":"backup/jindosdk/cos/jindosdk_quickstart/","text":"JindoSDK + COS \u5feb\u901f\u5165\u95e8 \u00a7 JindoSDK \u5168\u9762\u517c\u5bb9 Hadoop FileSystem \u63a5\u53e3\uff0c\u63d0\u4f9b\u4e86\u66f4\u597d\u7684\u517c\u5bb9\u6027\u548c\u6613\u7528\u6027\u3002 \u76f8\u5bf9\u4e8e Hadoop \u793e\u533a COS \u5ba2\u6237\u7aef\u5b9e\u73b0\uff0c\u60a8\u4ecd\u7136\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e JindoSDK COS \u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u5c06 JindoSDK COS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.cos.impl</name> <value>com.aliyun.jindodata.cos.COS</value> </property> <property> <name>fs.cos.impl</name> <value>com.aliyun.jindodata.cos.JindoCosFileSystem</value> </property> </configuration> 4. \u914d\u7f6e COS Access Key \u00a7 \u5c06 Access Key \u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 \u914d\u7f6e COS Access Key \u00a7 \u5c06 COS bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.cos.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.cos.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.cos.endpoint</name> <value>xxx</value> </property> </configuration> \u6309\u7167 bucket \u914d\u7f6e Access Key \u00a7 JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 COS bucket \u914d\u7f6e\u4e0d\u540c\u7684 Access Key\u3002 <configuration> <property> <name>fs.cos.bucket.XXX.accessKeyId</name> <value>COS bucket \u7684 AccessKey Id</value> </property> <property> <name>fs.cos.bucket.XXX.accessKeySecret</name> <value>COS bucket \u7684A ccessKey Secret</value> </property> </configuration> \u8bf4\u660e XXX \u4e3a COS bucket \u540d\u79f0\u3002 5. \u4f7f\u7528 JindoSDK \u8bbf\u95ee COS \u00a7 \u7528 Hadoop Shell \u8bbf\u95ee COS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> cos://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls cos://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir cos://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm cos://<bucket>/<path>","title":"JindoSDK + COS \u5feb\u901f\u5165\u95e8"},{"location":"backup/jindosdk/cos/jindosdk_quickstart/#jindosdk-cos","text":"JindoSDK \u5168\u9762\u517c\u5bb9 Hadoop FileSystem \u63a5\u53e3\uff0c\u63d0\u4f9b\u4e86\u66f4\u597d\u7684\u517c\u5bb9\u6027\u548c\u6613\u7528\u6027\u3002 \u76f8\u5bf9\u4e8e Hadoop \u793e\u533a COS \u5ba2\u6237\u7aef\u5b9e\u73b0\uff0c\u60a8\u4ecd\u7136\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"JindoSDK + COS \u5feb\u901f\u5165\u95e8"},{"location":"backup/jindosdk/cos/jindosdk_quickstart/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindosdk/cos/jindosdk_quickstart/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindosdk/cos/jindosdk_quickstart/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindosdk/cos/jindosdk_quickstart/#3-jindosdk-cos-access-key","text":"\u5c06 JindoSDK COS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.cos.impl</name> <value>com.aliyun.jindodata.cos.COS</value> </property> <property> <name>fs.cos.impl</name> <value>com.aliyun.jindodata.cos.JindoCosFileSystem</value> </property> </configuration>","title":"3. \u914d\u7f6e JindoSDK COS \u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"backup/jindosdk/cos/jindosdk_quickstart/#4-cos-access-key","text":"\u5c06 Access Key \u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002","title":"4. \u914d\u7f6e COS Access Key"},{"location":"backup/jindosdk/cos/jindosdk_quickstart/#cos-access-key","text":"\u5c06 COS bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.cos.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.cos.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.cos.endpoint</name> <value>xxx</value> </property> </configuration>","title":"\u914d\u7f6e COS Access Key"},{"location":"backup/jindosdk/cos/jindosdk_quickstart/#bucket-access-key","text":"JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 COS bucket \u914d\u7f6e\u4e0d\u540c\u7684 Access Key\u3002 <configuration> <property> <name>fs.cos.bucket.XXX.accessKeyId</name> <value>COS bucket \u7684 AccessKey Id</value> </property> <property> <name>fs.cos.bucket.XXX.accessKeySecret</name> <value>COS bucket \u7684A ccessKey Secret</value> </property> </configuration> \u8bf4\u660e XXX \u4e3a COS bucket \u540d\u79f0\u3002","title":"\u6309\u7167 bucket \u914d\u7f6e Access Key"},{"location":"backup/jindosdk/cos/jindosdk_quickstart/#5-jindosdk-cos","text":"\u7528 Hadoop Shell \u8bbf\u95ee COS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> cos://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls cos://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir cos://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm cos://<bucket>/<path>","title":"5. \u4f7f\u7528 JindoSDK \u8bbf\u95ee COS"},{"location":"backup/jindosdk/obs/jindosdk_quickstart/","text":"JindoSDK + \u963f\u91cc\u4e91 OBS \u5feb\u901f\u5165\u95e8 \u00a7 JindoSDK \u5168\u9762\u517c\u5bb9 Hadoop FileSystem \u63a5\u53e3\uff0c\u63d0\u4f9b\u4e86\u66f4\u597d\u7684\u517c\u5bb9\u6027\u548c\u6613\u7528\u6027\u3002 \u76f8\u5bf9\u4e8e Hadoop \u793e\u533a OBS \u5ba2\u6237\u7aef\u5b9e\u73b0\uff0c\u60a8\u4ecd\u7136\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e JindoSDK OBS \u5b9e\u73b0\u7c7b \u00a7 \u5c06 JindoSDK OBS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.obs.impl</name> <value>com.aliyun.jindodata.obs.OBS</value> </property> <property> <name>fs.obs.impl</name> <value>com.aliyun.jindodata.obs.JindoObsFileSystem</value> </property> </configuration> 4. \u914d\u7f6e OBS Access Key \u00a7 \u5c06 Access Key \u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 \u914d\u7f6e OBS Access Key \u00a7 \u5c06 OBS bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.obs.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.obs.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.obs.endpoint</name> <value>xxx</value> </property> </configuration> \u6309\u7167 bucket \u914d\u7f6e Access Key \u00a7 JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 OBS bucket \u914d\u7f6e\u4e0d\u540c\u7684 Access Key\u3002 <configuration> <property> <name>fs.obs.bucket.XXX.accessKeyId</name> <value>OBS bucket \u7684 AccessKey Id</value> </property> <property> <name>fs.obs.bucket.XXX.accessKeySecret</name> <value>OBS bucket \u7684 AccessKey Secret</value> </property> </configuration> \u8bf4\u660e XXX \u4e3a OBS bucket \u540d\u79f0\u3002 5. \u4f7f\u7528 JindoSDK \u8bbf\u95ee OBS \u00a7 \u7528 Hadoop Shell \u8bbf\u95ee OBS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> obs://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls obs://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir obs://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm obs://<bucket>/<path>","title":"JindoSDK + \u963f\u91cc\u4e91 OBS \u5feb\u901f\u5165\u95e8"},{"location":"backup/jindosdk/obs/jindosdk_quickstart/#jindosdk-obs","text":"JindoSDK \u5168\u9762\u517c\u5bb9 Hadoop FileSystem \u63a5\u53e3\uff0c\u63d0\u4f9b\u4e86\u66f4\u597d\u7684\u517c\u5bb9\u6027\u548c\u6613\u7528\u6027\u3002 \u76f8\u5bf9\u4e8e Hadoop \u793e\u533a OBS \u5ba2\u6237\u7aef\u5b9e\u73b0\uff0c\u60a8\u4ecd\u7136\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"JindoSDK + \u963f\u91cc\u4e91 OBS \u5feb\u901f\u5165\u95e8"},{"location":"backup/jindosdk/obs/jindosdk_quickstart/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindosdk/obs/jindosdk_quickstart/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindosdk/obs/jindosdk_quickstart/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindosdk/obs/jindosdk_quickstart/#3-jindosdk-obs","text":"\u5c06 JindoSDK OBS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.obs.impl</name> <value>com.aliyun.jindodata.obs.OBS</value> </property> <property> <name>fs.obs.impl</name> <value>com.aliyun.jindodata.obs.JindoObsFileSystem</value> </property> </configuration>","title":"3. \u914d\u7f6e JindoSDK OBS \u5b9e\u73b0\u7c7b"},{"location":"backup/jindosdk/obs/jindosdk_quickstart/#4-obs-access-key","text":"\u5c06 Access Key \u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002","title":"4. \u914d\u7f6e OBS Access Key"},{"location":"backup/jindosdk/obs/jindosdk_quickstart/#obs-access-key","text":"\u5c06 OBS bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.obs.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.obs.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.obs.endpoint</name> <value>xxx</value> </property> </configuration>","title":"\u914d\u7f6e OBS Access Key"},{"location":"backup/jindosdk/obs/jindosdk_quickstart/#bucket-access-key","text":"JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 OBS bucket \u914d\u7f6e\u4e0d\u540c\u7684 Access Key\u3002 <configuration> <property> <name>fs.obs.bucket.XXX.accessKeyId</name> <value>OBS bucket \u7684 AccessKey Id</value> </property> <property> <name>fs.obs.bucket.XXX.accessKeySecret</name> <value>OBS bucket \u7684 AccessKey Secret</value> </property> </configuration> \u8bf4\u660e XXX \u4e3a OBS bucket \u540d\u79f0\u3002","title":"\u6309\u7167 bucket \u914d\u7f6e Access Key"},{"location":"backup/jindosdk/obs/jindosdk_quickstart/#5-jindosdk-obs","text":"\u7528 Hadoop Shell \u8bbf\u95ee OBS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> obs://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls obs://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir obs://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm obs://<bucket>/<path>","title":"5. \u4f7f\u7528 JindoSDK \u8bbf\u95ee OBS"},{"location":"backup/jindosdk/s3/jindosdk_quickstart/","text":"JindoSDK + S3 \u5feb\u901f\u5165\u95e8 \u00a7 JindoSDK \u5168\u9762\u517c\u5bb9 Hadoop FileSystem \u63a5\u53e3\uff0c\u63d0\u4f9b\u4e86\u66f4\u597d\u7684\u517c\u5bb9\u6027\u548c\u6613\u7528\u6027\u3002 \u76f8\u5bf9\u4e8e Hadoop \u793e\u533a S3 \u5ba2\u6237\u7aef\u5b9e\u73b0\uff0c\u60a8\u4ecd\u7136\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e JindoSDK S3 \u5b9e\u73b0\u7c7b \u00a7 \u5c06 JindoSDK S3 \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.s3.impl</name> <value>com.aliyun.jindodata.s3.S3</value> </property> <property> <name>fs.s3.impl</name> <value>com.aliyun.jindodata.s3.JindoS3FileSystem</value> </property> </configuration> 4. \u914d\u7f6e S3 Access Key \u00a7 \u5c06 Access Key \u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 \u914d\u7f6e S3 Access Key \u00a7 \u5c06 S3 bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.s3.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.s3.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.s3.endpoint</name> <value>xxx</value> </property> </configuration> \u6309\u7167 bucket \u914d\u7f6e Access Key \u00a7 JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 S3 bucket \u914d\u7f6e\u4e0d\u540c\u7684 Access Key\u3002 <configuration> <property> <name>fs.s3.bucket.XXX.accessKeyId</name> <value>S3 bucket \u7684 AccessKey Id</value> </property> <property> <name>fs.s3.bucket.XXX.accessKeySecret</name> <value>S3 bucket \u7684 AccessKey Secret</value> </property> </configuration> \u8bf4\u660e XXX \u4e3a S3 bucket \u540d\u79f0\u3002 5. \u4f7f\u7528 JindoSDK \u8bbf\u95ee S3 \u00a7 \u7528 Hadoop Shell \u8bbf\u95ee S3\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> s3://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls s3://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir s3://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm s3://<bucket>/<path>","title":"JindoSDK + S3 \u5feb\u901f\u5165\u95e8"},{"location":"backup/jindosdk/s3/jindosdk_quickstart/#jindosdk-s3","text":"JindoSDK \u5168\u9762\u517c\u5bb9 Hadoop FileSystem \u63a5\u53e3\uff0c\u63d0\u4f9b\u4e86\u66f4\u597d\u7684\u517c\u5bb9\u6027\u548c\u6613\u7528\u6027\u3002 \u76f8\u5bf9\u4e8e Hadoop \u793e\u533a S3 \u5ba2\u6237\u7aef\u5b9e\u73b0\uff0c\u60a8\u4ecd\u7136\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"JindoSDK + S3 \u5feb\u901f\u5165\u95e8"},{"location":"backup/jindosdk/s3/jindosdk_quickstart/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/jindosdk/s3/jindosdk_quickstart/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/jindosdk/s3/jindosdk_quickstart/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/jindosdk/s3/jindosdk_quickstart/#3-jindosdk-s3","text":"\u5c06 JindoSDK S3 \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.s3.impl</name> <value>com.aliyun.jindodata.s3.S3</value> </property> <property> <name>fs.s3.impl</name> <value>com.aliyun.jindodata.s3.JindoS3FileSystem</value> </property> </configuration>","title":"3. \u914d\u7f6e JindoSDK S3 \u5b9e\u73b0\u7c7b"},{"location":"backup/jindosdk/s3/jindosdk_quickstart/#4-s3-access-key","text":"\u5c06 Access Key \u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002","title":"4. \u914d\u7f6e S3 Access Key"},{"location":"backup/jindosdk/s3/jindosdk_quickstart/#s3-access-key","text":"\u5c06 S3 bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.s3.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.s3.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.s3.endpoint</name> <value>xxx</value> </property> </configuration>","title":"\u914d\u7f6e S3 Access Key"},{"location":"backup/jindosdk/s3/jindosdk_quickstart/#bucket-access-key","text":"JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 S3 bucket \u914d\u7f6e\u4e0d\u540c\u7684 Access Key\u3002 <configuration> <property> <name>fs.s3.bucket.XXX.accessKeyId</name> <value>S3 bucket \u7684 AccessKey Id</value> </property> <property> <name>fs.s3.bucket.XXX.accessKeySecret</name> <value>S3 bucket \u7684 AccessKey Secret</value> </property> </configuration> \u8bf4\u660e XXX \u4e3a S3 bucket \u540d\u79f0\u3002","title":"\u6309\u7167 bucket \u914d\u7f6e Access Key"},{"location":"backup/jindosdk/s3/jindosdk_quickstart/#5-jindosdk-s3","text":"\u7528 Hadoop Shell \u8bbf\u95ee S3\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> s3://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls s3://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir s3://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm s3://<bucket>/<path>","title":"5. \u4f7f\u7528 JindoSDK \u8bbf\u95ee S3"},{"location":"backup/oss/jindosdk_quickstart/","text":"JindoSDK + \u963f\u91cc\u4e91 OSS \u5feb\u901f\u5165\u95e8 \u00a7 JindoSDK \u5168\u9762\u517c\u5bb9 Hadoop FileSystem \u63a5\u53e3\uff0c\u63d0\u4f9b\u4e86\u66f4\u597d\u7684\u517c\u5bb9\u6027\u548c\u6613\u7528\u6027\uff0c\u53c8\u80fd\u517c\u5177OSS\u7684\u6027\u80fd\u548c\u6210\u672c\u4f18\u52bf\u3002 \u76f8\u5bf9\u4e8e Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\u5b9e\u73b0\uff0c\u60a8\u4ecd\u7136\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u76ee\u524d\u652f\u6301\u5e02\u9762\u4e0a\u5927\u90e8\u5206 Hadoop \u7248\u672c\uff0c\u5728 Hadoop 2.3 \u53ca\u4ee5\u4e0a\u7684\u7248\u672c\u4e0a\u9a8c\u8bc1\u901a\u8fc7\uff082.3 \u4ee5\u524d\u7248\u672c\u6682\u672a\u6d4b\u8bd5\uff0c\u5982\u6709\u95ee\u9898\u8bf7 \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988\uff09\u3002 \u5173\u4e8e JindoSDK \u548c Hadoop \u793e\u533a OSS connector \u7684\u6027\u80fd\u5bf9\u6bd4\uff0c\u8bf7\u53c2\u8003\u6587\u6863 JindoSDK\u548cHadoop-OSS-SDK\u6027\u80fd\u5bf9\u6bd4\u6d4b\u8bd5 \u3002 \u6b65\u9aa4 \u00a7 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06 OSS bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 4. \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u00a7 \u7528 Hadoop Shell \u8bbf\u95ee OSS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls oss://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<bucket>/<path> 5. \u6e05\u7406\u56de\u6536\u7ad9 \u00a7 Hadoop \u901a\u8fc7\u5c06\u5220\u9664\u7684\u6587\u4ef6\u6216\u76ee\u5f55\u653e\u5165\u56de\u6536\u7ad9\u6765\u9632\u6b62\u8bef\u5220\u6587\u4ef6\u6216\u6587\u4ef6\u5939\u3002\u5f53\u4f7f\u7528 Hadoop Shell \u5220\u9664 OSS \u7684\u6587\u4ef6\u6216\u76ee\u5f55\u65f6\uff0c \u82e5\u5f00\u542f\u4e86\u56de\u6536\u7ad9\u529f\u80fd\uff0c\u4f1a\u5728 OSS \u7684 bucket \u4e0a\u5b58\u50a8\u88ab\u5220\u9664\u7684\u6587\u4ef6\u6216\u76ee\u5f55\uff0c\u56de\u6536\u7ad9\u7684\u76ee\u5f55\u4e3a oss://buckect-name/user/<username>/.Trash \u3002\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e OSS \u6240\u4f7f\u7528\u7684\u56de\u6536\u7ad9\u76ee\u5f55\u7684\u751f\u547d\u5468\u671f\u6765\u6e05\u7406\u56de\u6536\u7ad9\u3002 \u5173\u4e8e\u5982\u4f55\u8bbe\u7f6e OSS \u751f\u547d\u5468\u671f\uff0c\u8bf7\u53c2\u8003 \u8bbe\u7f6e\u751f\u547d\u5468\u671f\u89c4\u5219 \u82e5\u8981\u8df3\u8fc7\u56de\u6536\u7ad9\uff0c\u76f4\u63a5\u5220\u9664\uff0c\u5219\u53ef\u4ee5\u6307\u5b9a -skipTrash \u53c2\u6570\uff08\u614e\u7528\uff09\u3002 hadoop fs -rm -skipTrash oss://<bucket>/<path> 6. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 7. \u6743\u9650\u7ba1\u7406 \u00a7 \u81ea\u5efa\u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u963f\u91cc\u4e91 E-MapReduce \u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848","title":"JindoSDK + \u963f\u91cc\u4e91 OSS \u5feb\u901f\u5165\u95e8"},{"location":"backup/oss/jindosdk_quickstart/#jindosdk-oss","text":"JindoSDK \u5168\u9762\u517c\u5bb9 Hadoop FileSystem \u63a5\u53e3\uff0c\u63d0\u4f9b\u4e86\u66f4\u597d\u7684\u517c\u5bb9\u6027\u548c\u6613\u7528\u6027\uff0c\u53c8\u80fd\u517c\u5177OSS\u7684\u6027\u80fd\u548c\u6210\u672c\u4f18\u52bf\u3002 \u76f8\u5bf9\u4e8e Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\u5b9e\u73b0\uff0c\u60a8\u4ecd\u7136\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u76ee\u524d\u652f\u6301\u5e02\u9762\u4e0a\u5927\u90e8\u5206 Hadoop \u7248\u672c\uff0c\u5728 Hadoop 2.3 \u53ca\u4ee5\u4e0a\u7684\u7248\u672c\u4e0a\u9a8c\u8bc1\u901a\u8fc7\uff082.3 \u4ee5\u524d\u7248\u672c\u6682\u672a\u6d4b\u8bd5\uff0c\u5982\u6709\u95ee\u9898\u8bf7 \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988\uff09\u3002 \u5173\u4e8e JindoSDK \u548c Hadoop \u793e\u533a OSS connector \u7684\u6027\u80fd\u5bf9\u6bd4\uff0c\u8bf7\u53c2\u8003\u6587\u6863 JindoSDK\u548cHadoop-OSS-SDK\u6027\u80fd\u5bf9\u6bd4\u6d4b\u8bd5 \u3002","title":"JindoSDK + \u963f\u91cc\u4e91 OSS \u5feb\u901f\u5165\u95e8"},{"location":"backup/oss/jindosdk_quickstart/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/oss/jindosdk_quickstart/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/oss/jindosdk_quickstart/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/oss/jindosdk_quickstart/#3-oss-access-key","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06 OSS bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002","title":"3. \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"backup/oss/jindosdk_quickstart/#4-jindosdk-oss","text":"\u7528 Hadoop Shell \u8bbf\u95ee OSS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls oss://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<bucket>/<path>","title":"4. \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/oss/jindosdk_quickstart/#5","text":"Hadoop \u901a\u8fc7\u5c06\u5220\u9664\u7684\u6587\u4ef6\u6216\u76ee\u5f55\u653e\u5165\u56de\u6536\u7ad9\u6765\u9632\u6b62\u8bef\u5220\u6587\u4ef6\u6216\u6587\u4ef6\u5939\u3002\u5f53\u4f7f\u7528 Hadoop Shell \u5220\u9664 OSS \u7684\u6587\u4ef6\u6216\u76ee\u5f55\u65f6\uff0c \u82e5\u5f00\u542f\u4e86\u56de\u6536\u7ad9\u529f\u80fd\uff0c\u4f1a\u5728 OSS \u7684 bucket \u4e0a\u5b58\u50a8\u88ab\u5220\u9664\u7684\u6587\u4ef6\u6216\u76ee\u5f55\uff0c\u56de\u6536\u7ad9\u7684\u76ee\u5f55\u4e3a oss://buckect-name/user/<username>/.Trash \u3002\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e OSS \u6240\u4f7f\u7528\u7684\u56de\u6536\u7ad9\u76ee\u5f55\u7684\u751f\u547d\u5468\u671f\u6765\u6e05\u7406\u56de\u6536\u7ad9\u3002 \u5173\u4e8e\u5982\u4f55\u8bbe\u7f6e OSS \u751f\u547d\u5468\u671f\uff0c\u8bf7\u53c2\u8003 \u8bbe\u7f6e\u751f\u547d\u5468\u671f\u89c4\u5219 \u82e5\u8981\u8df3\u8fc7\u56de\u6536\u7ad9\uff0c\u76f4\u63a5\u5220\u9664\uff0c\u5219\u53ef\u4ee5\u6307\u5b9a -skipTrash \u53c2\u6570\uff08\u614e\u7528\uff09\u3002 hadoop fs -rm -skipTrash oss://<bucket>/<path>","title":"5. \u6e05\u7406\u56de\u6536\u7ad9"},{"location":"backup/oss/jindosdk_quickstart/#6","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"6. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/oss/jindosdk_quickstart/#7","text":"\u81ea\u5efa\u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u963f\u91cc\u4e91 E-MapReduce \u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848","title":"7. \u6743\u9650\u7ba1\u7406"},{"location":"backup/oss/jindoshell_oss_howto/","text":"JindoShell CLI \u652f\u6301\u963f\u91cc\u4e91 OSS \u4f7f\u7528\u8bf4\u660e \u00a7 JindoShell CLI \u652f\u6301\u963f\u91cc\u4e91 OSS archive, unarchive, restore \u7b49\u547d\u4ee4\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ Archive \u547d\u4ee4 \u00a7 Archive\u547d\u4ee4\u53ef\u4ee5\u5f52\u6863OSS\u4e0a\u7684\u6570\u636e\u81f3\u4f4e\u9891\u8bbf\u95ee\u5b58\u50a8\u6216\u8005\u5f52\u6863\u5b58\u50a8\u4e0a\u3002 jindo oss -archive [-i|c] <path> -i\u53c2\u6570\u53ef\u4ee5\u5f52\u6863\u6570\u636e\u81f3OSS\u4f4e\u9891\u5b58\u50a8\u7c7b\u578b\u3002-c\u53c2\u6570\u53ef\u4ee5\u5f52\u6863\u6570\u636e\u81f3OSS\u51b7\u5f52\u6863\u5b58\u50a8\u7c7b\u578b\u3002\u9ed8\u8ba4\u53ef\u4ee5\u5f52\u6863\u6570\u636e\u81f3OSS\u5f52\u6863\u5b58\u50a8\u7c7b\u578b\u3002 Unarchive \u547d\u4ee4 \u00a7 Unarchive\u547d\u4ee4\u53ef\u4ee5\u5c06\u6570\u636e\u4ece\u5f52\u6863\u5b58\u50a8\u7c7b\u578b\u6062\u590d\u5230\u4f4e\u9891\u5b58\u50a8\u6216\u8005\u6807\u51c6\u5b58\u3002 jindo oss -unarchive [-i|a] <path> -i\u53c2\u6570\u53ef\u4ee5\u6062\u590d\u6570\u636e\u81f3OSS\u4f4e\u9891\u5b58\u50a8\u7c7b\u578b\u3002-a\u53c2\u6570\u53ef\u4ee5\u6062\u590d\u6570\u636e\u81f3OSS\u5f52\u6863\u5b58\u50a8\u7c7b\u578b\u3002\u9ed8\u8ba4\u53ef\u4ee5\u6062\u590d\u6570\u636e\u81f3OSS\u6807\u51c6\u5b58\u50a8\u3002 Restore \u547d\u4ee4 \u00a7 Restore \u547d\u4ee4\u53ef\u4ee5\u89e3\u51bb\u5f52\u6863\u7c7b\u578b\uff08Archive\uff09\u6216\u51b7\u5f52\u6863\uff08Cold Archive\uff09\u7684\u76ee\u5f55\u3002 jindo oss -restore [-days <days>] <path>","title":"JindoShell CLI \u652f\u6301\u963f\u91cc\u4e91 OSS \u4f7f\u7528\u8bf4\u660e"},{"location":"backup/oss/jindoshell_oss_howto/#jindoshell-cli-oss","text":"JindoShell CLI \u652f\u6301\u963f\u91cc\u4e91 OSS archive, unarchive, restore \u7b49\u547d\u4ee4\u3002","title":"JindoShell CLI \u652f\u6301\u963f\u91cc\u4e91 OSS \u4f7f\u7528\u8bf4\u660e"},{"location":"backup/oss/jindoshell_oss_howto/#_1","text":"","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/oss/jindoshell_oss_howto/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/oss/jindoshell_oss_howto/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/oss/jindoshell_oss_howto/#archive","text":"Archive\u547d\u4ee4\u53ef\u4ee5\u5f52\u6863OSS\u4e0a\u7684\u6570\u636e\u81f3\u4f4e\u9891\u8bbf\u95ee\u5b58\u50a8\u6216\u8005\u5f52\u6863\u5b58\u50a8\u4e0a\u3002 jindo oss -archive [-i|c] <path> -i\u53c2\u6570\u53ef\u4ee5\u5f52\u6863\u6570\u636e\u81f3OSS\u4f4e\u9891\u5b58\u50a8\u7c7b\u578b\u3002-c\u53c2\u6570\u53ef\u4ee5\u5f52\u6863\u6570\u636e\u81f3OSS\u51b7\u5f52\u6863\u5b58\u50a8\u7c7b\u578b\u3002\u9ed8\u8ba4\u53ef\u4ee5\u5f52\u6863\u6570\u636e\u81f3OSS\u5f52\u6863\u5b58\u50a8\u7c7b\u578b\u3002","title":"Archive \u547d\u4ee4"},{"location":"backup/oss/jindoshell_oss_howto/#unarchive","text":"Unarchive\u547d\u4ee4\u53ef\u4ee5\u5c06\u6570\u636e\u4ece\u5f52\u6863\u5b58\u50a8\u7c7b\u578b\u6062\u590d\u5230\u4f4e\u9891\u5b58\u50a8\u6216\u8005\u6807\u51c6\u5b58\u3002 jindo oss -unarchive [-i|a] <path> -i\u53c2\u6570\u53ef\u4ee5\u6062\u590d\u6570\u636e\u81f3OSS\u4f4e\u9891\u5b58\u50a8\u7c7b\u578b\u3002-a\u53c2\u6570\u53ef\u4ee5\u6062\u590d\u6570\u636e\u81f3OSS\u5f52\u6863\u5b58\u50a8\u7c7b\u578b\u3002\u9ed8\u8ba4\u53ef\u4ee5\u6062\u590d\u6570\u636e\u81f3OSS\u6807\u51c6\u5b58\u50a8\u3002","title":"Unarchive \u547d\u4ee4"},{"location":"backup/oss/jindoshell_oss_howto/#restore","text":"Restore \u547d\u4ee4\u53ef\u4ee5\u89e3\u51bb\u5f52\u6863\u7c7b\u578b\uff08Archive\uff09\u6216\u51b7\u5f52\u6863\uff08Cold Archive\uff09\u7684\u76ee\u5f55\u3002 jindo oss -restore [-days <days>] <path>","title":"Restore \u547d\u4ee4"},{"location":"backup/oss/outline/","text":"JindoSDK + \u963f\u91cc\u4e91 OSS \u7528\u6237\u6587\u6863 \u00a7 \u5feb\u901f\u5165\u95e8 \u00a7 JindoSDK + \u963f\u91cc\u4e91 OSS \u5feb\u901f\u5165\u95e8 Hadoop \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 \u57fa\u672c\u529f\u80fd \u00a7 JindoShell CLI \u652f\u6301\u963f\u91cc\u4e91 OSS \u4f7f\u7528\u8bf4\u660e \u5927\u6570\u636e\u751f\u6001 \u00a7 Spark \u7ec4\u4ef6 \u00a7 Spark \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e Hive \u7ec4\u4ef6 \u00a7 Hive \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e Presto \u7ec4\u4ef6 \u00a7 Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS \u4e0a\u7684\u6570\u636e Impala \u7ec4\u4ef6 \u00a7 Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS \u4e0a\u7684\u6570\u636e HBase \u7ec4\u4ef6 \u00a7 HBase \u4f7f\u7528 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 Flink \u7ec4\u4ef6 \u00a7 Flink \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e Flume \u7ec4\u4ef6 \u00a7 Flume \u4f7f\u7528 JindoSDK \u5199\u5165 OSS Sqoop \u7ec4\u4ef6 \u00a7 Sqoop \u4f7f\u7528 Kite SDK \u8bfb\u5199 OSS AI \u751f\u6001 \u00a7 JindoFuse \u4f7f\u7528 \u00a7 \u4f7f\u7528 JindoFuse \u8bbf\u95ee OSS \u6700\u4f73\u5b9e\u8df5 \u00a7 JindoDistCp \u6570\u636e\u8fc1\u79fb \u00a7 JindoDistCp \u6570\u636e\u8fc1\u79fb JindoTable \u6570\u4ed3\u8fc1\u79fb \u00a7 \u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a \u00a7 JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a \u793e\u533a\u6700\u4f73\u5b9e\u8df5 \u00a7 Flink-table \u4f7f\u7528 JindoSDK + OSS \u4ee3\u66ff HDFS \u5b58\u50a8\u5b9e\u8df5\uff08\u4f5c\u8005\uff1a\u674e\u4f1f\uff09 \u00a7 Flink-table \u4f7f\u7528 JindoSDK + OSS \u4ee3\u66ff HDFS \u5b58\u50a8\u5b9e\u8df5 \u76f8\u5173\u6587\u7ae0 \u00a7 \u4f7f\u7528 JindoSDK \u5927\u5e45\u63d0\u5347 OSS \u6587\u4ef6\u5404\u9879\u64cd\u4f5c\u6027\u80fd \u5e38\u89c1\u95ee\u9898 \u00a7 \u5e38\u89c1\u95ee\u9898 \u5386\u53f2\u7248\u672c \u00a7 JindoSDK 4.6.0 \u7528\u6237\u6587\u6863 \u00a7 JindoSDK 4.5.1 \u7528\u6237\u6587\u6863 \u00a7 JindoSDK 4.5.0 \u7528\u6237\u6587\u6863 \u00a7 JindoSDK 4.4.0 \u7528\u6237\u6587\u6863 \u00a7 JindoSDK 4.3.0 \u7528\u6237\u6587\u6863 \u00a7 JindoSDK 4.2.0 \u7528\u6237\u6587\u6863 \u00a7 JindoSDK 4.1.0 \u7528\u6237\u6587\u6863 \u00a7 JindoSDK 4.0.0 \u7528\u6237\u6587\u6863 \u00a7 JindoSDK 3.x + OSS \u7528\u6237\u6587\u6863 \u00a7","title":"JindoSDK + \u963f\u91cc\u4e91 OSS \u7528\u6237\u6587\u6863"},{"location":"backup/oss/outline/#jindosdk-oss","text":"","title":"JindoSDK + \u963f\u91cc\u4e91 OSS \u7528\u6237\u6587\u6863"},{"location":"backup/oss/outline/#_1","text":"JindoSDK + \u963f\u91cc\u4e91 OSS \u5feb\u901f\u5165\u95e8 Hadoop \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5","title":"\u5feb\u901f\u5165\u95e8"},{"location":"backup/oss/outline/#_2","text":"JindoShell CLI \u652f\u6301\u963f\u91cc\u4e91 OSS \u4f7f\u7528\u8bf4\u660e","title":"\u57fa\u672c\u529f\u80fd"},{"location":"backup/oss/outline/#_3","text":"","title":"\u5927\u6570\u636e\u751f\u6001"},{"location":"backup/oss/outline/#spark","text":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e","title":"Spark \u7ec4\u4ef6"},{"location":"backup/oss/outline/#hive","text":"Hive \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e","title":"Hive \u7ec4\u4ef6"},{"location":"backup/oss/outline/#presto","text":"Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS \u4e0a\u7684\u6570\u636e","title":"Presto \u7ec4\u4ef6"},{"location":"backup/oss/outline/#impala","text":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS \u4e0a\u7684\u6570\u636e","title":"Impala \u7ec4\u4ef6"},{"location":"backup/oss/outline/#hbase","text":"HBase \u4f7f\u7528 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8","title":"HBase \u7ec4\u4ef6"},{"location":"backup/oss/outline/#flink","text":"Flink \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e","title":"Flink \u7ec4\u4ef6"},{"location":"backup/oss/outline/#flume","text":"Flume \u4f7f\u7528 JindoSDK \u5199\u5165 OSS","title":"Flume \u7ec4\u4ef6"},{"location":"backup/oss/outline/#sqoop","text":"Sqoop \u4f7f\u7528 Kite SDK \u8bfb\u5199 OSS","title":"Sqoop \u7ec4\u4ef6"},{"location":"backup/oss/outline/#ai","text":"","title":"AI \u751f\u6001"},{"location":"backup/oss/outline/#jindofuse","text":"\u4f7f\u7528 JindoFuse \u8bbf\u95ee OSS","title":"JindoFuse \u4f7f\u7528"},{"location":"backup/oss/outline/#_4","text":"","title":"\u6700\u4f73\u5b9e\u8df5"},{"location":"backup/oss/outline/#jindodistcp","text":"JindoDistCp \u6570\u636e\u8fc1\u79fb","title":"JindoDistCp \u6570\u636e\u8fc1\u79fb"},{"location":"backup/oss/outline/#jindotable","text":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS","title":"JindoTable \u6570\u4ed3\u8fc1\u79fb"},{"location":"backup/oss/outline/#jindotable-oss","text":"JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a","title":"JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a"},{"location":"backup/oss/outline/#_5","text":"","title":"\u793e\u533a\u6700\u4f73\u5b9e\u8df5"},{"location":"backup/oss/outline/#flink-table-jindosdk-oss-hdfs","text":"Flink-table \u4f7f\u7528 JindoSDK + OSS \u4ee3\u66ff HDFS \u5b58\u50a8\u5b9e\u8df5","title":"Flink-table \u4f7f\u7528 JindoSDK + OSS \u4ee3\u66ff HDFS \u5b58\u50a8\u5b9e\u8df5\uff08\u4f5c\u8005\uff1a\u674e\u4f1f\uff09"},{"location":"backup/oss/outline/#_6","text":"\u4f7f\u7528 JindoSDK \u5927\u5e45\u63d0\u5347 OSS \u6587\u4ef6\u5404\u9879\u64cd\u4f5c\u6027\u80fd","title":"\u76f8\u5173\u6587\u7ae0"},{"location":"backup/oss/outline/#_7","text":"\u5e38\u89c1\u95ee\u9898","title":"\u5e38\u89c1\u95ee\u9898"},{"location":"backup/oss/outline/#_8","text":"","title":"\u5386\u53f2\u7248\u672c"},{"location":"backup/oss/outline/#jindosdk-460","text":"","title":"JindoSDK 4.6.0 \u7528\u6237\u6587\u6863"},{"location":"backup/oss/outline/#jindosdk-451","text":"","title":"JindoSDK 4.5.1 \u7528\u6237\u6587\u6863"},{"location":"backup/oss/outline/#jindosdk-450","text":"","title":"JindoSDK 4.5.0 \u7528\u6237\u6587\u6863"},{"location":"backup/oss/outline/#jindosdk-440","text":"","title":"JindoSDK 4.4.0 \u7528\u6237\u6587\u6863"},{"location":"backup/oss/outline/#jindosdk-430","text":"","title":"JindoSDK 4.3.0 \u7528\u6237\u6587\u6863"},{"location":"backup/oss/outline/#jindosdk-420","text":"","title":"JindoSDK 4.2.0 \u7528\u6237\u6587\u6863"},{"location":"backup/oss/outline/#jindosdk-410","text":"","title":"JindoSDK 4.1.0 \u7528\u6237\u6587\u6863"},{"location":"backup/oss/outline/#jindosdk-400","text":"","title":"JindoSDK 4.0.0 \u7528\u6237\u6587\u6863"},{"location":"backup/oss/outline/#jindosdk-3x-oss","text":"","title":"JindoSDK 3.x + OSS \u7528\u6237\u6587\u6863"},{"location":"backup/oss/configuration/jindosdk_configuration_list/","text":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e \u00a7 SDK\u914d\u7f6e\u9879\u5217\u8868 \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u5ba2\u6237\u7aef\u5199\u5165\u65f6\u7684\u4e34\u65f6\u6587\u4ef6\u76ee\u5f55\uff0c\u53ef\u914d\u7f6e\u591a\u4e2a\uff08\u9017\u53f7\u9694\u5f00\uff09\uff0c\u4f1a\u8f6e\u6d41\u5199\u5165\uff0c\u591a\u7528\u6237\u73af\u5883\u9700\u914d\u7f6e\u53ef\u8bfb\u5199\u6743\u9650 --> <name>fs.oss.tmp.data.dirs</name> <value>/tmp/</value> </property> <property> <!-- \u4e34\u65f6\u6587\u4ef6\u81ea\u6e05\u7406\u670d\u52a1 --> <name>fs.oss.tmp.data.cleaner.enable</name> <value>true</value> </property> <property> <!-- \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u6b21\u6570 --> <name>fs.oss.retry.count</name> <value>5</value> </property> <property> <!-- \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u95f4\u9694\uff08\u6beb\u79d2\uff09 --> <name>fs.oss.retry.interval.millisecond</name> <value>500</value> </property> <property> <!-- \u8bf7\u6c42 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 --> <name>fs.oss.timeout.millisecond</name> <value>30000</value> </property> <property> <!-- \u8fde\u63a5 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 --> <name>fs.oss.connection.timeout.millisecond</name> <value>3000</value> </property> <property> <!-- \u8fde\u63a5 oss \u7684\u8fde\u63a5\u6c60\u5bf9\u6bcf\u4e2ahost\u7684\u6700\u5927\u8fde\u63a5\u6570\uff08\u8d85\u8fc7\u9608\u503c\u5916\u7684\u8fde\u63a5\u4f1a\u4f7f\u7528\u77ed\u8fde\u63a5\uff09 --> <name>fs.oss.max.connections.per.host</name> <value>100</value> </property> <property> <!-- \u5355\u4e2a\u6587\u4ef6 oss \u5e76\u53d1\u4e0a\u4f20\u7ebf\u7a0b\u6570 --> <name>fs.oss.upload.thread.concurrency</name> <value>5</value> </property> <property> <!-- oss \u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u961f\u5217\u5927\u5c0f --> <name>fs.oss.upload.queue.size</name> <value>5</value> </property> <property> <!-- \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u6570 --> <name>fs.oss.upload.max.pending.tasks.per.stream</name> <value>16</value> </property> <property> <!-- \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u6570 --> <name>fs.oss.download.thread.concurrency</name> <value>16</value> </property> <property> <!-- \u6700\u5927\u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 --> <name>fs.oss.read.readahead.max.buffer.count</name> <value>48</value> </property> <property> <!-- oss \u8bfb\u53d6\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 --> <name>fs.oss.read.buffer.size</name> <value>10485764</value> </property> <property> <!-- oss \u5199\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 --> <name>fs.oss.write.buffer.size</name> <value>1048576</value> </property> <property> <!--oss \u5237\u65b0\u7f13\u51b2\u533a\u95f4\u9694\uff08\u6beb\u79d2\uff09\uff0c\u5c0f\u4e8e 0 \u65f6\u4e0d\u751f\u6548 --> <name>fs.oss.flush.interval.millisecond</name> <value>-1</value> </property> <property> <!-- \u6587\u4ef6\u7ea7\u522b crc64 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u6253\u5f00 --> <name>fs.oss.checksum.crc64.enable</name> <value>true</value> </property> <property> <!--\u8bf7\u6c42\u7ea7\u522b md5 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u5173\u95ed --> <name>fs.oss.checksum.md5.enable</name> <value>false</value> </property> <property> <!-- oss \u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u961f\u5217\u5927\u5c0f (\u5f03\u7528) --> <name>fs.oss.download.queue.size</name> <value>5</value> </property> <property> <!-- \u9884\u8bfb oss \u7684 buffer \u5927\u5c0f (\u5f03\u7528) --> <name>fs.oss.read.readahead.buffer.size</name> <value>1048576</value> </property> <property> <!-- \u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 (\u5f03\u7528) --> <name>fs.oss.read.readahead.buffer.count</name> <value>4</value> </property> </configuration> \u3010\u6ce8\uff1a4.5\u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u65b9\u5f0f\u3011","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/oss/configuration/jindosdk_configuration_list/#jindosdk","text":"","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/oss/configuration/jindosdk_configuration_list/#sdk","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <!-- \u5ba2\u6237\u7aef\u5199\u5165\u65f6\u7684\u4e34\u65f6\u6587\u4ef6\u76ee\u5f55\uff0c\u53ef\u914d\u7f6e\u591a\u4e2a\uff08\u9017\u53f7\u9694\u5f00\uff09\uff0c\u4f1a\u8f6e\u6d41\u5199\u5165\uff0c\u591a\u7528\u6237\u73af\u5883\u9700\u914d\u7f6e\u53ef\u8bfb\u5199\u6743\u9650 --> <name>fs.oss.tmp.data.dirs</name> <value>/tmp/</value> </property> <property> <!-- \u4e34\u65f6\u6587\u4ef6\u81ea\u6e05\u7406\u670d\u52a1 --> <name>fs.oss.tmp.data.cleaner.enable</name> <value>true</value> </property> <property> <!-- \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u6b21\u6570 --> <name>fs.oss.retry.count</name> <value>5</value> </property> <property> <!-- \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u95f4\u9694\uff08\u6beb\u79d2\uff09 --> <name>fs.oss.retry.interval.millisecond</name> <value>500</value> </property> <property> <!-- \u8bf7\u6c42 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 --> <name>fs.oss.timeout.millisecond</name> <value>30000</value> </property> <property> <!-- \u8fde\u63a5 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 --> <name>fs.oss.connection.timeout.millisecond</name> <value>3000</value> </property> <property> <!-- \u8fde\u63a5 oss \u7684\u8fde\u63a5\u6c60\u5bf9\u6bcf\u4e2ahost\u7684\u6700\u5927\u8fde\u63a5\u6570\uff08\u8d85\u8fc7\u9608\u503c\u5916\u7684\u8fde\u63a5\u4f1a\u4f7f\u7528\u77ed\u8fde\u63a5\uff09 --> <name>fs.oss.max.connections.per.host</name> <value>100</value> </property> <property> <!-- \u5355\u4e2a\u6587\u4ef6 oss \u5e76\u53d1\u4e0a\u4f20\u7ebf\u7a0b\u6570 --> <name>fs.oss.upload.thread.concurrency</name> <value>5</value> </property> <property> <!-- oss \u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u961f\u5217\u5927\u5c0f --> <name>fs.oss.upload.queue.size</name> <value>5</value> </property> <property> <!-- \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u6570 --> <name>fs.oss.upload.max.pending.tasks.per.stream</name> <value>16</value> </property> <property> <!-- \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u6570 --> <name>fs.oss.download.thread.concurrency</name> <value>16</value> </property> <property> <!-- \u6700\u5927\u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 --> <name>fs.oss.read.readahead.max.buffer.count</name> <value>48</value> </property> <property> <!-- oss \u8bfb\u53d6\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 --> <name>fs.oss.read.buffer.size</name> <value>10485764</value> </property> <property> <!-- oss \u5199\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 --> <name>fs.oss.write.buffer.size</name> <value>1048576</value> </property> <property> <!--oss \u5237\u65b0\u7f13\u51b2\u533a\u95f4\u9694\uff08\u6beb\u79d2\uff09\uff0c\u5c0f\u4e8e 0 \u65f6\u4e0d\u751f\u6548 --> <name>fs.oss.flush.interval.millisecond</name> <value>-1</value> </property> <property> <!-- \u6587\u4ef6\u7ea7\u522b crc64 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u6253\u5f00 --> <name>fs.oss.checksum.crc64.enable</name> <value>true</value> </property> <property> <!--\u8bf7\u6c42\u7ea7\u522b md5 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u5173\u95ed --> <name>fs.oss.checksum.md5.enable</name> <value>false</value> </property> <property> <!-- oss \u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u961f\u5217\u5927\u5c0f (\u5f03\u7528) --> <name>fs.oss.download.queue.size</name> <value>5</value> </property> <property> <!-- \u9884\u8bfb oss \u7684 buffer \u5927\u5c0f (\u5f03\u7528) --> <name>fs.oss.read.readahead.buffer.size</name> <value>1048576</value> </property> <property> <!-- \u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 (\u5f03\u7528) --> <name>fs.oss.read.readahead.buffer.count</name> <value>4</value> </property> </configuration> \u3010\u6ce8\uff1a4.5\u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u65b9\u5f0f\u3011","title":"SDK\u914d\u7f6e\u9879\u5217\u8868"},{"location":"backup/oss/configuration/jindosdk_configuration_list_ini/","text":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e \u00a7 SDK\u914d\u7f6e\u9879\u5217\u8868 \u00a7 \u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 jindosdk.cfg \u4e2d\u3002 [jindosdk] # \u5ba2\u6237\u7aef\u5199\u5165\u65f6\u7684\u4e34\u65f6\u6587\u4ef6\u76ee\u5f55\uff0c\u53ef\u914d\u7f6e\u591a\u4e2a\uff08\u9017\u53f7\u9694\u5f00\uff09\uff0c\u4f1a\u8f6e\u6d41\u5199\u5165\uff0c\u591a\u7528\u6237\u73af\u5883\u9700\u914d\u7f6e\u53ef\u8bfb\u5199\u6743\u9650 fs.oss.tmp.data.dirs=/tmp/ # \u4e34\u65f6\u6587\u4ef6\u81ea\u6e05\u7406\u670d\u52a1 fs.oss.tmp.data.cleaner.enable=true # \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u6b21\u6570 fs.oss.retry.count=5 # \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u95f4\u9694\uff08\u6beb\u79d2\uff09 fs.oss.retry.interval.millisecond=500 # \u8bf7\u6c42 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 fs.oss.timeout.millisecond=30000 # \u8fde\u63a5 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 fs.oss.connection.timeout.millisecond=3000 # \u8fde\u63a5 oss \u7684\u8fde\u63a5\u6c60\u5bf9\u6bcf\u4e2ahost\u7684\u6700\u5927\u8fde\u63a5\u6570\uff08\u8d85\u8fc7\u9608\u503c\u5916\u7684\u8fde\u63a5\u4f1a\u4f7f\u7528\u77ed\u8fde\u63a5\uff09 fs.oss.max.connections.per.host=100 # \u5355\u4e2a\u6587\u4ef6 oss \u5e76\u53d1\u4e0a\u4f20\u7ebf\u7a0b\u6570 fs.oss.upload.thread.concurrency=5 # oss \u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u961f\u5217\u5927\u5c0f fs.oss.upload.queue.size=5 # \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u6570 fs.oss.upload.max.pending.tasks.per.stream=16 # \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u6570 fs.oss.download.thread.concurrency=16 # \u6700\u5927\u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 fs.oss.read.readahead.max.buffer.count=48 # \u4f7f\u7528ECS\u514d\u5bc6\u670d\u52a1\uff0c\u907f\u514d\u914d\u7f6eAK\uff08\u4e0d\u63a8\u8350\uff0c\u5efa\u8bae\u4f7f\u7528\u56fa\u5b9aAK\u65b9\u5f0f\u8bbf\u95ee\uff09 fs.oss.provider.endpoint=ECS_ROLE # oss \u8bfb\u53d6\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 fs.oss.read.buffer.size=1048576 # oss \u5199\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 fs.oss.write.buffer.size=1048576 # oss \u5237\u65b0\u7f13\u51b2\u533a\u95f4\u9694\uff08\u6beb\u79d2\uff09\uff0c\u5c0f\u4e8e 0 \u65f6\u4e0d\u751f\u6548 fs.oss.flush.interval.millisecond=-1 # \u6587\u4ef6\u7ea7\u522b crc64 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u6253\u5f00 fs.oss.checksum.crc64.enable=true # \u8bf7\u6c42\u7ea7\u522b md5 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u5173\u95ed fs.oss.checksum.md5.enable=false # oss \u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u961f\u5217\u5927\u5c0f (\u5f03\u7528) fs.oss.download.queue.size=5 # \u9884\u8bfb oss \u7684 buffer \u5927\u5c0f (\u5f03\u7528) fs.oss.read.readahead.buffer.size=1048576 # \u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 (\u5f03\u7528) fs.oss.read.readahead.buffer.count=4 \u3010\u6ce8\uff1a4.5\u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u65b9\u5f0f\u3011","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/oss/configuration/jindosdk_configuration_list_ini/#jindosdk","text":"","title":"JindoSDK \u9ad8\u7ea7\u53c2\u6570\u914d\u7f6e"},{"location":"backup/oss/configuration/jindosdk_configuration_list_ini/#sdk","text":"\u53ef\u6839\u636e\u60c5\u51b5\u5c06\u4ee5\u4e0b\u914d\u7f6e\u6dfb\u52a0\u5230 jindosdk.cfg \u4e2d\u3002 [jindosdk] # \u5ba2\u6237\u7aef\u5199\u5165\u65f6\u7684\u4e34\u65f6\u6587\u4ef6\u76ee\u5f55\uff0c\u53ef\u914d\u7f6e\u591a\u4e2a\uff08\u9017\u53f7\u9694\u5f00\uff09\uff0c\u4f1a\u8f6e\u6d41\u5199\u5165\uff0c\u591a\u7528\u6237\u73af\u5883\u9700\u914d\u7f6e\u53ef\u8bfb\u5199\u6743\u9650 fs.oss.tmp.data.dirs=/tmp/ # \u4e34\u65f6\u6587\u4ef6\u81ea\u6e05\u7406\u670d\u52a1 fs.oss.tmp.data.cleaner.enable=true # \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u6b21\u6570 fs.oss.retry.count=5 # \u8bbf\u95ee oss \u5931\u8d25\u91cd\u8bd5\u95f4\u9694\uff08\u6beb\u79d2\uff09 fs.oss.retry.interval.millisecond=500 # \u8bf7\u6c42 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 fs.oss.timeout.millisecond=30000 # \u8fde\u63a5 oss \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 fs.oss.connection.timeout.millisecond=3000 # \u8fde\u63a5 oss \u7684\u8fde\u63a5\u6c60\u5bf9\u6bcf\u4e2ahost\u7684\u6700\u5927\u8fde\u63a5\u6570\uff08\u8d85\u8fc7\u9608\u503c\u5916\u7684\u8fde\u63a5\u4f1a\u4f7f\u7528\u77ed\u8fde\u63a5\uff09 fs.oss.max.connections.per.host=100 # \u5355\u4e2a\u6587\u4ef6 oss \u5e76\u53d1\u4e0a\u4f20\u7ebf\u7a0b\u6570 fs.oss.upload.thread.concurrency=5 # oss \u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u961f\u5217\u5927\u5c0f fs.oss.upload.queue.size=5 # \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u6570 fs.oss.upload.max.pending.tasks.per.stream=16 # \u8fdb\u7a0b\u5185 oss \u6700\u5927\u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u6570 fs.oss.download.thread.concurrency=16 # \u6700\u5927\u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 fs.oss.read.readahead.max.buffer.count=48 # \u4f7f\u7528ECS\u514d\u5bc6\u670d\u52a1\uff0c\u907f\u514d\u914d\u7f6eAK\uff08\u4e0d\u63a8\u8350\uff0c\u5efa\u8bae\u4f7f\u7528\u56fa\u5b9aAK\u65b9\u5f0f\u8bbf\u95ee\uff09 fs.oss.provider.endpoint=ECS_ROLE # oss \u8bfb\u53d6\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 fs.oss.read.buffer.size=1048576 # oss \u5199\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 fs.oss.write.buffer.size=1048576 # oss \u5237\u65b0\u7f13\u51b2\u533a\u95f4\u9694\uff08\u6beb\u79d2\uff09\uff0c\u5c0f\u4e8e 0 \u65f6\u4e0d\u751f\u6548 fs.oss.flush.interval.millisecond=-1 # \u6587\u4ef6\u7ea7\u522b crc64 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u6253\u5f00 fs.oss.checksum.crc64.enable=true # \u8bf7\u6c42\u7ea7\u522b md5 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u9ed8\u8ba4\u5173\u95ed fs.oss.checksum.md5.enable=false # oss \u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u961f\u5217\u5927\u5c0f (\u5f03\u7528) fs.oss.download.queue.size=5 # \u9884\u8bfb oss \u7684 buffer \u5927\u5c0f (\u5f03\u7528) fs.oss.read.readahead.buffer.size=1048576 # \u540c\u65f6\u9884\u8bfb oss \u7684 buffer \u4e2a\u6570 (\u5f03\u7528) fs.oss.read.readahead.buffer.count=4 \u3010\u6ce8\uff1a4.5\u7248\u672c\u4ee5\u4e0a\u7248\u672c\u652f\u6301\u65b9\u5f0f\u3011","title":"SDK\u914d\u7f6e\u9879\u5217\u8868"},{"location":"backup/oss/flink/jindosdk_on_flink/","text":"Flink \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 \u5728\u96c6\u7fa4\u4e0a\u6709\u5f00\u6e90\u7248\u672c Flink \u8f6f\u4ef6\uff0c\u7248\u672c\u4e0d\u4f4e\u4e8e 1.10.1\u3002 \u6ce8\uff1aFlink 1.16.0 \u53ca\u66f4\u9ad8\u7248\u672c\u7684\u53ef\u7528\u6027\u5c1a\u672a\u5f97\u5230\u9a8c\u8bc1\u3002 \u4e3a\u4ec0\u4e48 Flink \u9700\u8981\u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u00a7 \u82e5\u9700\u8981 Flink \u4fdd\u6301 EXACTLY_ONCE \u8bed\u4e49\u6d41\u5f0f\u5199\u5165 Aliyun OSS\uff0c\u5219\u9700\u8981\u4f7f\u7528 JindoSDK\u3002 Apache Flink \u662f\u4e00\u79cd\u5f53\u524d\u4e1a\u754c\u6d41\u884c\u7684\u5f00\u6e90\u5927\u6570\u636e\u6d41\u5f0f\u8ba1\u7b97\u5f15\u64ce\uff0c\u652f\u6301 \u201c\u4e25\u683c\u4e00\u6b21\u201d\uff08EXACTLY_ONCE\uff09\u8bed\u4e49\u5199\u5165\u90e8\u5206\u5b58\u50a8\u4ecb\u8d28\u3002Aliyun Object Storage Service (Aliyun OSS) \u5219\u662f\u963f\u91cc\u4e91\u65d7\u4e0b\u7684\u6838\u5fc3\u4ea7\u54c1\u4e4b\u4e00\uff0c\u63d0\u4f9b\u4e91\u539f\u751f\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1\uff0c\u51ed\u501f\u4f18\u5f02\u7684\u6027\u80fd\u548c\u5b89\u5168\u6027\u5df2\u670d\u52a1\u4e8e\u6d77\u91cf\u7528\u6237\u3002 \u76ee\u524d\uff0c\u5f00\u6e90\u7248\u672c Flink \u5bf9\u6d41\u5f0f\u5199\u5165 Aliyun OSS \u5c1a\u4e0d\u80fd\u652f\u6301 EXACTLY_ONCE \u8bed\u4e49\uff0c\u5982\u6709\u8be5\u9700\u6c42\u5219\u9700\u8981\u4f7f\u7528 JindoSDK\u3002 \u6ce8\uff1aApache Flink \u81ea 1.16 \u7248\u672c\u8d77\u539f\u751f\u63d0\u4f9b\u4e86\u6d41\u5f0f\u5199\u5165 Aliyun OSS \u5bf9\u8c61\u5b58\u50a8\u7684\u652f\u6301\u3002JindoSDK \u63d0\u4f9b\u7684\u5b9e\u73b0\u4e0e Apache Flink \u7684\u81ea\u6709\u5b9e\u73b0\u6709\u4e24\u4e2a\u4e3b\u8981\u533a\u522b\uff1a - JindoSDK \u7684\u5b9e\u73b0\u80fd\u591f\u540c\u65f6\u652f\u6301 Aliyun OSS \u5bf9\u8c61\u5b58\u50a8\u548c Aliyun OSS-HDFS \u6587\u4ef6\u5b58\u50a8\u3002\u5bf9 OSS-HDFS \u7684\u4ecb\u7ecd\u53c2\u8003 OSS-HDFS\u670d\u52a1\u6982\u8ff0 - JindoSDK \u7684\u5b9e\u73b0\u5e95\u5c42\u91c7\u7528\u4e86 Native \u65b9\u6848\u3002 JindoSDK \u914d\u7f6e \u00a7 \u9700\u8981\u5728\u6240\u6709 Flink \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flink \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e .jar \u6587\u4ef6\uff1a * jindo-flink-${version}-full.jar \u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 plugins/flink/ \u76ee\u5f55\u4e0b\u627e\u5230\u3002 \u53e6\u5916\uff0c\u5982\u679c\u96c6\u7fa4\u5185\u5b58\u5728 Apache Flink \u63d0\u4f9b\u7684\u81ea\u5e26 Flink OSS Connector\uff0c\u9700\u79fb\u9664\u3002\u79fb\u9664\u65b9\u6cd5\u4e3a\uff0c\u4ece Flink \u96c6\u7fa4\u7684 lib \u76ee\u5f55\u6216 plugins/oss-fs-hadoop \u79fb\u8d70\u5982\u4e0b jar \u5305\uff1a * flink-oss-fs-hadoop-${flink-version}.jar \u6ce8\uff1aJindoSDK \u76ee\u524d\u63d0\u4f9b\u591a\u5e73\u53f0\u652f\u6301\uff0c\u5f53\u524d\u652f\u6301\u7684\u5e73\u53f0\u4ee5\u53ca\u9700\u8981\u989d\u5916\u653e\u7f6e\u7684\u8d44\u6e90\uff0c\u53c2\u8003 JindoData 4.6.4 \u7248\u672c\u8bf4\u660e \u5982\u4f55\u4f7f\u7528 \u00a7 \u914d\u7f6e\u597d SDK \u540e\uff0c\u65e0\u9700\u989d\u5916\u914d\u7f6e\uff0c\u4ee5\u5e38\u89c4 Flink \u6d41\u5f0f\u4f5c\u4e1a\u65b9\u6cd5\u4f7f\u7528\u5373\u53ef\uff0c\u6ce8\u610f\u4f7f\u7528\u6b63\u786e\u7684\u8def\u5f84\u3002\u5199\u5165 OSS (Aliyun Object Storage Service) \u987b\u4ee5 oss:// \u4e3a\u524d\u7f00\u3002 \u793a\u4f8b \u00a7 \u4e0b\u9762\u662f\u4e00\u4e2a\u7b80\u5355\u7684\u793a\u4f8b\u7a0b\u5e8f\uff0c\u5982\u679c\u5df2\u7ecf\u6309\u7167\u4e0a\u4e00\u8282\u914d\u7f6e\u597d SDK\uff0c\u5219\u4e0b\u5217\u7a0b\u5e8f\u5e94\u80fd\u6b63\u786e\u8fd0\u884c\uff1a (1) \u901a\u7528\u914d\u7f6e \u6253\u5f00 Flink \u7684 \u201c\u68c0\u67e5\u70b9\u201d\uff08Checkpoint\uff09\u3002\u4f8b\u5982\uff0cenv \u662f\u4e00\u4e2a\u5df2\u7ecf\u5efa\u7acb\u7684 StreamExecutionEnvironment\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u65b9\u5f0f\u5efa\u7acb\uff1a StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); \u5219\u53ef\u4ee5\u8fd9\u6837\u542f\u7528 checkpoint\uff1a env.enableCheckpointing(<userDefinedCheckpointInterval>, CheckpointingMode.EXACTLY_ONCE); \u4f7f\u7528\u53ef\u91cd\u53d1\u7684\u6570\u636e\u6e90\uff0c\u4f8b\u5982 Kafka (2) \u793a\u4f8b\u7a0b\u5e8f \u4e0b\u6587\u4e2d\uff0coutputStream \u662f\u4e00\u4e2a\u9884\u5148\u5f62\u6210\u7684 DataStream<String> \u5bf9\u8c61\uff0c\u5e76\u671f\u671b\u5c06\u5176\u5199\u5165 OSS\uff0c\u90a3\u4e48\u53ef\u4ee5\u8fd9\u6837\u6dfb\u52a0 sink\uff1a String outputPath = \"oss://<user-defined-bucket>/<user-defined-dir>\" StreamingFileSink<String> sink = StreamingFileSink.forRowFormat( new Path(outputPath), new SimpleStringEncoder<String>(\"UTF-8\") ).build(); outputStream.addSink(sink); \u6700\u540e\u7528 env.execute() \u6267\u884c Flink \u4f5c\u4e1a\u5373\u53ef\u3002 \u9ad8\u7ea7\u4f7f\u7528\uff1a\u81ea\u5b9a\u4e49\u53c2\u6570 \u00a7 \u7528\u6237\u5728\u63d0\u4ea4 Flink \u4f5c\u4e1a\u65f6\uff0c\u53ef\u4ee5\u81ea\u5b9a\u4e49\u914d\u7f6e\u4e00\u4e9b\u53c2\u6570\uff0c\u4ee5\u5f00\u542f\u6216\u63a7\u5236\u7279\u5b9a\u529f\u80fd\u3002\u4f8b\u5982\uff0c\u4ee5 yarn-cluster \u6a21\u5f0f\u63d0\u4ea4\u65f6\uff0c\u53ef\u4ee5\u901a\u8fc7 -yD \u8fdb\u884c\u914d\u7f6e\uff0c\u65b9\u5f0f\u7c7b\u4f3c\u4e8e\uff1a <flink_home>/bin/flink run -m yarn-cluster -yD key1=value1 -yD key2=value2 ... \u76ee\u524d\u652f\u6301\u901a\u8fc7\u914d\u7f6e\u5f00\u542f \u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\uff0c\u6216\u8005\u63a7\u5236 \u201c\u5206\u7247\u4e0a\u4f20\u201d \u7684\u5e76\u884c\u5ea6\u3002 \u71b5\u6ce8\u5165 \u00a7 \u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\u7684\u76ee\u7684\u662f\uff0c\u5c06\u5199\u5165\u8def\u5f84\u7684\u4e00\u6bb5\u7279\u5b9a\u5b57\u7b26\u4e32\u5339\u914d\u51fa\u6765\uff0c\u7528\u4e00\u6bb5\u968f\u673a\u7684\u5b57\u7b26\u4e32\u8fdb\u884c\u66ff\u6362\uff0c\u4ee5\u524a\u5f31\u6240\u8c13 \u201c\u7247\u533a\u201d (sharding) \u6548\u5e94\uff0c\u63d0\u9ad8\u5199\u5165\u6548\u7387\u3002\u914d\u7f6e\u53c2\u6570\uff1a oss.entropy.key=<user-defined-key> oss.entropy.length=<user-defined-length> \u5982\u6b64\uff0c\u6bcf\u6b21\u6253\u5f00\u5199\u4e00\u4e2a\u65b0\u6587\u4ef6\u65f6\uff0c\u8def\u5f84\u4e2d\u4e0e <user-defined-key> \u76f8\u540c\u7684\u5b57\u7b26\u4e32\u4f1a\u88ab\u66ff\u6362\u4e3a\u4e00\u4e2a\u968f\u673a\u5b57\u7b26\u4e32\uff0c\u968f\u673a\u4e32\u7684\u957f\u5ea6\u4e3a <user-defined-length> \u3002\u914d\u7f6e\u8be5\u53c2\u6570\u65f6\uff0c <user-defined-length> \u5fc5\u987b\u5927\u4e8e\u96f6\u3002 \u5206\u7247\u4e0a\u4f20\u5e76\u884c\u5ea6 \u00a7 \u5199\u5165 OSS \u4f1a\u81ea\u52a8\u8c03\u7528\u9ad8\u6548\u7684 \u201c\u5206\u7247\u4e0a\u4f20\u201d (Multipart Upload) \u673a\u5236\uff0c\u5c06\u5f85\u4e0a\u4f20\u7684\u6587\u4ef6\u5206\u4e3a\u591a\u4e2a\u6570\u636e\u5757 (part) \u5206\u522b\u4e0a\u4f20\uff0c\u6700\u540e\u7ec4\u5408\u3002\u76ee\u524d\u652f\u6301\u914d\u7f6e\u53c2\u6570 oss.upload.max.concurrent.uploads \uff0c\u7528\u6765\u63a7\u5236\u4e0a\u4f20\u6570\u636e\u5757 (part) \u7684\u5e76\u884c\u5ea6\uff0c\u5982\u679c\u8bbe\u7f6e\u8f83\u9ad8\u7684\u6570\u503c\u5219\u53ef\u80fd\u4f1a\u63d0\u9ad8\u5199\u5165\u6548\u7387\uff08\u4f46\u4e5f\u4f1a\u5360\u7528\u66f4\u591a\u8d44\u6e90\uff09\u3002\u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u8be5\u503c\u4e3a\u5f53\u524d\u53ef\u7528\u7684\u5904\u7406\u5668\u6570\u91cf\u3002","title":"Flink \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e"},{"location":"backup/oss/flink/jindosdk_on_flink/#flink-jindosdk-oss","text":"","title":"Flink \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e"},{"location":"backup/oss/flink/jindosdk_on_flink/#_1","text":"\u5728\u96c6\u7fa4\u4e0a\u6709\u5f00\u6e90\u7248\u672c Flink \u8f6f\u4ef6\uff0c\u7248\u672c\u4e0d\u4f4e\u4e8e 1.10.1\u3002 \u6ce8\uff1aFlink 1.16.0 \u53ca\u66f4\u9ad8\u7248\u672c\u7684\u53ef\u7528\u6027\u5c1a\u672a\u5f97\u5230\u9a8c\u8bc1\u3002","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/oss/flink/jindosdk_on_flink/#flink-jindosdk-oss_1","text":"\u82e5\u9700\u8981 Flink \u4fdd\u6301 EXACTLY_ONCE \u8bed\u4e49\u6d41\u5f0f\u5199\u5165 Aliyun OSS\uff0c\u5219\u9700\u8981\u4f7f\u7528 JindoSDK\u3002 Apache Flink \u662f\u4e00\u79cd\u5f53\u524d\u4e1a\u754c\u6d41\u884c\u7684\u5f00\u6e90\u5927\u6570\u636e\u6d41\u5f0f\u8ba1\u7b97\u5f15\u64ce\uff0c\u652f\u6301 \u201c\u4e25\u683c\u4e00\u6b21\u201d\uff08EXACTLY_ONCE\uff09\u8bed\u4e49\u5199\u5165\u90e8\u5206\u5b58\u50a8\u4ecb\u8d28\u3002Aliyun Object Storage Service (Aliyun OSS) \u5219\u662f\u963f\u91cc\u4e91\u65d7\u4e0b\u7684\u6838\u5fc3\u4ea7\u54c1\u4e4b\u4e00\uff0c\u63d0\u4f9b\u4e91\u539f\u751f\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1\uff0c\u51ed\u501f\u4f18\u5f02\u7684\u6027\u80fd\u548c\u5b89\u5168\u6027\u5df2\u670d\u52a1\u4e8e\u6d77\u91cf\u7528\u6237\u3002 \u76ee\u524d\uff0c\u5f00\u6e90\u7248\u672c Flink \u5bf9\u6d41\u5f0f\u5199\u5165 Aliyun OSS \u5c1a\u4e0d\u80fd\u652f\u6301 EXACTLY_ONCE \u8bed\u4e49\uff0c\u5982\u6709\u8be5\u9700\u6c42\u5219\u9700\u8981\u4f7f\u7528 JindoSDK\u3002 \u6ce8\uff1aApache Flink \u81ea 1.16 \u7248\u672c\u8d77\u539f\u751f\u63d0\u4f9b\u4e86\u6d41\u5f0f\u5199\u5165 Aliyun OSS \u5bf9\u8c61\u5b58\u50a8\u7684\u652f\u6301\u3002JindoSDK \u63d0\u4f9b\u7684\u5b9e\u73b0\u4e0e Apache Flink \u7684\u81ea\u6709\u5b9e\u73b0\u6709\u4e24\u4e2a\u4e3b\u8981\u533a\u522b\uff1a - JindoSDK \u7684\u5b9e\u73b0\u80fd\u591f\u540c\u65f6\u652f\u6301 Aliyun OSS \u5bf9\u8c61\u5b58\u50a8\u548c Aliyun OSS-HDFS \u6587\u4ef6\u5b58\u50a8\u3002\u5bf9 OSS-HDFS \u7684\u4ecb\u7ecd\u53c2\u8003 OSS-HDFS\u670d\u52a1\u6982\u8ff0 - JindoSDK \u7684\u5b9e\u73b0\u5e95\u5c42\u91c7\u7528\u4e86 Native \u65b9\u6848\u3002","title":"\u4e3a\u4ec0\u4e48 Flink \u9700\u8981\u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/oss/flink/jindosdk_on_flink/#jindosdk","text":"\u9700\u8981\u5728\u6240\u6709 Flink \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flink \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e .jar \u6587\u4ef6\uff1a * jindo-flink-${version}-full.jar \u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 plugins/flink/ \u76ee\u5f55\u4e0b\u627e\u5230\u3002 \u53e6\u5916\uff0c\u5982\u679c\u96c6\u7fa4\u5185\u5b58\u5728 Apache Flink \u63d0\u4f9b\u7684\u81ea\u5e26 Flink OSS Connector\uff0c\u9700\u79fb\u9664\u3002\u79fb\u9664\u65b9\u6cd5\u4e3a\uff0c\u4ece Flink \u96c6\u7fa4\u7684 lib \u76ee\u5f55\u6216 plugins/oss-fs-hadoop \u79fb\u8d70\u5982\u4e0b jar \u5305\uff1a * flink-oss-fs-hadoop-${flink-version}.jar \u6ce8\uff1aJindoSDK \u76ee\u524d\u63d0\u4f9b\u591a\u5e73\u53f0\u652f\u6301\uff0c\u5f53\u524d\u652f\u6301\u7684\u5e73\u53f0\u4ee5\u53ca\u9700\u8981\u989d\u5916\u653e\u7f6e\u7684\u8d44\u6e90\uff0c\u53c2\u8003 JindoData 4.6.4 \u7248\u672c\u8bf4\u660e","title":"JindoSDK \u914d\u7f6e"},{"location":"backup/oss/flink/jindosdk_on_flink/#_2","text":"\u914d\u7f6e\u597d SDK \u540e\uff0c\u65e0\u9700\u989d\u5916\u914d\u7f6e\uff0c\u4ee5\u5e38\u89c4 Flink \u6d41\u5f0f\u4f5c\u4e1a\u65b9\u6cd5\u4f7f\u7528\u5373\u53ef\uff0c\u6ce8\u610f\u4f7f\u7528\u6b63\u786e\u7684\u8def\u5f84\u3002\u5199\u5165 OSS (Aliyun Object Storage Service) \u987b\u4ee5 oss:// \u4e3a\u524d\u7f00\u3002","title":"\u5982\u4f55\u4f7f\u7528"},{"location":"backup/oss/flink/jindosdk_on_flink/#_3","text":"\u4e0b\u9762\u662f\u4e00\u4e2a\u7b80\u5355\u7684\u793a\u4f8b\u7a0b\u5e8f\uff0c\u5982\u679c\u5df2\u7ecf\u6309\u7167\u4e0a\u4e00\u8282\u914d\u7f6e\u597d SDK\uff0c\u5219\u4e0b\u5217\u7a0b\u5e8f\u5e94\u80fd\u6b63\u786e\u8fd0\u884c\uff1a (1) \u901a\u7528\u914d\u7f6e \u6253\u5f00 Flink \u7684 \u201c\u68c0\u67e5\u70b9\u201d\uff08Checkpoint\uff09\u3002\u4f8b\u5982\uff0cenv \u662f\u4e00\u4e2a\u5df2\u7ecf\u5efa\u7acb\u7684 StreamExecutionEnvironment\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u65b9\u5f0f\u5efa\u7acb\uff1a StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); \u5219\u53ef\u4ee5\u8fd9\u6837\u542f\u7528 checkpoint\uff1a env.enableCheckpointing(<userDefinedCheckpointInterval>, CheckpointingMode.EXACTLY_ONCE); \u4f7f\u7528\u53ef\u91cd\u53d1\u7684\u6570\u636e\u6e90\uff0c\u4f8b\u5982 Kafka (2) \u793a\u4f8b\u7a0b\u5e8f \u4e0b\u6587\u4e2d\uff0coutputStream \u662f\u4e00\u4e2a\u9884\u5148\u5f62\u6210\u7684 DataStream<String> \u5bf9\u8c61\uff0c\u5e76\u671f\u671b\u5c06\u5176\u5199\u5165 OSS\uff0c\u90a3\u4e48\u53ef\u4ee5\u8fd9\u6837\u6dfb\u52a0 sink\uff1a String outputPath = \"oss://<user-defined-bucket>/<user-defined-dir>\" StreamingFileSink<String> sink = StreamingFileSink.forRowFormat( new Path(outputPath), new SimpleStringEncoder<String>(\"UTF-8\") ).build(); outputStream.addSink(sink); \u6700\u540e\u7528 env.execute() \u6267\u884c Flink \u4f5c\u4e1a\u5373\u53ef\u3002","title":"\u793a\u4f8b"},{"location":"backup/oss/flink/jindosdk_on_flink/#_4","text":"\u7528\u6237\u5728\u63d0\u4ea4 Flink \u4f5c\u4e1a\u65f6\uff0c\u53ef\u4ee5\u81ea\u5b9a\u4e49\u914d\u7f6e\u4e00\u4e9b\u53c2\u6570\uff0c\u4ee5\u5f00\u542f\u6216\u63a7\u5236\u7279\u5b9a\u529f\u80fd\u3002\u4f8b\u5982\uff0c\u4ee5 yarn-cluster \u6a21\u5f0f\u63d0\u4ea4\u65f6\uff0c\u53ef\u4ee5\u901a\u8fc7 -yD \u8fdb\u884c\u914d\u7f6e\uff0c\u65b9\u5f0f\u7c7b\u4f3c\u4e8e\uff1a <flink_home>/bin/flink run -m yarn-cluster -yD key1=value1 -yD key2=value2 ... \u76ee\u524d\u652f\u6301\u901a\u8fc7\u914d\u7f6e\u5f00\u542f \u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\uff0c\u6216\u8005\u63a7\u5236 \u201c\u5206\u7247\u4e0a\u4f20\u201d \u7684\u5e76\u884c\u5ea6\u3002","title":"\u9ad8\u7ea7\u4f7f\u7528\uff1a\u81ea\u5b9a\u4e49\u53c2\u6570"},{"location":"backup/oss/flink/jindosdk_on_flink/#_5","text":"\u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\u7684\u76ee\u7684\u662f\uff0c\u5c06\u5199\u5165\u8def\u5f84\u7684\u4e00\u6bb5\u7279\u5b9a\u5b57\u7b26\u4e32\u5339\u914d\u51fa\u6765\uff0c\u7528\u4e00\u6bb5\u968f\u673a\u7684\u5b57\u7b26\u4e32\u8fdb\u884c\u66ff\u6362\uff0c\u4ee5\u524a\u5f31\u6240\u8c13 \u201c\u7247\u533a\u201d (sharding) \u6548\u5e94\uff0c\u63d0\u9ad8\u5199\u5165\u6548\u7387\u3002\u914d\u7f6e\u53c2\u6570\uff1a oss.entropy.key=<user-defined-key> oss.entropy.length=<user-defined-length> \u5982\u6b64\uff0c\u6bcf\u6b21\u6253\u5f00\u5199\u4e00\u4e2a\u65b0\u6587\u4ef6\u65f6\uff0c\u8def\u5f84\u4e2d\u4e0e <user-defined-key> \u76f8\u540c\u7684\u5b57\u7b26\u4e32\u4f1a\u88ab\u66ff\u6362\u4e3a\u4e00\u4e2a\u968f\u673a\u5b57\u7b26\u4e32\uff0c\u968f\u673a\u4e32\u7684\u957f\u5ea6\u4e3a <user-defined-length> \u3002\u914d\u7f6e\u8be5\u53c2\u6570\u65f6\uff0c <user-defined-length> \u5fc5\u987b\u5927\u4e8e\u96f6\u3002","title":"\u71b5\u6ce8\u5165"},{"location":"backup/oss/flink/jindosdk_on_flink/#_6","text":"\u5199\u5165 OSS \u4f1a\u81ea\u52a8\u8c03\u7528\u9ad8\u6548\u7684 \u201c\u5206\u7247\u4e0a\u4f20\u201d (Multipart Upload) \u673a\u5236\uff0c\u5c06\u5f85\u4e0a\u4f20\u7684\u6587\u4ef6\u5206\u4e3a\u591a\u4e2a\u6570\u636e\u5757 (part) \u5206\u522b\u4e0a\u4f20\uff0c\u6700\u540e\u7ec4\u5408\u3002\u76ee\u524d\u652f\u6301\u914d\u7f6e\u53c2\u6570 oss.upload.max.concurrent.uploads \uff0c\u7528\u6765\u63a7\u5236\u4e0a\u4f20\u6570\u636e\u5757 (part) \u7684\u5e76\u884c\u5ea6\uff0c\u5982\u679c\u8bbe\u7f6e\u8f83\u9ad8\u7684\u6570\u503c\u5219\u53ef\u80fd\u4f1a\u63d0\u9ad8\u5199\u5165\u6548\u7387\uff08\u4f46\u4e5f\u4f1a\u5360\u7528\u66f4\u591a\u8d44\u6e90\uff09\u3002\u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u8be5\u503c\u4e3a\u5f53\u524d\u53ef\u7528\u7684\u5904\u7406\u5668\u6570\u91cf\u3002","title":"\u5206\u7247\u4e0a\u4f20\u5e76\u884c\u5ea6"},{"location":"backup/oss/flume/jindosdk_on_flume/","text":"Flume \u4f7f\u7528 JindoSDK \u5199\u5165 OSS \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 \u5728\u96c6\u7fa4\u4e0a\u5df2\u7ecf\u90e8\u7f72 Flume\uff0c\u5df2\u90e8\u7f72 JindoSDK 4.0 \u4ee5\u4e0a\u7248\u672c\u3002 SDK \u914d\u7f6e \u00a7 \u9700\u8981\u5728 Flume \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flume \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e JindoSDK\u3002 \u4e0b\u8f7d\u5e76\u89e3\u538b\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 ) cp jindosdk-x.x.x/lib/*.jar $FLUME_HOME/jars/ Sink \u914d\u7f6e\u793a\u4f8b \u00a7 # \u914d\u7f6e OSS Sink xxx.sinks.oss_sink.hdfs.path = oss://${your_bucket}/flume_dir/%Y-%m-%d/%H # Sink\u53c2\u6570\uff0cbatchSize \u9700\u8981\u8bbe\u7f6e\u5927\u4e00\u4e9b\uff0c\u63a8\u8350\u6bcf\u6b21 Flush \u7684\u91cf\u5728 32MB\u4ee5\u4e0a\uff0c\u5426\u5219\u4f1a\u5f71\u54cd\u6027\u80fd xxx.sinks.oss_sink.hdfs.batchSize = 100000 ... xxx.sinks.oss_sink.hdfs.round = true xxx.sinks.oss_sink.hdfs.roundValue = 15 xxx.sinks.oss_sink.hdfs.Unit = minute xxx.sinks.oss_sink.hdfs.filePrefix = your_topic xxx.sinks.oss_sink.rollSize = 3600 xxx.sinks.oss_sink.threadsPoolSize = 30 ...","title":"Flume \u4f7f\u7528 JindoSDK \u5199\u5165 OSS"},{"location":"backup/oss/flume/jindosdk_on_flume/#flume-jindosdk-oss","text":"","title":"Flume \u4f7f\u7528 JindoSDK \u5199\u5165 OSS"},{"location":"backup/oss/flume/jindosdk_on_flume/#_1","text":"\u5728\u96c6\u7fa4\u4e0a\u5df2\u7ecf\u90e8\u7f72 Flume\uff0c\u5df2\u90e8\u7f72 JindoSDK 4.0 \u4ee5\u4e0a\u7248\u672c\u3002","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/oss/flume/jindosdk_on_flume/#sdk","text":"\u9700\u8981\u5728 Flume \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flume \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e JindoSDK\u3002 \u4e0b\u8f7d\u5e76\u89e3\u538b\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 ) cp jindosdk-x.x.x/lib/*.jar $FLUME_HOME/jars/","title":"SDK \u914d\u7f6e"},{"location":"backup/oss/flume/jindosdk_on_flume/#sink","text":"# \u914d\u7f6e OSS Sink xxx.sinks.oss_sink.hdfs.path = oss://${your_bucket}/flume_dir/%Y-%m-%d/%H # Sink\u53c2\u6570\uff0cbatchSize \u9700\u8981\u8bbe\u7f6e\u5927\u4e00\u4e9b\uff0c\u63a8\u8350\u6bcf\u6b21 Flush \u7684\u91cf\u5728 32MB\u4ee5\u4e0a\uff0c\u5426\u5219\u4f1a\u5f71\u54cd\u6027\u80fd xxx.sinks.oss_sink.hdfs.batchSize = 100000 ... xxx.sinks.oss_sink.hdfs.round = true xxx.sinks.oss_sink.hdfs.roundValue = 15 xxx.sinks.oss_sink.hdfs.Unit = minute xxx.sinks.oss_sink.hdfs.filePrefix = your_topic xxx.sinks.oss_sink.rollSize = 3600 xxx.sinks.oss_sink.threadsPoolSize = 30 ...","title":"Sink \u914d\u7f6e\u793a\u4f8b"},{"location":"backup/oss/hadoop/jindosdk_ide_hadoop/","text":"Hadoop \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 \u00a7 \u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindosdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.net.URI; public class TestJindoSDK { public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. FileSystem fs = FileSystem.get(URI.create(\"oss://<bucket>/\"), conf); FSDataInputStream in = fs.open(new Path(\"/uttest/file1\")); in.read(); in.close(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency>","title":"Hadoop \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5"},{"location":"backup/oss/hadoop/jindosdk_ide_hadoop/#hadoop-jindosdk-ide","text":"\u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindosdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.net.URI; public class TestJindoSDK { public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. FileSystem fs = FileSystem.get(URI.create(\"oss://<bucket>/\"), conf); FSDataInputStream in = fs.open(new Path(\"/uttest/file1\")); in.read(); in.close(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency>","title":"Hadoop \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5"},{"location":"backup/oss/hadoop/jindosdk_on_hadoop/","text":"Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u00a7 JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0\u3002 \u5373\u4f7f\u60a8\u4f7f\u7528 JindoSDK \u4ec5\u4ec5\u4f5c\u4e3a OSS \u5ba2\u6237\u7aef\uff0c\u76f8\u5bf9\u4e8e Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\u5b9e\u73b0\uff0c\u60a8\u8fd8\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u76ee\u524d\u652f\u6301\u5e02\u9762\u4e0a\u5927\u90e8\u5206 Hadoop \u7248\u672c\uff0c\u5728 Hadoop 2.3 \u53ca\u4ee5\u4e0a\u7684\u7248\u672c\u4e0a\u9a8c\u8bc1\u901a\u8fc7\uff082.3 \u4ee5\u524d\u7248\u672c\u6682\u672a\u6d4b\u8bd5\uff0c\u5982\u6709\u95ee\u9898\u8bf7 \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988\uff09\u3002 \u5173\u4e8e JindoSDK \u548c Hadoop \u793e\u533a OSS connector \u7684\u6027\u80fd\u5bf9\u6bd4\uff0c\u8bf7\u53c2\u8003\u6587\u6863 JindoSDK\u548cHadoop-OSS-SDK\u6027\u80fd\u5bf9\u6bd4\u6d4b\u8bd5 \u3002 \u6b65\u9aa4 \u00a7 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06 OSS bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 4. \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u00a7 \u7528Hadoop Shell\u8bbf\u95eeOSS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls oss://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<bucket>/<path> 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 6. \u6743\u9650\u7ba1\u7406 \u00a7 \u81ea\u5efa\u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u963f\u91cc\u4e91 E-MapReduce \u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848","title":"Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/oss/hadoop/jindosdk_on_hadoop/#hadoop-jindosdk-oss","text":"JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0\u3002 \u5373\u4f7f\u60a8\u4f7f\u7528 JindoSDK \u4ec5\u4ec5\u4f5c\u4e3a OSS \u5ba2\u6237\u7aef\uff0c\u76f8\u5bf9\u4e8e Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\u5b9e\u73b0\uff0c\u60a8\u8fd8\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u76ee\u524d\u652f\u6301\u5e02\u9762\u4e0a\u5927\u90e8\u5206 Hadoop \u7248\u672c\uff0c\u5728 Hadoop 2.3 \u53ca\u4ee5\u4e0a\u7684\u7248\u672c\u4e0a\u9a8c\u8bc1\u901a\u8fc7\uff082.3 \u4ee5\u524d\u7248\u672c\u6682\u672a\u6d4b\u8bd5\uff0c\u5982\u6709\u95ee\u9898\u8bf7 \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988\uff09\u3002 \u5173\u4e8e JindoSDK \u548c Hadoop \u793e\u533a OSS connector \u7684\u6027\u80fd\u5bf9\u6bd4\uff0c\u8bf7\u53c2\u8003\u6587\u6863 JindoSDK\u548cHadoop-OSS-SDK\u6027\u80fd\u5bf9\u6bd4\u6d4b\u8bd5 \u3002","title":"Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/oss/hadoop/jindosdk_on_hadoop/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/oss/hadoop/jindosdk_on_hadoop/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/oss/hadoop/jindosdk_on_hadoop/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/oss/hadoop/jindosdk_on_hadoop/#3-oss-access-key","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06 OSS bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002","title":"3. \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"backup/oss/hadoop/jindosdk_on_hadoop/#4-jindosdk-oss","text":"\u7528Hadoop Shell\u8bbf\u95eeOSS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls oss://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<bucket>/<path>","title":"4. \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/oss/hadoop/jindosdk_on_hadoop/#5","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/oss/hadoop/jindosdk_on_hadoop/#6","text":"\u81ea\u5efa\u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u963f\u91cc\u4e91 E-MapReduce \u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848","title":"6. \u6743\u9650\u7ba1\u7406"},{"location":"backup/oss/hbase/jindosdk_on_hbase/","text":"HBase \u4f7f\u7528 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 \u00a7 HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\uff0c\u4f46\u9700\u901a\u8fc7\u672c\u5730 HDFS \u96c6\u7fa4\u5b58\u50a8 WAL \u6587\u4ef6\u3002 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06 OSS bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84 \u00a7 \u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84\uff0c\u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u7684\u503c\u4e3a OSS \u5730\u5740\uff0c\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84\u548c WAL \u6587\u4ef6\u7684\u5b58\u50a8\u8def\u5f84\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 OSS\u3002\u53c2\u6570\u503c\u4e3a oss://bucket/hbase-root-dir","title":"HBase \u4f7f\u7528 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8"},{"location":"backup/oss/hbase/jindosdk_on_hbase/#hbase-oss","text":"HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\uff0c\u4f46\u9700\u901a\u8fc7\u672c\u5730 HDFS \u96c6\u7fa4\u5b58\u50a8 WAL \u6587\u4ef6\u3002","title":"HBase \u4f7f\u7528 OSS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8"},{"location":"backup/oss/hbase/jindosdk_on_hbase/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/oss/hbase/jindosdk_on_hbase/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"backup/oss/hbase/jindosdk_on_hbase/#3-oss-access-key","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06 OSS bucket \u5bf9\u5e94 \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002","title":"3. \u914d\u7f6e OSS \u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"backup/oss/hbase/jindosdk_on_hbase/#hbase","text":"\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84\uff0c\u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u7684\u503c\u4e3a OSS \u5730\u5740\uff0c\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84\u548c WAL \u6587\u4ef6\u7684\u5b58\u50a8\u8def\u5f84\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 OSS\u3002\u53c2\u6570\u503c\u4e3a oss://bucket/hbase-root-dir","title":"\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84"},{"location":"backup/oss/hive/jindosdk_on_hive/","text":"Hive \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e \u00a7 Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS \u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002 \u5b89\u88c5\u914d\u7f6e \u00a7 1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/ 2. \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> 3. \u914d\u7f6e OSS Access Key \u00a7 \u5c06 OSS \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u6700\u540e\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 4. Hive on MR \u00a7 \u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002 5. Hive on Tez \u00a7 \u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002 6. Hive on Spark \u00a7 \u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002 OSS \u7528\u4e8e\u8868\u7684\u5b58\u50a8 \u00a7 \u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS \u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS \u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'oss://bucketname/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'oss://bucketname/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS \u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>oss://bucketname/path/to/warehouse</value> </property> </configuration> \u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a \u00a7 ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'oss://bucketname/path/to/us/part210301cn'; \u6e05\u7406\u56de\u6536\u7ad9 \u00a7 \u5f53\u4f7f\u7528 Hive \u5220\u9664 OSS \u7684\u8868\u65f6\uff0c\u82e5\u5f00\u542f\u4e86\u56de\u6536\u7ad9\u529f\u80fd\uff0c\u4f1a\u5728 OSS \u7684 bucket \u4e0a\u5b58\u50a8\u88ab\u5220\u9664\u7684\u8868\u7684\u6570\u636e\uff0c\u56de\u6536\u7ad9\u7684\u76ee\u5f55\u4e3a oss://buckect-name/user/<username>/.Trash \u3002\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e OSS \u6240\u4f7f\u7528\u7684\u56de\u6536\u7ad9\u76ee\u5f55\u7684\u751f\u547d\u5468\u671f\u6765\u6e05\u7406\u56de\u6536\u7ad9\u3002 \u5173\u4e8e\u5982\u4f55\u8bbe\u7f6e OSS \u751f\u547d\u5468\u671f\uff0c\u8bf7\u53c2\u8003 \u8bbe\u7f6e\u751f\u547d\u5468\u671f\u89c4\u5219 \u82e5\u8981\u8df3\u8fc7\u56de\u6536\u7ad9\uff0c\u76f4\u63a5\u5220\u9664\u8868\uff0c\u5219\u53ef\u4ee5\u6307\u5b9a PURGE \u53c2\u6570\uff08\u614e\u7528\uff09\u3002 DROP TABLE [IF EXISTS] table_name [PURGE] \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Hive \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e"},{"location":"backup/oss/hive/jindosdk_on_hive/#hive-jindosdk-oss","text":"Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS \u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002","title":"Hive \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e"},{"location":"backup/oss/hive/jindosdk_on_hive/#_1","text":"","title":"\u5b89\u88c5\u914d\u7f6e"},{"location":"backup/oss/hive/jindosdk_on_hive/#1-hive-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/","title":"1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002"},{"location":"backup/oss/hive/jindosdk_on_hive/#2-jindosdk-oss","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration>","title":"2. \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b"},{"location":"backup/oss/hive/jindosdk_on_hive/#3-oss-access-key","text":"\u5c06 OSS \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u6700\u540e\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002","title":"3. \u914d\u7f6e OSS Access Key"},{"location":"backup/oss/hive/jindosdk_on_hive/#4-hive-on-mr","text":"\u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002","title":"4. Hive on MR"},{"location":"backup/oss/hive/jindosdk_on_hive/#5-hive-on-tez","text":"\u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002","title":"5. Hive on Tez"},{"location":"backup/oss/hive/jindosdk_on_hive/#6-hive-on-spark","text":"\u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002","title":"6. Hive on Spark"},{"location":"backup/oss/hive/jindosdk_on_hive/#oss","text":"\u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS \u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS \u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'oss://bucketname/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'oss://bucketname/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS \u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>oss://bucketname/path/to/warehouse</value> </property> </configuration>","title":"OSS \u7528\u4e8e\u8868\u7684\u5b58\u50a8"},{"location":"backup/oss/hive/jindosdk_on_hive/#oss_1","text":"ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'oss://bucketname/path/to/us/part210301cn';","title":"\u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a"},{"location":"backup/oss/hive/jindosdk_on_hive/#_2","text":"\u5f53\u4f7f\u7528 Hive \u5220\u9664 OSS \u7684\u8868\u65f6\uff0c\u82e5\u5f00\u542f\u4e86\u56de\u6536\u7ad9\u529f\u80fd\uff0c\u4f1a\u5728 OSS \u7684 bucket \u4e0a\u5b58\u50a8\u88ab\u5220\u9664\u7684\u8868\u7684\u6570\u636e\uff0c\u56de\u6536\u7ad9\u7684\u76ee\u5f55\u4e3a oss://buckect-name/user/<username>/.Trash \u3002\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e OSS \u6240\u4f7f\u7528\u7684\u56de\u6536\u7ad9\u76ee\u5f55\u7684\u751f\u547d\u5468\u671f\u6765\u6e05\u7406\u56de\u6536\u7ad9\u3002 \u5173\u4e8e\u5982\u4f55\u8bbe\u7f6e OSS \u751f\u547d\u5468\u671f\uff0c\u8bf7\u53c2\u8003 \u8bbe\u7f6e\u751f\u547d\u5468\u671f\u89c4\u5219 \u82e5\u8981\u8df3\u8fc7\u56de\u6536\u7ad9\uff0c\u76f4\u63a5\u5220\u9664\u8868\uff0c\u5219\u53ef\u4ee5\u6307\u5b9a PURGE \u53c2\u6570\uff08\u614e\u7528\uff09\u3002 DROP TABLE [IF EXISTS] table_name [PURGE]","title":"\u6e05\u7406\u56de\u6536\u7ad9"},{"location":"backup/oss/hive/jindosdk_on_hive/#_3","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"backup/oss/impala/jindosdk_on_impala/","text":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS \u4e0a\u7684\u6570\u636e \u00a7 JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Impala \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Impala \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fddImpala\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728\u6240\u6709 Impala \u8282\u70b9\u5b89\u88c5 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Impala \u7684 classpath \u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $IMPALA_HOME/lib/ 3. \u914d\u7f6e JindoSDK \u00a7 \u914d\u7f6e IMPALA \u4f7f\u7528\u7684 core-site.xml \u914d\u7f6e JindoSDK \u8bbf\u95ee OSS \u00a7 \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 4. \u4f7f\u7528 Impala \u8bbf\u95ee OSS\u3002 \u00a7 \u4e0b\u8f7d\u6d4b\u8bd5\u6570\u636e\u96c6 \u94fe\u63a5 \u4e0a\u4f20\u6570\u636e bash hadoop fs -put test_data oss://bucket/dir \u521b\u5efa\u8868 sql CREATE EXTERNAL TABLE customer_demographics ( `cd_demo_sk` INT, `cd_gender` STRING, `cd_marital_status` STRING, `cd_education_status` STRING, `cd_purchase_estimate` INT, `cd_credit_rating` STRING, `cd_dep_count` INT, `cd_dep_employed_count` INT, `cd_dep_college_count` INT) STORED AS PARQUET LOCATION 'oss://bucket/dir'; \u67e5\u8be2 OSS \u8868 sql select * from customer_demographics; 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS \u4e0a\u7684\u6570\u636e"},{"location":"backup/oss/impala/jindosdk_on_impala/#impala-jindosdk-oss","text":"JindoSDK \u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411 Hadoop/Spark \u751f\u6001\u7684 OSS \u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684 Hadoop FileSystem \u5b9e\u73b0, Impala \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Impala \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS \u4e0a\u7684\u6570\u636e"},{"location":"backup/oss/impala/jindosdk_on_impala/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/oss/impala/jindosdk_on_impala/#1-impala-hadoop","text":"\u786e\u4fddImpala\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Impala \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"backup/oss/impala/jindosdk_on_impala/#2-impala-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Impala \u7684 classpath \u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $IMPALA_HOME/lib/","title":"2. \u5728\u6240\u6709 Impala \u8282\u70b9\u5b89\u88c5 JindoSDK"},{"location":"backup/oss/impala/jindosdk_on_impala/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/oss/impala/jindosdk_on_impala/#impala-core-sitexml-jindosdk-oss","text":"\u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002","title":"\u914d\u7f6e IMPALA \u4f7f\u7528\u7684 core-site.xml \u914d\u7f6e JindoSDK \u8bbf\u95ee OSS"},{"location":"backup/oss/impala/jindosdk_on_impala/#4-impala-oss","text":"\u4e0b\u8f7d\u6d4b\u8bd5\u6570\u636e\u96c6 \u94fe\u63a5 \u4e0a\u4f20\u6570\u636e bash hadoop fs -put test_data oss://bucket/dir \u521b\u5efa\u8868 sql CREATE EXTERNAL TABLE customer_demographics ( `cd_demo_sk` INT, `cd_gender` STRING, `cd_marital_status` STRING, `cd_education_status` STRING, `cd_purchase_estimate` INT, `cd_credit_rating` STRING, `cd_dep_count` INT, `cd_dep_employed_count` INT, `cd_dep_college_count` INT) STORED AS PARQUET LOCATION 'oss://bucket/dir'; \u67e5\u8be2 OSS \u8868 sql select * from customer_demographics;","title":"4. \u4f7f\u7528 Impala \u8bbf\u95ee OSS\u3002"},{"location":"backup/oss/impala/jindosdk_on_impala/#5","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/oss/jindo_distcp/jindo_distcp_QA/","text":"1\u3001list \u6027\u80fd\u4f4e \u00a7 \u73b0\u8c61 \u00a7 \u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u53d1\u73b0 list \u6027\u80fd\u8f83\u6162\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Successfully list objects with prefix xxx/yyy/ in bucket xxx recursive 0 result 315 dur 100036.615031MS \u5176\u4e2d dur 100036.615031MS \u4ee3\u8868\u6b64\u6b21 list \u7528\u65f6\uff0c\u5355\u4f4d\u6beb\u79d2\uff0c\u5982\u4e0a\u8ff0\u4e00\u6b21 list \u8017\u65f6 100 \u79d2\uff0cOSS \u4e0a\u6587\u4ef6\u6b63\u5e38\u7684 list \u901f\u5ea6\u662f 1000 \u6587\u4ef6\u5728 1s \u4ee5\u4e0b\uff0c\u60a8\u53ef\u6839\u636e\u5f53\u524d\u76ee\u5f55\u4e0b\u6587\u4ef6\u6570\u91cf\u6765\u5224\u65ad\u8be5 list \u8017\u65f6\u662f\u5426\u5f02\u5e38\uff0c\u5982\u4e0a\u8ff0\u4fe1\u606f\u663e\u793a list 315 \u4e2a\u6587\u4ef6\u7684\u76ee\u5f55\u9700\u8981 100s \u663e\u7136\u662f\u4e0d\u592a\u6b63\u5e38\u7684\u3002 \u89e3\u51b3\u529e\u6cd5 \u00a7 \u589e\u52a0\u5ba2\u6237\u7aef\u5185\u5b58\uff0c\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u589e\u5927\u5ba2\u6237\u7aef\u5185\u5b58 export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx4096m\"","title":"jindo distcp QA"},{"location":"backup/oss/jindo_distcp/jindo_distcp_QA/#1list","text":"","title":"1\u3001list \u6027\u80fd\u4f4e"},{"location":"backup/oss/jindo_distcp/jindo_distcp_QA/#_1","text":"\u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u53d1\u73b0 list \u6027\u80fd\u8f83\u6162\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Successfully list objects with prefix xxx/yyy/ in bucket xxx recursive 0 result 315 dur 100036.615031MS \u5176\u4e2d dur 100036.615031MS \u4ee3\u8868\u6b64\u6b21 list \u7528\u65f6\uff0c\u5355\u4f4d\u6beb\u79d2\uff0c\u5982\u4e0a\u8ff0\u4e00\u6b21 list \u8017\u65f6 100 \u79d2\uff0cOSS \u4e0a\u6587\u4ef6\u6b63\u5e38\u7684 list \u901f\u5ea6\u662f 1000 \u6587\u4ef6\u5728 1s \u4ee5\u4e0b\uff0c\u60a8\u53ef\u6839\u636e\u5f53\u524d\u76ee\u5f55\u4e0b\u6587\u4ef6\u6570\u91cf\u6765\u5224\u65ad\u8be5 list \u8017\u65f6\u662f\u5426\u5f02\u5e38\uff0c\u5982\u4e0a\u8ff0\u4fe1\u606f\u663e\u793a list 315 \u4e2a\u6587\u4ef6\u7684\u76ee\u5f55\u9700\u8981 100s \u663e\u7136\u662f\u4e0d\u592a\u6b63\u5e38\u7684\u3002","title":"\u73b0\u8c61"},{"location":"backup/oss/jindo_distcp/jindo_distcp_QA/#_2","text":"\u589e\u52a0\u5ba2\u6237\u7aef\u5185\u5b58\uff0c\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u589e\u5927\u5ba2\u6237\u7aef\u5185\u5b58 export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx4096m\"","title":"\u89e3\u51b3\u529e\u6cd5"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/","text":"\u4f7f\u7528 JindoDistCp \u00a7 \u4ecb\u7ecd \u00a7 JindoDistCp\uff08\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u5de5\u5177\uff09\u662f\u7528\u4e8e\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u62f7\u8d1d\u6587\u4ef6\u7684\u5de5\u5177\u3002 \u5b83\u4f7f\u7528 MapReduce \u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3a MapReduce \u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002\u5168\u91cf\u652f\u6301 HDFS->OSS\uff0cHDFS->HDFS\uff0cOSS->HDFS\uff0cOSS->OSS \u7684\u6570\u636e\u62f7\u8d1d\u573a\u666f\uff0c\u63d0\u4f9b\u591a\u79cd\u4e2a\u6027\u5316\u62f7\u8d1d\u53c2\u6570\u548c\u591a\u79cd\u62f7\u8d1d\u7b56\u7565\u3002\u91cd\u70b9\u4f18\u5316 HDFS \u5230 OSS \u7684\u6570\u636e\u62f7\u8d1d\uff0c\u901a\u8fc7\u5b9a\u5236\u5316 CopyCommitter\uff0c\u5b9e\u73b0 No-Rename \u62f7\u8d1d\uff0c\u5e76\u4fdd\u8bc1\u6570\u636e\u62f7\u8d1d\u843d\u5730\u7684\u4e00\u81f4\u6027\u3002\u529f\u80fd\u5168\u91cf\u5bf9\u9f50 S3 DistCp \u548c HDFS DistCp\uff0c\u6027\u80fd\u8f83 HDFS DistCp \u6709\u8f83\u5927\u63d0\u5347\uff0c\u76ee\u6807\u63d0\u4f9b\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177\u3002 \u60a8\u53ef\u4ee5\u4f7f\u7528 JindoDistCp \u8fdb\u884c HDFS \u548c OSS \u4e4b\u95f4\u7684\u6570\u636e\u8fc1\u79fb\uff0c\u76f8\u5bf9\u4e8e Hadoop \u793e\u533a DistCp \u5b9e\u73b0\uff0c\u60a8\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u4f7f\u7528\u6307\u5357 \u00a7 JindoDistCp\u63d0\u4f9bjar\u5305\u5f62\u5f0f\u4f7f\u7528\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528hadoop jar\u547d\u4ee4\u914d\u5408\u4e00\u7cfb\u5217\u53c2\u6570\u6765\u5b8c\u6210distcp\u64cd\u4f5c\u3002 1\u3001\u4f7f\u7528--src\u548c--dest (\u5fc5\u9009) \u00a7 --src \u8868\u793a\u6307\u5b9a\u6e90\u6587\u4ef6\u7684\u8def\u5f84 --dest \u8868\u793a\u76ee\u6807\u6587\u4ef6\u7684\u8def\u5f84 \u5982\u4eceHDFS\u4e0a/opt/tmp\u76ee\u5f55\u62f7\u8d1d\u5230OSS bucket\uff0c\u53ef\u4ee5\u6267\u884c\u5982\u4e0b\u8bed\u53e5 hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht/tmp \u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9adest\u8def\u5f84\u6765\u786e\u5b9a\u62f7\u8d1d\u540e\u7684\u6587\u4ef6\u5c42\u6b21\uff0c\u5982\u60a8\u9700\u8981\u5c06/opt/tmp\u4e0b\u7684\u6587\u4ef6\u62f7\u8d1d\u5230yang-hhht\u8fd9\u4e2abucket\u4e0b\u7684tmp\u76ee\u5f55\u4e0b\uff0c\u5219\u53ef\u4ee5\u4f7f\u7528\u4e0a\u8ff0\u8bed\u53e5\u6765\u5b8c\u6210\u3002\u6b64\u5904\u548cHadoop\u7684distcp\u884c\u4e3a\u6709\u6240\u4e0d\u540c\uff0cJindoDistCp\u4f1a\u9ed8\u8ba4\u5c06src\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6587\u4ef6\u62f7\u8d1d\u5230\u60a8\u6307\u5b9adest\u8def\u5f84\u4e0b\uff0c\u5e76\u4e0d\u5305\u62ec\u5f53\u524d\u7684\u6839\u76ee\u5f55\u540d\u79f0\uff0c\u60a8\u53ef\u4ee5\u5728dest\u4e2d\u6307\u5b9a\u62f7\u8d1d\u8def\u5f84\u7684\u6839\u76ee\u5f55\uff0c\u5982\u679c\u4e0d\u5b58\u5728\u4f1a\u81ea\u52a8\u521b\u5efa\u3002 \u6ce8\u610f\uff1a\u5982\u9700\u62f7\u8d1d\u5355\u4e2a\u6587\u4ef6, dest\u9700\u6307\u5b9a\u4e3a\u76ee\u5f55\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /test.txt --dest oss://destBucket/tmp 2\u3001\u4f7f\u7528--bandWidth \u00a7 \u5728\u60a8\u7684distcp\u8fc7\u7a0b\u4e2d\uff0c\u60a8\u53ef\u4ee5\u6307\u5b9a\u672c\u6b21distcp\u4efb\u52a1\u6240\u7528\u7684\u5355\u673a\u5e26\u5bbd(\u4ee5MB\u4e3a\u5355\u4f4d)\uff0c\u907f\u514d\u5355\u673a\u5360\u7528\u8fc7\u5927\u5e26\u5bbd \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --bandWidth 6 3\u3001\u4f7f\u7528--codec \u00a7 \u539f\u59cb\u6587\u4ef6\u901a\u5e38\u4ee5\u672a\u538b\u7f29\u7684\u6587\u672c\u683c\u5f0f\u8fdb\u5165OSS\u6216\u8005HDFS\u3002\u65e0\u8bba\u662f\u5b58\u50a8\u6210\u672c\u8fd8\u662f\u5bf9\u8be5\u6570\u636e\u8fdb\u884c\u5206\u6790\uff0c\u6b64\u683c\u5f0f\u90fd\u4e0d\u751a\u7406\u60f3\u3002JindoDistCp \u53ef\u4ee5\u4f7f\u7528 --codec \u9009\u9879\u5e2e\u52a9\u60a8\u5728\u7ebf\u9ad8\u6548\u5730\u5b58\u50a8\u6570\u636e\u548c\u538b\u7f29\u6587\u4ef6 \u547d\u4ee4\u793a\u4f8b\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --codec gz --parallelism 20 JindoDistCp \u7684\u5f53\u524d\u7248\u672c\u652f\u6301\u7f16\u89e3\u7801\u5668 gzip\u3001gz\u3001lzo\u3001lzop \u548c snappy \u4ee5\u53ca\u5173\u952e\u5b57 none \u548c keep \uff08\u9ed8\u8ba4\u503c\uff09\u3002\u8fd9\u4e9b\u5173\u952e\u5b57\u542b\u4e49\u5982\u4e0b\uff1a \"none\" \u2013 \u4fdd\u5b58\u4e3a\u672a\u538b\u7f29\u7684\u6587\u4ef6\u3002\u5982\u679c\u6587\u4ef6\u5df2\u538b\u7f29\uff0c\u5219 JindoDistCp \u4f1a\u5c06\u5176\u89e3\u538b\u7f29\u3002 \"keep\" \u2013 \u4e0d\u66f4\u6539\u6587\u4ef6\u538b\u7f29\u5f62\u6001\uff0c\u6309\u539f\u6837\u590d\u5236\u3002 \u6211\u4eec\u68c0\u67e5\u4e00\u4e0b\u76ee\u6807\u6587\u4ef6\u5939\u4e2d\u7684\u6587\u4ef6\uff0c\u8fd9\u4e9b\u6587\u4ef6\u73b0\u5728\u5df2\u7ecf\u7528 gz \u7f16\u89e3\u7801\u5668\u538b\u7f29\u4e86\uff1a [root@emr-header-1 opt]# hdfs dfs -ls oss://yang-hhht/hourly_table/2017-02-01/03 Found 6 items -rw-rw-rw- 1 938 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/000151.sst.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/1.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/2.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/OPTIONS-000109.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/emp01.txt.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/emp06.txt.gz \u5982\u60a8\u5728\u5f00\u6e90Hadoop\u96c6\u7fa4\u73af\u5883\u4e2d\u4f7f\u7528lzo\u538b\u7f29\u529f\u80fd\uff0c\u5219\u60a8\u9700\u8981\u53bb\u5b89\u88c5gplcompression\u7684native\u5e93\u548chadoop-lzo\u5305\uff0c\u5982\u60a8\u7f3a\u5c11\u76f8\u5173\u73af\u5883\uff0c\u5efa\u8bae\u4f7f\u7528\u5176\u4ed6\u538b\u7f29\u65b9\u5f0f\u8fdb\u884c\u538b\u7f29\u3002 4\u3001\u4f7f\u7528\u51b7\u5f52\u6863/\u5f52\u6863/\u4f4e\u9891\u5199\u5165OSS \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--policy\u6765\u6307\u5b9a\u4ee5\u51b7\u5f52\u6863\u3001\u5f52\u6863\u548c\u4f4e\u9891\u7684\u6a21\u5f0f\u5199\u5165OSS\uff0c\u8fdb\u884c\u6570\u636e\u5b58\u50a8\u3002 4.1\u3001\u4f7f\u7528\u51b7\u5f52\u6863\u5199\u5165OSS\uff08coldArchive\uff09 \u00a7 \u76ee\u524d\u53ea\u5728\u90e8\u5206region\u53ef\u7528\uff0c\u5177\u4f53\u53c2\u89c1 OSS\u5b58\u50a8\u7c7b\u578b\u4ecb\u7ecd \uff0c\u4f7f\u7528\u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --policy coldArchive --parallelism 20 4.2\u3001\u4f7f\u7528\u5f52\u6863\u5199\u5165OSS\uff08archive\uff09 \u00a7 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --policy archive --parallelism 20 4.3\u3001\u4f7f\u7528\u4f4e\u9891\u5199\u5165OSS\uff08ia\uff09 \u00a7 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --policy ia --parallelism 20 \u5982\u60a8\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4\u4ee5standard\u53ca\u6807\u51c6\u6a21\u5f0f\u5199\u5165\uff0c\u4e0d\u8fdb\u884c\u51b7\u5f52\u6863\u3001\u5f52\u6863\u548c\u4f4e\u9891\u64cd\u4f5c 5\u3001\u4f7f\u7528--filters \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--filters\u6765\u6307\u5b9a\u6709\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --filters filter.txt \u5047\u8bbe filter.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u4e0d\u4f1a\u5c06\u8def\u5f84\u4e2d\u5e26\u6709 test \u7684\u6587\u4ef6\u62f7\u8d1d\u5230OSS\u3002 6\u3001\u4f7f\u7528--srcPrefixesFile \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--srcPrefixesFile\u6765\u6307\u5b9a\u6709\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --srcPrefixesFile prefixes.txt \u5047\u8bbe prefixes.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u53ea\u6709\u8def\u5f84\u4e2d\u5e26\u6709 test \u7684\u6587\u4ef6\u624d\u4f1a\u62f7\u8d1d\u5230OSS\u3002 7\u3001\u4f7f\u7528--parallelism \u00a7 parallelism \u53c2\u6570\u7528\u6765\u6307\u5b9aMR\u4efb\u52a1\u91cc\u7684 mapreduce.job.maps \u53c2\u6570\uff0c\u60a8\u53ef\u4ee5\u6839\u636e\u96c6\u7fa4\u7684\u8d44\u6e90\u60c5\u51b5\u81ea\u5b9a\u4e49 parallelism \u7684\u5927\u5c0f\u6765\u63a7\u5236 distcp \u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\u3002 \u5982\u4eceHDFS\u4e0a/opt/tmp\u76ee\u5f55\u62f7\u8d1d\u5230OSS bucket\uff0c\u53ef\u4ee5\u6267\u884c hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht/tmp --parallelism 20 8\u3001\u4f7f\u7528--taskBatch \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--taskBatch\u6765\u6307\u5b9a\u6bcf\u4e2adistcp task\u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --taskBatch 10 9\u3001\u4f7f\u7528--tmp \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7tmp\u53c2\u6570\u6307\u5b9a\u4e00\u4e2a\u4e34\u65f6\u76ee\u5f55\u5728hdfs\u4e0a\u4f20\u5b58\u653e\u4e34\u65f6\u6570\u636e\uff0c\u9ed8\u8ba4\u503c\u4e3a/tmp\uff0c\u5373hdfs:///tmp/\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --parallelism 20 --tmp /tmp 10\u3001\u4f7f\u7528OSS AK \u00a7 \u5728 EMR \u5916\u6216\u8005\u514d\u5bc6\u670d\u52a1\u51fa\u73b0\u95ee\u9898\u7684\u60c5\u51b5\u4e0b\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9aAK\u6765\u83b7\u5f97\u8bbf\u95ee OSS \u7684\u6743\u9650\u3002\u60a8\u53ef\u4ee5\u5728\u547d\u4ee4\u4e2d\u4f7f\u7528 --hadoopConf \u9009\u9879\u6765\u6307\u5b9aAK\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --parallelism 20 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) \u60a8\u4e5f\u53ef\u4ee5\u5c06oss\u7684ak\u3001secret\u3001endpoint\u9884\u5148\u914d\u7f6e\u5728 hadoop\u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u907f\u514d\u6bcf\u6b21\u4f7f\u7528\u65f6\u4e34\u65f6\u586b\u5199ak\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> 11\u3001\u4f7f\u7528--disableChecksum \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--disableChecksum\u5173\u95ed\u68c0\u67e5\u6587\u4ef6checksum\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --disableChecksum 12\u3001\u4f7f\u7528--deleteOnSuccess \u00a7 \u6709\u65f6\u5019\u6211\u4eec\u662f\u60f3\u8981\u79fb\u52a8\u6570\u636e\u800c\u4e0d\u662f\u590d\u5236\u6570\u636e\uff0c\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 --deleteOnSuccess \u9009\u9879\u3002\u6b64\u9009\u9879\u7c7b\u4f3c\u4e8emv\u64cd\u4f5c\uff0c\u9996\u5148\u590d\u5236\u6587\u4ef6\uff0c\u7136\u540e\u4ece\u6e90\u4f4d\u7f6e\u5220\u9664\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --deleteOnSuccess --parallelism 20 13\u3001\u4f7f\u7528--enableTransaction \u00a7 JindoDistCp\u9ed8\u8ba4\u4f7f\u7528task\u7ea7\u522b\u5b8c\u6574\u6027\uff0c\u5982\u60a8\u9700\u8981\u4fdd\u8bc1Job\u7ea7\u522b\u7684\u5b8c\u6574\u6027\u4ee5\u53ca\u4fdd\u8bc1Job\u4e4b\u95f4\u7684\u4e8b\u52a1\u652f\u6301\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528--enableTransaction\u53c2\u6570\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --enableTransaction --parallelism 20 14\u3001\u4f7f\u7528--ignore \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7ignore\u53c2\u6570\uff0c\u5ffd\u7565\u6570\u636e\u8fc1\u79fb\u671f\u95f4\u53d1\u751f\u7684\u5f02\u5e38\uff0c\u76f8\u5173\u62a5\u9519\u4e0d\u4f1a\u4e2d\u65ad\u4efb\u52a1\uff0c\u5e76\u6700\u7ec8\u4ee5DistCp Counter\u7684\u5f62\u5f0f\u900f\u51fa\u3002\uff08\u5982\u679c\u5f00\u542fCMS\u7684\u8bdd\uff0c\u4e5f\u4f1a\u4ee5\u6307\u5b9a\u65b9\u5f0f\u8fdb\u884c\u901a\u77e5\u3002\uff09 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --ignore --parallelism 20 15\u3001\u4f7f\u7528--diff \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5b8c\u6210\u540e\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9asrc\u548cdest\u6765\u67e5\u770b\u5f53\u524ddistcp\u7684\u6587\u4ef6\u5dee\u5f02, \u5982\u679csrc\u7684\u6587\u4ef6\u672a\u80fd\u540c\u6b65\u5230dest\u4e0a\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\u3002\u5982\u60a8\u7684distcp\u4efb\u52a1\u5305\u542b\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u5219--diff\u4e0d\u80fd\u663e\u793a\u6b63\u786e\u7684\u6587\u4ef6\u5dee\u5f02\uff0c\u56e0\u4e3a\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u4f1a\u6539\u53d8\u6587\u4ef6\u7684\u5927\u5c0f\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --diff \u5982\u679c\u6587\u4ef6\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter DIFF_FILES=1 \u5982\u679c\u60a8\u7684--dest\u4e3aHDFS\u8def\u5f84\uff0c\u73b0\u652f\u6301 /path \uff0c hdfs://hostname:ip/path \uff0c hdfs://headerIp:ip/path \u7684\u5199\u6cd5\uff0c\u6682\u4e0d\u652f\u6301 hdfs:///path \uff0c hdfs:/path \u548c\u5176\u4ed6\u81ea\u5b9a\u4e49\u5199\u6cd5\u3002 \u5982\u679c\u60f3\u67e5\u770b\u6587\u4ef6\u5143\u6570\u636e\u7684\u5dee\u5f02\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 --diff --preserveMeta \u547d\u4ee4\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --diff --preserveMeta \u5982\u679c\u6587\u4ef6\u5143\u6570\u636e\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5143\u6570\u636e\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter META_FILE_DIFF=1 16\u3001\u4f7f\u7528--update \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7update\u53c2\u6570\u4f7f\u7528\u589e\u91cf\u540c\u6b65\u529f\u80fd\uff0c\u8df3\u8fc7\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u548c\u76ee\u5f55\uff0c\u76f4\u63a5\u5c06src\u4e2d\u65b0\u589e\u6216\u53d1\u751f\u6539\u53d8\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u540c\u6b65\u5230dest\u4e0a\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --update --parallelism 20 17\u3001\u4f7f\u7528--preserveMeta \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht/tmp --preserveMeta 18\u3001\u4f7f\u7528--jobBatch \u00a7 \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--jobBatch\u6765\u6307\u5b9a\u6bcf\u4e2adistcp job\u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10000\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --jobBatch 50000","title":"\u4f7f\u7528 JindoDistCp"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#jindodistcp","text":"","title":"\u4f7f\u7528 JindoDistCp"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#_1","text":"JindoDistCp\uff08\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u5de5\u5177\uff09\u662f\u7528\u4e8e\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u62f7\u8d1d\u6587\u4ef6\u7684\u5de5\u5177\u3002 \u5b83\u4f7f\u7528 MapReduce \u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3a MapReduce \u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002\u5168\u91cf\u652f\u6301 HDFS->OSS\uff0cHDFS->HDFS\uff0cOSS->HDFS\uff0cOSS->OSS \u7684\u6570\u636e\u62f7\u8d1d\u573a\u666f\uff0c\u63d0\u4f9b\u591a\u79cd\u4e2a\u6027\u5316\u62f7\u8d1d\u53c2\u6570\u548c\u591a\u79cd\u62f7\u8d1d\u7b56\u7565\u3002\u91cd\u70b9\u4f18\u5316 HDFS \u5230 OSS \u7684\u6570\u636e\u62f7\u8d1d\uff0c\u901a\u8fc7\u5b9a\u5236\u5316 CopyCommitter\uff0c\u5b9e\u73b0 No-Rename \u62f7\u8d1d\uff0c\u5e76\u4fdd\u8bc1\u6570\u636e\u62f7\u8d1d\u843d\u5730\u7684\u4e00\u81f4\u6027\u3002\u529f\u80fd\u5168\u91cf\u5bf9\u9f50 S3 DistCp \u548c HDFS DistCp\uff0c\u6027\u80fd\u8f83 HDFS DistCp \u6709\u8f83\u5927\u63d0\u5347\uff0c\u76ee\u6807\u63d0\u4f9b\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177\u3002 \u60a8\u53ef\u4ee5\u4f7f\u7528 JindoDistCp \u8fdb\u884c HDFS \u548c OSS \u4e4b\u95f4\u7684\u6570\u636e\u8fc1\u79fb\uff0c\u76f8\u5bf9\u4e8e Hadoop \u793e\u533a DistCp \u5b9e\u73b0\uff0c\u60a8\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd\u548c\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"\u4ecb\u7ecd"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#_2","text":"JindoDistCp\u63d0\u4f9bjar\u5305\u5f62\u5f0f\u4f7f\u7528\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528hadoop jar\u547d\u4ee4\u914d\u5408\u4e00\u7cfb\u5217\u53c2\u6570\u6765\u5b8c\u6210distcp\u64cd\u4f5c\u3002","title":"\u4f7f\u7528\u6307\u5357"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#1-src-dest","text":"--src \u8868\u793a\u6307\u5b9a\u6e90\u6587\u4ef6\u7684\u8def\u5f84 --dest \u8868\u793a\u76ee\u6807\u6587\u4ef6\u7684\u8def\u5f84 \u5982\u4eceHDFS\u4e0a/opt/tmp\u76ee\u5f55\u62f7\u8d1d\u5230OSS bucket\uff0c\u53ef\u4ee5\u6267\u884c\u5982\u4e0b\u8bed\u53e5 hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht/tmp \u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9adest\u8def\u5f84\u6765\u786e\u5b9a\u62f7\u8d1d\u540e\u7684\u6587\u4ef6\u5c42\u6b21\uff0c\u5982\u60a8\u9700\u8981\u5c06/opt/tmp\u4e0b\u7684\u6587\u4ef6\u62f7\u8d1d\u5230yang-hhht\u8fd9\u4e2abucket\u4e0b\u7684tmp\u76ee\u5f55\u4e0b\uff0c\u5219\u53ef\u4ee5\u4f7f\u7528\u4e0a\u8ff0\u8bed\u53e5\u6765\u5b8c\u6210\u3002\u6b64\u5904\u548cHadoop\u7684distcp\u884c\u4e3a\u6709\u6240\u4e0d\u540c\uff0cJindoDistCp\u4f1a\u9ed8\u8ba4\u5c06src\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6587\u4ef6\u62f7\u8d1d\u5230\u60a8\u6307\u5b9adest\u8def\u5f84\u4e0b\uff0c\u5e76\u4e0d\u5305\u62ec\u5f53\u524d\u7684\u6839\u76ee\u5f55\u540d\u79f0\uff0c\u60a8\u53ef\u4ee5\u5728dest\u4e2d\u6307\u5b9a\u62f7\u8d1d\u8def\u5f84\u7684\u6839\u76ee\u5f55\uff0c\u5982\u679c\u4e0d\u5b58\u5728\u4f1a\u81ea\u52a8\u521b\u5efa\u3002 \u6ce8\u610f\uff1a\u5982\u9700\u62f7\u8d1d\u5355\u4e2a\u6587\u4ef6, dest\u9700\u6307\u5b9a\u4e3a\u76ee\u5f55\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /test.txt --dest oss://destBucket/tmp","title":"1\u3001\u4f7f\u7528--src\u548c--dest (\u5fc5\u9009)"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#2-bandwidth","text":"\u5728\u60a8\u7684distcp\u8fc7\u7a0b\u4e2d\uff0c\u60a8\u53ef\u4ee5\u6307\u5b9a\u672c\u6b21distcp\u4efb\u52a1\u6240\u7528\u7684\u5355\u673a\u5e26\u5bbd(\u4ee5MB\u4e3a\u5355\u4f4d)\uff0c\u907f\u514d\u5355\u673a\u5360\u7528\u8fc7\u5927\u5e26\u5bbd \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --bandWidth 6","title":"2\u3001\u4f7f\u7528--bandWidth"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#3-codec","text":"\u539f\u59cb\u6587\u4ef6\u901a\u5e38\u4ee5\u672a\u538b\u7f29\u7684\u6587\u672c\u683c\u5f0f\u8fdb\u5165OSS\u6216\u8005HDFS\u3002\u65e0\u8bba\u662f\u5b58\u50a8\u6210\u672c\u8fd8\u662f\u5bf9\u8be5\u6570\u636e\u8fdb\u884c\u5206\u6790\uff0c\u6b64\u683c\u5f0f\u90fd\u4e0d\u751a\u7406\u60f3\u3002JindoDistCp \u53ef\u4ee5\u4f7f\u7528 --codec \u9009\u9879\u5e2e\u52a9\u60a8\u5728\u7ebf\u9ad8\u6548\u5730\u5b58\u50a8\u6570\u636e\u548c\u538b\u7f29\u6587\u4ef6 \u547d\u4ee4\u793a\u4f8b\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --codec gz --parallelism 20 JindoDistCp \u7684\u5f53\u524d\u7248\u672c\u652f\u6301\u7f16\u89e3\u7801\u5668 gzip\u3001gz\u3001lzo\u3001lzop \u548c snappy \u4ee5\u53ca\u5173\u952e\u5b57 none \u548c keep \uff08\u9ed8\u8ba4\u503c\uff09\u3002\u8fd9\u4e9b\u5173\u952e\u5b57\u542b\u4e49\u5982\u4e0b\uff1a \"none\" \u2013 \u4fdd\u5b58\u4e3a\u672a\u538b\u7f29\u7684\u6587\u4ef6\u3002\u5982\u679c\u6587\u4ef6\u5df2\u538b\u7f29\uff0c\u5219 JindoDistCp \u4f1a\u5c06\u5176\u89e3\u538b\u7f29\u3002 \"keep\" \u2013 \u4e0d\u66f4\u6539\u6587\u4ef6\u538b\u7f29\u5f62\u6001\uff0c\u6309\u539f\u6837\u590d\u5236\u3002 \u6211\u4eec\u68c0\u67e5\u4e00\u4e0b\u76ee\u6807\u6587\u4ef6\u5939\u4e2d\u7684\u6587\u4ef6\uff0c\u8fd9\u4e9b\u6587\u4ef6\u73b0\u5728\u5df2\u7ecf\u7528 gz \u7f16\u89e3\u7801\u5668\u538b\u7f29\u4e86\uff1a [root@emr-header-1 opt]# hdfs dfs -ls oss://yang-hhht/hourly_table/2017-02-01/03 Found 6 items -rw-rw-rw- 1 938 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/000151.sst.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/1.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/2.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/OPTIONS-000109.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/emp01.txt.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://yang-hhht/hourly_table/2017-02-01/03/emp06.txt.gz \u5982\u60a8\u5728\u5f00\u6e90Hadoop\u96c6\u7fa4\u73af\u5883\u4e2d\u4f7f\u7528lzo\u538b\u7f29\u529f\u80fd\uff0c\u5219\u60a8\u9700\u8981\u53bb\u5b89\u88c5gplcompression\u7684native\u5e93\u548chadoop-lzo\u5305\uff0c\u5982\u60a8\u7f3a\u5c11\u76f8\u5173\u73af\u5883\uff0c\u5efa\u8bae\u4f7f\u7528\u5176\u4ed6\u538b\u7f29\u65b9\u5f0f\u8fdb\u884c\u538b\u7f29\u3002","title":"3\u3001\u4f7f\u7528--codec"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#4oss","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--policy\u6765\u6307\u5b9a\u4ee5\u51b7\u5f52\u6863\u3001\u5f52\u6863\u548c\u4f4e\u9891\u7684\u6a21\u5f0f\u5199\u5165OSS\uff0c\u8fdb\u884c\u6570\u636e\u5b58\u50a8\u3002","title":"4\u3001\u4f7f\u7528\u51b7\u5f52\u6863/\u5f52\u6863/\u4f4e\u9891\u5199\u5165OSS"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#41osscoldarchive","text":"\u76ee\u524d\u53ea\u5728\u90e8\u5206region\u53ef\u7528\uff0c\u5177\u4f53\u53c2\u89c1 OSS\u5b58\u50a8\u7c7b\u578b\u4ecb\u7ecd \uff0c\u4f7f\u7528\u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --policy coldArchive --parallelism 20","title":"4.1\u3001\u4f7f\u7528\u51b7\u5f52\u6863\u5199\u5165OSS\uff08coldArchive\uff09"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#42ossarchive","text":"\u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --policy archive --parallelism 20","title":"4.2\u3001\u4f7f\u7528\u5f52\u6863\u5199\u5165OSS\uff08archive\uff09"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#43ossia","text":"\u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --policy ia --parallelism 20 \u5982\u60a8\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4\u4ee5standard\u53ca\u6807\u51c6\u6a21\u5f0f\u5199\u5165\uff0c\u4e0d\u8fdb\u884c\u51b7\u5f52\u6863\u3001\u5f52\u6863\u548c\u4f4e\u9891\u64cd\u4f5c","title":"4.3\u3001\u4f7f\u7528\u4f4e\u9891\u5199\u5165OSS\uff08ia\uff09"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#5-filters","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--filters\u6765\u6307\u5b9a\u6709\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --filters filter.txt \u5047\u8bbe filter.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u4e0d\u4f1a\u5c06\u8def\u5f84\u4e2d\u5e26\u6709 test \u7684\u6587\u4ef6\u62f7\u8d1d\u5230OSS\u3002","title":"5\u3001\u4f7f\u7528--filters"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#6-srcprefixesfile","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--srcPrefixesFile\u6765\u6307\u5b9a\u6709\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --srcPrefixesFile prefixes.txt \u5047\u8bbe prefixes.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u53ea\u6709\u8def\u5f84\u4e2d\u5e26\u6709 test \u7684\u6587\u4ef6\u624d\u4f1a\u62f7\u8d1d\u5230OSS\u3002","title":"6\u3001\u4f7f\u7528--srcPrefixesFile"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#7-parallelism","text":"parallelism \u53c2\u6570\u7528\u6765\u6307\u5b9aMR\u4efb\u52a1\u91cc\u7684 mapreduce.job.maps \u53c2\u6570\uff0c\u60a8\u53ef\u4ee5\u6839\u636e\u96c6\u7fa4\u7684\u8d44\u6e90\u60c5\u51b5\u81ea\u5b9a\u4e49 parallelism \u7684\u5927\u5c0f\u6765\u63a7\u5236 distcp \u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\u3002 \u5982\u4eceHDFS\u4e0a/opt/tmp\u76ee\u5f55\u62f7\u8d1d\u5230OSS bucket\uff0c\u53ef\u4ee5\u6267\u884c hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht/tmp --parallelism 20","title":"7\u3001\u4f7f\u7528--parallelism"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#8-taskbatch","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--taskBatch\u6765\u6307\u5b9a\u6bcf\u4e2adistcp task\u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --taskBatch 10","title":"8\u3001\u4f7f\u7528--taskBatch"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#9-tmp","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7tmp\u53c2\u6570\u6307\u5b9a\u4e00\u4e2a\u4e34\u65f6\u76ee\u5f55\u5728hdfs\u4e0a\u4f20\u5b58\u653e\u4e34\u65f6\u6570\u636e\uff0c\u9ed8\u8ba4\u503c\u4e3a/tmp\uff0c\u5373hdfs:///tmp/\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --parallelism 20 --tmp /tmp","title":"9\u3001\u4f7f\u7528--tmp"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#10oss-ak","text":"\u5728 EMR \u5916\u6216\u8005\u514d\u5bc6\u670d\u52a1\u51fa\u73b0\u95ee\u9898\u7684\u60c5\u51b5\u4e0b\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9aAK\u6765\u83b7\u5f97\u8bbf\u95ee OSS \u7684\u6743\u9650\u3002\u60a8\u53ef\u4ee5\u5728\u547d\u4ee4\u4e2d\u4f7f\u7528 --hadoopConf \u9009\u9879\u6765\u6307\u5b9aAK\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --parallelism 20 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) \u60a8\u4e5f\u53ef\u4ee5\u5c06oss\u7684ak\u3001secret\u3001endpoint\u9884\u5148\u914d\u7f6e\u5728 hadoop\u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u907f\u514d\u6bcf\u6b21\u4f7f\u7528\u65f6\u4e34\u65f6\u586b\u5199ak\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration>","title":"10\u3001\u4f7f\u7528OSS AK"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#11-disablechecksum","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--disableChecksum\u5173\u95ed\u68c0\u67e5\u6587\u4ef6checksum\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --disableChecksum","title":"11\u3001\u4f7f\u7528--disableChecksum"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#12-deleteonsuccess","text":"\u6709\u65f6\u5019\u6211\u4eec\u662f\u60f3\u8981\u79fb\u52a8\u6570\u636e\u800c\u4e0d\u662f\u590d\u5236\u6570\u636e\uff0c\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 --deleteOnSuccess \u9009\u9879\u3002\u6b64\u9009\u9879\u7c7b\u4f3c\u4e8emv\u64cd\u4f5c\uff0c\u9996\u5148\u590d\u5236\u6587\u4ef6\uff0c\u7136\u540e\u4ece\u6e90\u4f4d\u7f6e\u5220\u9664\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --deleteOnSuccess --parallelism 20","title":"12\u3001\u4f7f\u7528--deleteOnSuccess"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#13-enabletransaction","text":"JindoDistCp\u9ed8\u8ba4\u4f7f\u7528task\u7ea7\u522b\u5b8c\u6574\u6027\uff0c\u5982\u60a8\u9700\u8981\u4fdd\u8bc1Job\u7ea7\u522b\u7684\u5b8c\u6574\u6027\u4ee5\u53ca\u4fdd\u8bc1Job\u4e4b\u95f4\u7684\u4e8b\u52a1\u652f\u6301\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528--enableTransaction\u53c2\u6570\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --enableTransaction --parallelism 20","title":"13\u3001\u4f7f\u7528--enableTransaction"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#14-ignore","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7ignore\u53c2\u6570\uff0c\u5ffd\u7565\u6570\u636e\u8fc1\u79fb\u671f\u95f4\u53d1\u751f\u7684\u5f02\u5e38\uff0c\u76f8\u5173\u62a5\u9519\u4e0d\u4f1a\u4e2d\u65ad\u4efb\u52a1\uff0c\u5e76\u6700\u7ec8\u4ee5DistCp Counter\u7684\u5f62\u5f0f\u900f\u51fa\u3002\uff08\u5982\u679c\u5f00\u542fCMS\u7684\u8bdd\uff0c\u4e5f\u4f1a\u4ee5\u6307\u5b9a\u65b9\u5f0f\u8fdb\u884c\u901a\u77e5\u3002\uff09 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --ignore --parallelism 20","title":"14\u3001\u4f7f\u7528--ignore"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#15-diff","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5b8c\u6210\u540e\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9asrc\u548cdest\u6765\u67e5\u770b\u5f53\u524ddistcp\u7684\u6587\u4ef6\u5dee\u5f02, \u5982\u679csrc\u7684\u6587\u4ef6\u672a\u80fd\u540c\u6b65\u5230dest\u4e0a\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\u3002\u5982\u60a8\u7684distcp\u4efb\u52a1\u5305\u542b\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u5219--diff\u4e0d\u80fd\u663e\u793a\u6b63\u786e\u7684\u6587\u4ef6\u5dee\u5f02\uff0c\u56e0\u4e3a\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u4f1a\u6539\u53d8\u6587\u4ef6\u7684\u5927\u5c0f\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --diff \u5982\u679c\u6587\u4ef6\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter DIFF_FILES=1 \u5982\u679c\u60a8\u7684--dest\u4e3aHDFS\u8def\u5f84\uff0c\u73b0\u652f\u6301 /path \uff0c hdfs://hostname:ip/path \uff0c hdfs://headerIp:ip/path \u7684\u5199\u6cd5\uff0c\u6682\u4e0d\u652f\u6301 hdfs:///path \uff0c hdfs:/path \u548c\u5176\u4ed6\u81ea\u5b9a\u4e49\u5199\u6cd5\u3002 \u5982\u679c\u60f3\u67e5\u770b\u6587\u4ef6\u5143\u6570\u636e\u7684\u5dee\u5f02\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 --diff --preserveMeta \u547d\u4ee4\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht.cn-xxx.oss-dls.aliyuncs.com/hourly_table --diff --preserveMeta \u5982\u679c\u6587\u4ef6\u5143\u6570\u636e\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5143\u6570\u636e\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter META_FILE_DIFF=1","title":"15\u3001\u4f7f\u7528--diff"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#16-update","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7update\u53c2\u6570\u4f7f\u7528\u589e\u91cf\u540c\u6b65\u529f\u80fd\uff0c\u8df3\u8fc7\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u548c\u76ee\u5f55\uff0c\u76f4\u63a5\u5c06src\u4e2d\u65b0\u589e\u6216\u53d1\u751f\u6539\u53d8\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u540c\u6b65\u5230dest\u4e0a\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --update --parallelism 20","title":"16\u3001\u4f7f\u7528--update"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#17-preservemeta","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://yang-hhht/tmp --preserveMeta","title":"17\u3001\u4f7f\u7528--preserveMeta"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to/#18-jobbatch","text":"\u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--jobBatch\u6765\u6307\u5b9a\u6bcf\u4e2adistcp job\u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10000\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/incoming/hourly_table --dest oss://yang-hhht/hourly_table --jobBatch 50000","title":"18\u3001\u4f7f\u7528--jobBatch"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_api/","text":"JindoDistCp API \u4f7f\u7528\u8bf4\u660e \u00a7 \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 \u5b89\u88c5\u4f9d\u8d56\u5305 \u00a7 \u76ee\u524d JindoSDK/JindoDistCP \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u4ee54.6.12\u7248\u672c\u4e3a\u4f8b\uff0c\u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoDistCP \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-distcp --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop with provided scope, your runtime should install hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> </dependencies> </project> \u793a\u4f8b\u4ee3\u7801 \u00a7 \u4ee5\u62f7\u8d1d HDFS \u6587\u4ef6\u5939\u5230\u963f\u91cc\u4e91 OSS \u4e3a\u4f8b\u3002 import com.aliyun.jindodata.distcp.util.DistCpCounter; import com.aliyun.jindodata.distjob.request.DistJobResponse; import org.apache.hadoop.conf.Configuration; import java.io.IOException; public class DistcpExample { static public void main(String[] args) throws IOException { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. conf.set(\"fs.oss.credentials.provider\", \"com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider\"); conf.set(\"fs.oss.accessKeyId\", \"XXX\"); conf.set(\"fs.oss.accessKeySecret\", \"XXX\"); conf.set(\"fs.oss.endpoint\", \"oss-cn-xxx.aliyuncs.com\"); DistCpRequest request = new DistCpRequest(); request.setSrcDir(\"hdfs:///user/root/random-data\"); request.setDestDir(\"oss://oss-bucket/dest\"); request.setWorkTempDir(\"hdfs:///user/root/tmp\"); request.setConf(conf); // Jindo DistCp Service DistCpService service = new DistCpService(); service.buildAllJobs(request); service.runAllJobs(); DistJobResponse response = service.collectAllJobResult(); printResponse(response); } static public void printResponse(DistJobResponse response) { System.out.println(\"ErrorCode: \" + response.getErrorCode()); System.out.println(\"ErrorMsg: \" + response.getErrorMsg()); System.out.println(\"FileCopied: \" + response.getCount(DistCpCounter.FILES_COPIED.getName())); System.out.println(\"BytesCopied: \" + response.getCount(DistCpCounter.BYTES_COPIED.getName())); } } \u8bf7\u6c42\u53c2\u6570 \u00a7 \u76f8\u5173\u65b9\u6cd5\u7684\u5177\u4f53\u63cf\u8ff0\u53ef\u53c2\u8003 \u300a\u4f7f\u7528Jindo DistCp\u300b \u65b9\u6cd5 \u53c2\u6570\u7c7b\u578b \u8bf4\u660e \u9ed8\u8ba4\u503c \u517c\u5bb9 void setSrc(String src) \u5fc5\u9009 \u8bbe\u7f6e\u6e90\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setDest(String dest) \u5fc5\u9009 \u8bbe\u7f6e\u76ee\u6807\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setConf(Configuration conf) \u5fc5\u9009 \u8bbe\u7f6e Configuration - 4.3.0+ void setMode(DistCpMode mode) \u53ef\u9009 \u8bbe\u7f6eDistCp\u6a21\u5f0f\uff0c\u652f\u6301\u7684\u6a21\u5f0f\u6709 COPY\uff08\u62f7\u8d1d\uff09 DIFF\uff08\u5dee\u5f02\u6bd4\u8f83\uff09 UPADTE\uff08\u589e\u91cf\u62f7\u8d1d\uff09 DistCpMode.COPY 4.3.0+ void setNumWorkers(int numWorkers) \u53ef\u9009 \u8bbe\u7f6eDistCp\u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\uff0c\u5bf9\u5e94MR\u4efb\u52a1\u4e2d\u7684 mapreduce.job.maps 10 4.3.0+ void setBandWidthLimit(int bandWidthLimit) \u53ef\u9009 \u8bbe\u7f6e\u5355\u4e2a\u8282\u70b9\u7684\u5e26\u5bbd\u9650\u5236\uff0c\u5355\u4f4dM -1 4.3.0+ void setWorkTempDir(String workTempDir) \u53ef\u9009 \u8bbe\u7f6e\u4e34\u65f6\u76ee\u5f55 /tmp 4.3.0+ void setOutputCodec(String outputCodec) \u53ef\u9009 \u8bbe\u7f6e\u538b\u7f29\u7c7b\u578b\uff0c\u652f\u6301\u7f16\u89e3\u7801\u5668\u6709 gzip\u3001gz\u3001lzo\u3001lzop\u3001lzop\u3001snappy keep\uff08\u4e0d\u66f4\u6539\u538b\u7f29\u7c7b\u578b\uff09 4.3.0+ void setEnableCMS(boolean enableCMS) \u53ef\u9009 \u662f\u5426\u5f00\u542f\u76d1\u63a7\u544a\u8b66 false 4.3.0+ void setDisableChecksum(boolean disableChecksum) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5173\u95edChecksum\u68c0\u67e5 false 4.3.0+ void setDeleteOnSuccess(boolean deleteOnSuccess) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5220\u9664\u6e90\u6587\u4ef6\uff0c\u7528\u4e8e\u79fb\u52a8\u6570\u636e false 4.3.0+ void setIncludeMatcherFile(String includeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setExcludeMatcherFile(String excludeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setIgnoreFailures(boolean ignoreFailures) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5ffd\u7565\u62f7\u8d1d\u4efb\u52a1\u4e2d\u629b\u51fa\u7684\u5f02\u5e38\uff0c\u907f\u514d\u4e2d\u65ad\u4efb\u52a1 False 4.3.0+ void setStorageClass(String storageClass) \u53ef\u9009 \u8bbe\u7f6e\u76ee\u6807\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301Standard\u3001IA\u3001Archive\u3001ColdArchive Standard 4.3.0+ void setEnableTransaction(boolean enableTransaction) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4e8b\u52a1\uff0c\u4ee5\u4fdd\u8bc1Job\u7ea7\u522b\u7684\u539f\u5b50\u6027 false 4.3.0+ void setEnablePreserveMetaStatus(boolean enablePreserveMetaStatus) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f false 4.4.0+ \u54cd\u5e94\u53c2\u6570 \u00a7 \u65b9\u6cd5 \u8bf4\u660e \u9ed8\u8ba4\u503c int getErrorCode() \u83b7\u53d6\u9519\u8bef\u7801 0 String getErrorMsg() \u83b7\u53d6\u9519\u8bef\u539f\u56e0 - long getStartTime() \u83b7\u53d6\u4efb\u52a1\u7684\u5f00\u59cb\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - long getEndTime() \u83b7\u53d6\u4efb\u52a1\u7684\u7ed3\u675f\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - getCount(String key) \u83b7\u53d6\u8ba1\u6570\u5668\u503c\uff0ckey\u4e3a\u8ba1\u6570\u5668\u540d\u79f0\uff0c - \u8ba1\u6570\u5668\u8bf4\u660e \u00a7 \u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf BYTES_EXPECTED \u9884\u671f\u7684copy\u5b57\u8282\u6570 FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 BYTES_COPIED copy\u6210\u529f\u7684\u5b57\u8282\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"JindoDistCp API \u4f7f\u7528\u8bf4\u660e"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_api/#jindodistcp-api","text":"","title":"JindoDistCp API \u4f7f\u7528\u8bf4\u660e"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_api/#jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"\u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_api/#_1","text":"\u76ee\u524d JindoSDK/JindoDistCP \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u4ee54.6.12\u7248\u672c\u4e3a\u4f8b\uff0c\u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoDistCP \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-distcp --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop with provided scope, your runtime should install hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> </dependencies> </project>","title":"\u5b89\u88c5\u4f9d\u8d56\u5305"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_api/#_2","text":"\u4ee5\u62f7\u8d1d HDFS \u6587\u4ef6\u5939\u5230\u963f\u91cc\u4e91 OSS \u4e3a\u4f8b\u3002 import com.aliyun.jindodata.distcp.util.DistCpCounter; import com.aliyun.jindodata.distjob.request.DistJobResponse; import org.apache.hadoop.conf.Configuration; import java.io.IOException; public class DistcpExample { static public void main(String[] args) throws IOException { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. conf.set(\"fs.oss.credentials.provider\", \"com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider\"); conf.set(\"fs.oss.accessKeyId\", \"XXX\"); conf.set(\"fs.oss.accessKeySecret\", \"XXX\"); conf.set(\"fs.oss.endpoint\", \"oss-cn-xxx.aliyuncs.com\"); DistCpRequest request = new DistCpRequest(); request.setSrcDir(\"hdfs:///user/root/random-data\"); request.setDestDir(\"oss://oss-bucket/dest\"); request.setWorkTempDir(\"hdfs:///user/root/tmp\"); request.setConf(conf); // Jindo DistCp Service DistCpService service = new DistCpService(); service.buildAllJobs(request); service.runAllJobs(); DistJobResponse response = service.collectAllJobResult(); printResponse(response); } static public void printResponse(DistJobResponse response) { System.out.println(\"ErrorCode: \" + response.getErrorCode()); System.out.println(\"ErrorMsg: \" + response.getErrorMsg()); System.out.println(\"FileCopied: \" + response.getCount(DistCpCounter.FILES_COPIED.getName())); System.out.println(\"BytesCopied: \" + response.getCount(DistCpCounter.BYTES_COPIED.getName())); } }","title":"\u793a\u4f8b\u4ee3\u7801"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_api/#_3","text":"\u76f8\u5173\u65b9\u6cd5\u7684\u5177\u4f53\u63cf\u8ff0\u53ef\u53c2\u8003 \u300a\u4f7f\u7528Jindo DistCp\u300b \u65b9\u6cd5 \u53c2\u6570\u7c7b\u578b \u8bf4\u660e \u9ed8\u8ba4\u503c \u517c\u5bb9 void setSrc(String src) \u5fc5\u9009 \u8bbe\u7f6e\u6e90\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setDest(String dest) \u5fc5\u9009 \u8bbe\u7f6e\u76ee\u6807\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setConf(Configuration conf) \u5fc5\u9009 \u8bbe\u7f6e Configuration - 4.3.0+ void setMode(DistCpMode mode) \u53ef\u9009 \u8bbe\u7f6eDistCp\u6a21\u5f0f\uff0c\u652f\u6301\u7684\u6a21\u5f0f\u6709 COPY\uff08\u62f7\u8d1d\uff09 DIFF\uff08\u5dee\u5f02\u6bd4\u8f83\uff09 UPADTE\uff08\u589e\u91cf\u62f7\u8d1d\uff09 DistCpMode.COPY 4.3.0+ void setNumWorkers(int numWorkers) \u53ef\u9009 \u8bbe\u7f6eDistCp\u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\uff0c\u5bf9\u5e94MR\u4efb\u52a1\u4e2d\u7684 mapreduce.job.maps 10 4.3.0+ void setBandWidthLimit(int bandWidthLimit) \u53ef\u9009 \u8bbe\u7f6e\u5355\u4e2a\u8282\u70b9\u7684\u5e26\u5bbd\u9650\u5236\uff0c\u5355\u4f4dM -1 4.3.0+ void setWorkTempDir(String workTempDir) \u53ef\u9009 \u8bbe\u7f6e\u4e34\u65f6\u76ee\u5f55 /tmp 4.3.0+ void setOutputCodec(String outputCodec) \u53ef\u9009 \u8bbe\u7f6e\u538b\u7f29\u7c7b\u578b\uff0c\u652f\u6301\u7f16\u89e3\u7801\u5668\u6709 gzip\u3001gz\u3001lzo\u3001lzop\u3001lzop\u3001snappy keep\uff08\u4e0d\u66f4\u6539\u538b\u7f29\u7c7b\u578b\uff09 4.3.0+ void setEnableCMS(boolean enableCMS) \u53ef\u9009 \u662f\u5426\u5f00\u542f\u76d1\u63a7\u544a\u8b66 false 4.3.0+ void setDisableChecksum(boolean disableChecksum) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5173\u95edChecksum\u68c0\u67e5 false 4.3.0+ void setDeleteOnSuccess(boolean deleteOnSuccess) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5220\u9664\u6e90\u6587\u4ef6\uff0c\u7528\u4e8e\u79fb\u52a8\u6570\u636e false 4.3.0+ void setIncludeMatcherFile(String includeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setExcludeMatcherFile(String excludeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setIgnoreFailures(boolean ignoreFailures) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5ffd\u7565\u62f7\u8d1d\u4efb\u52a1\u4e2d\u629b\u51fa\u7684\u5f02\u5e38\uff0c\u907f\u514d\u4e2d\u65ad\u4efb\u52a1 False 4.3.0+ void setStorageClass(String storageClass) \u53ef\u9009 \u8bbe\u7f6e\u76ee\u6807\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301Standard\u3001IA\u3001Archive\u3001ColdArchive Standard 4.3.0+ void setEnableTransaction(boolean enableTransaction) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4e8b\u52a1\uff0c\u4ee5\u4fdd\u8bc1Job\u7ea7\u522b\u7684\u539f\u5b50\u6027 false 4.3.0+ void setEnablePreserveMetaStatus(boolean enablePreserveMetaStatus) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f false 4.4.0+","title":"\u8bf7\u6c42\u53c2\u6570"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_api/#_4","text":"\u65b9\u6cd5 \u8bf4\u660e \u9ed8\u8ba4\u503c int getErrorCode() \u83b7\u53d6\u9519\u8bef\u7801 0 String getErrorMsg() \u83b7\u53d6\u9519\u8bef\u539f\u56e0 - long getStartTime() \u83b7\u53d6\u4efb\u52a1\u7684\u5f00\u59cb\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - long getEndTime() \u83b7\u53d6\u4efb\u52a1\u7684\u7ed3\u675f\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - getCount(String key) \u83b7\u53d6\u8ba1\u6570\u5668\u503c\uff0ckey\u4e3a\u8ba1\u6570\u5668\u540d\u79f0\uff0c -","title":"\u54cd\u5e94\u53c2\u6570"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_api/#_5","text":"\u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf BYTES_EXPECTED \u9884\u671f\u7684copy\u5b57\u8282\u6570 FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 BYTES_COPIED copy\u6210\u529f\u7684\u5b57\u8282\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"\u8ba1\u6570\u5668\u8bf4\u660e"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/","text":"JindoDistCp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66 \u00a7 JindoDistCp\uff08\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u5de5\u5177\uff09\u662f\u7528\u4e8e\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u62f7\u8d1d\u6587\u4ef6\u7684\u5de5\u5177\u3002\u4f7f\u7528MapReduce\u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3amap/reduce\u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002 CMS\uff08\u4e91\u76d1\u63a7\u670d\u52a1\uff09\uff0c\u53ef\u7528\u4e8e\u6536\u96c6\u963f\u91cc\u4e91\u8d44\u6e90\u7684\u76d1\u63a7\u6307\u6807\u6216\u7528\u6237\u81ea\u5b9a\u4e49\u7684\u76d1\u63a7\u6307\u6807\uff0c\u63a2\u6d4b\u670d\u52a1\u53ef\u7528\u6027\uff0c\u4ee5\u53ca\u9488\u5bf9\u6307\u6807\u8bbe\u7f6e\u8b66\u62a5\u3002\u4f7f\u60a8\u5168\u9762\u4e86\u89e3\u963f\u91cc\u4e91\u4e0a\u8d44\u6e90\u7684\u4f7f\u7528\u60c5\u51b5\u548c\u4e1a\u52a1\u8fd0\u884c\u72b6\u51b5\uff0c\u5e76\u53ca\u65f6\u5bf9\u6545\u969c\u8d44\u6e90\u8fdb\u884c\u5904\u7406\uff0c\u4fdd\u8bc1\u4e1a\u52a1\u6b63\u5e38\u8fd0\u884c\u3002 \u60a8\u53ef\u4ee5\u6307\u5b9a\u672c\u6b21JindoDistCp\u4efb\u52a1\u7ed3\u675f\u540e\u662f\u5426\u9700\u8981\u5f00\u542fCMS\uff0c\u4ee5\u4e0a\u62a5\u4efb\u52a1\u5931\u8d25\u4fe1\u606f\uff0c\u5e76\u4f7f\u7528CMS\u63a7\u5236\u53f0\u6765\u914d\u7f6e\u544a\u8b66\u529f\u80fd\u3002\u544a\u8b66\u6761\u4ef6\u53c2\u89c1 \u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668 \u3002 \u6b65\u9aa4 \u00a7 1. \u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u548c\u62a5\u8b66\u8054\u7cfb\u7ec4 \u00a7 \u5177\u4f53\u8bf7\u53c2\u8003\u6587\u6863 \u300a\u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u6216\u62a5\u8b66\u8054\u7cfb\u7ec4\u300b 2. \u83b7\u53d6\u62a5\u8b66Token \u00a7 \u70b9\u51fb\u5de6\u4fa7\u5bfc\u822a\u680f\u201c\u62a5\u8b66\u670d\u52a1\u201d->\"\u62a5\u8b66\u8054\u7cfb\u4eba\"->\u5207\u6362\u4e0a\u6d4b\u9875\u7b7e\u5230\"\u62a5\u8b66\u8054\u7cfb\u7ec4\"->\u9009\u62e9\u5bf9\u5e94\u7684\u62a5\u8b66\u8054\u7cfb\u7ec4->\u70b9\u51fb\u53f3\u4fa7\u63a5\u5165\u5916\u90e8\u62a5\u8b66 3. \u914d\u7f6e\u73af\u5883\u53d8\u91cf \u00a7 \u73af\u5883\u53d8\u91cf \u8bf4\u660e cmsAccessKeyId \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94AccessKey ID cmsAccessSecret \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94Access Key Secret cmsRegion \u8bbe\u7f6eCMS\u6240\u5728Region\uff0c\u9700\u8981\u4e0e\u5176\u4ed6\u76f8\u5173\u7684\u4e91\u4e0a\u8d44\u6e90\u4e00\u81f4\uff0c\u4f8b\u5982ECS\uff0cOSS\uff0cEMR\u6240\u5bf9\u5e94\u7684Region cmsLevel \u8bbe\u7f6e\u6807\u8bc6\u544a\u8b66\u901a\u77e5\u65b9\u5f0f\uff0c\u5982\u679c\u672a\u8bbe\u7f6e\uff0c\u9ed8\u8ba4\u4e3aWARN\u3002INFO\u4e3a \u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cWARN\u4e3a \u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cCRITICAL\u4e3a \u7535\u8bdd+\u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \u793a\u4f8b\u5982\u4e0b export cmsAccessKeyId=<your_key_id> export cmsAccessSecret=<your_key_secret> export cmsRegion=cn-hangzhou export cmsToken=<your_cms_token> export cmsLevel=WARN 4. \u5f00\u542fCMS\u5bf9\u5931\u8d25\u4efb\u52a1\u8fdb\u884c\u544a\u8b66 \u00a7 copy\u64cd\u4f5c \u00a7 hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht/hourly_table \\ --enableCMS diff\u64cd\u4f5c \u00a7 hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht/hourly_table \\ --diff \\ --enableCMS update\u64cd\u4f5c \u00a7 hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht/hourly_table \\ --update \\ --enableCMS \u544a\u8b66\u5185\u5bb9 \u00a7 \u544a\u8b66\u5185\u5bb9\u683c\u5f0f \u00a7 { \"JobTime\": \"2021-03-09 19:45:27~2021-03-09 19:45:28\", \"SrcPath\": \"hdfs://cluster-xxx:9000//data/incoming/hourly_table\", \"DestPath\": \"oss://yang-hhht/hourly_table\", \"Counter\": { \"COUNTER_1\": XXX, \"COUNTER_2\": XXX, ... } } ... \u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668 \u00a7 \u544a\u8b66\u5185\u5bb9\u4e2d\u7684counter\u8ba1\u6570\u5668\u6709\u4ee5\u4e0b\u51e0\u79cd\uff1a copy\u64cd\u4f5c \u00a7 \u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 diff\u64cd\u4f5c \u00a7 \u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"JindoDistCp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#jindodistcpcms","text":"JindoDistCp\uff08\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u5de5\u5177\uff09\u662f\u7528\u4e8e\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u62f7\u8d1d\u6587\u4ef6\u7684\u5de5\u5177\u3002\u4f7f\u7528MapReduce\u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3amap/reduce\u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002 CMS\uff08\u4e91\u76d1\u63a7\u670d\u52a1\uff09\uff0c\u53ef\u7528\u4e8e\u6536\u96c6\u963f\u91cc\u4e91\u8d44\u6e90\u7684\u76d1\u63a7\u6307\u6807\u6216\u7528\u6237\u81ea\u5b9a\u4e49\u7684\u76d1\u63a7\u6307\u6807\uff0c\u63a2\u6d4b\u670d\u52a1\u53ef\u7528\u6027\uff0c\u4ee5\u53ca\u9488\u5bf9\u6307\u6807\u8bbe\u7f6e\u8b66\u62a5\u3002\u4f7f\u60a8\u5168\u9762\u4e86\u89e3\u963f\u91cc\u4e91\u4e0a\u8d44\u6e90\u7684\u4f7f\u7528\u60c5\u51b5\u548c\u4e1a\u52a1\u8fd0\u884c\u72b6\u51b5\uff0c\u5e76\u53ca\u65f6\u5bf9\u6545\u969c\u8d44\u6e90\u8fdb\u884c\u5904\u7406\uff0c\u4fdd\u8bc1\u4e1a\u52a1\u6b63\u5e38\u8fd0\u884c\u3002 \u60a8\u53ef\u4ee5\u6307\u5b9a\u672c\u6b21JindoDistCp\u4efb\u52a1\u7ed3\u675f\u540e\u662f\u5426\u9700\u8981\u5f00\u542fCMS\uff0c\u4ee5\u4e0a\u62a5\u4efb\u52a1\u5931\u8d25\u4fe1\u606f\uff0c\u5e76\u4f7f\u7528CMS\u63a7\u5236\u53f0\u6765\u914d\u7f6e\u544a\u8b66\u529f\u80fd\u3002\u544a\u8b66\u6761\u4ef6\u53c2\u89c1 \u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668 \u3002","title":"JindoDistCp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#1","text":"\u5177\u4f53\u8bf7\u53c2\u8003\u6587\u6863 \u300a\u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u6216\u62a5\u8b66\u8054\u7cfb\u7ec4\u300b","title":"1. \u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u548c\u62a5\u8b66\u8054\u7cfb\u7ec4"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#2-token","text":"\u70b9\u51fb\u5de6\u4fa7\u5bfc\u822a\u680f\u201c\u62a5\u8b66\u670d\u52a1\u201d->\"\u62a5\u8b66\u8054\u7cfb\u4eba\"->\u5207\u6362\u4e0a\u6d4b\u9875\u7b7e\u5230\"\u62a5\u8b66\u8054\u7cfb\u7ec4\"->\u9009\u62e9\u5bf9\u5e94\u7684\u62a5\u8b66\u8054\u7cfb\u7ec4->\u70b9\u51fb\u53f3\u4fa7\u63a5\u5165\u5916\u90e8\u62a5\u8b66","title":"2. \u83b7\u53d6\u62a5\u8b66Token"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#3","text":"\u73af\u5883\u53d8\u91cf \u8bf4\u660e cmsAccessKeyId \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94AccessKey ID cmsAccessSecret \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94Access Key Secret cmsRegion \u8bbe\u7f6eCMS\u6240\u5728Region\uff0c\u9700\u8981\u4e0e\u5176\u4ed6\u76f8\u5173\u7684\u4e91\u4e0a\u8d44\u6e90\u4e00\u81f4\uff0c\u4f8b\u5982ECS\uff0cOSS\uff0cEMR\u6240\u5bf9\u5e94\u7684Region cmsLevel \u8bbe\u7f6e\u6807\u8bc6\u544a\u8b66\u901a\u77e5\u65b9\u5f0f\uff0c\u5982\u679c\u672a\u8bbe\u7f6e\uff0c\u9ed8\u8ba4\u4e3aWARN\u3002INFO\u4e3a \u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cWARN\u4e3a \u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cCRITICAL\u4e3a \u7535\u8bdd+\u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \u793a\u4f8b\u5982\u4e0b export cmsAccessKeyId=<your_key_id> export cmsAccessSecret=<your_key_secret> export cmsRegion=cn-hangzhou export cmsToken=<your_cms_token> export cmsLevel=WARN","title":"3. \u914d\u7f6e\u73af\u5883\u53d8\u91cf"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#4-cms","text":"","title":"4. \u5f00\u542fCMS\u5bf9\u5931\u8d25\u4efb\u52a1\u8fdb\u884c\u544a\u8b66"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#copy","text":"hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht/hourly_table \\ --enableCMS","title":"copy\u64cd\u4f5c"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#diff","text":"hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht/hourly_table \\ --diff \\ --enableCMS","title":"diff\u64cd\u4f5c"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#update","text":"hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://yang-hhht/hourly_table \\ --update \\ --enableCMS","title":"update\u64cd\u4f5c"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#_2","text":"","title":"\u544a\u8b66\u5185\u5bb9"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#_3","text":"{ \"JobTime\": \"2021-03-09 19:45:27~2021-03-09 19:45:28\", \"SrcPath\": \"hdfs://cluster-xxx:9000//data/incoming/hourly_table\", \"DestPath\": \"oss://yang-hhht/hourly_table\", \"Counter\": { \"COUNTER_1\": XXX, \"COUNTER_2\": XXX, ... } } ...","title":"\u544a\u8b66\u5185\u5bb9\u683c\u5f0f"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#_4","text":"\u544a\u8b66\u5185\u5bb9\u4e2d\u7684counter\u8ba1\u6570\u5668\u6709\u4ee5\u4e0b\u51e0\u79cd\uff1a","title":"\u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#copy_1","text":"\u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570","title":"copy\u64cd\u4f5c"},{"location":"backup/oss/jindo_distcp/jindo_distcp_how_to_cms/#diff_1","text":"\u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"diff\u64cd\u4f5c"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/","text":"\u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS \u4e0a \u00a7 \u4f7f\u7528\u524d\u987b\u77e5 \u00a7 \u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\uff0c\u4e5f\u53ef \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988 1\u3001\u62f7\u8d1d\u6570\u636e\u5230 OSS \u4e0a \u00a7 \u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06 HDFS \u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS \u4e0a hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --parallelism 10 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src HDFS \u7684\u6e90\u8def\u5f84\u3002 /data --dest OSS \u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket/ --hadoopConf \u6307\u5b9a OSS Access Key ID , Access Key Secret , Endpoint * \u914d\u7f6e OSS \u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS \u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret * \u914d\u7f6e OSS \u7684 Endpoint \u4fe1\u606f: fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10 2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6 \u00a7 \u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53eaCopy\u5269\u4e0b\u672aCopy\u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a \u4f7f\u7528 --update \u547d\u4ee4 \u00a7 \u4f7f\u7528 --update \u547d\u4ee4\u65f6\uff0c\u9ed8\u8ba4\u5f00\u542f checksum \u6bd4\u8f83\uff0c\u4e5f\u53ef\u901a\u8fc7 --disableChecksum \u5173\u95ed\u3002 \u5f00\u542f\u65f6\uff0c\u6bd4\u8f83\u7684\u65b9\u5f0f\u662f\uff0c\u4ece hdfs \u4e2d\u83b7\u53d6\u7684 checksum\uff0c\u5224\u65ad\u4e0e\u4e0a\u6b21\u62f7\u8d1d\u65f6\u8bb0\u5f55\u5728 OSS \u4e2d\u7684 checksum \u662f\u5426\u76f8\u540c\u3002\u56e0\u6b64\u4ec5\u652f\u6301\u6bd4\u8f83\u901a\u8fc7 3.4.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u62f7\u8d1d\u5f97\u5230\u7684\u6587\u4ef6\uff0c\u5982\u5e0c\u671b\u589e\u91cf\u6bd4\u8f83\u8001\u7248\u672c\u62f7\u8d1d\u5f97\u5230\u7684\u6587\u4ef6\uff0c\u63a8\u8350\u5173\u95ed checksum \u6bd4\u8f83\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --update --parallelism 20 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) \u5173\u95ed\u65f6\uff0c\u4ec5\u5bf9\u6587\u4ef6\u540d\u548c\u6587\u4ef6\u5927\u5c0f\u505a\u6bd4\u8f83\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --update --disableChecksum --parallelism 20 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) --disableChecksum\uff1a\u8df3\u8fc7\u4f20\u8f93\u65f6\u5bf9 checksum \u7684\u8ba1\u7b97\u548c\u68c0\u67e5 3\u3001\u6587\u4ef6\u51b7\u5907\u4efd \u00a7 \u5982\u60a8\u60f3\u5bf9\u5199\u5165\u5230 OSS \u4e0a\u7684\u6587\u4ef6\u8fdb\u884c\u51b7\u5907\uff0c\u5982\u8f6c\u5316\u6210\u51b7\u5f52\u6863\uff08coldArchive\uff09\u3001\u5f52\u6863\uff08archive\uff09\u548c\u4f4e\u9891\uff08ia\uff09\u6587\u4ef6\uff0c\u53ef\u5229\u7528 JindoDistCp \u76f4\u63a5\u8fdb\u884c\u8be5\u6d41\u7a0b \u5199\u5165\u51b7\u5f52\u6863\u6587\u4ef6 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy coldArchive --parallelism 10 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) --policy coldArchive\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u51b7\u5f52\u6863\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e\uff0c\u51b7\u5f52\u6863\u76ee\u524d\u53ea\u5728\u90e8\u5206region\u53ef\u7528\uff0c\u5177\u4f53\u53c2\u89c1 OSS\u5b58\u50a8\u7c7b\u578b\u4ecb\u7ecd \u5199\u5165\u5f52\u6863\u6587\u4ef6 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy archive --parallelism 10 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) --policy archive\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u5f52\u6863\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e \u5199\u5165\u4f4e\u9891\u6587\u4ef6 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy ia --parallelism 10 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) --policy ia\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u4f4e\u9891\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e 4\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9 \u00a7 \u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB 5\u3001\u514d\u5bc6\u53ca\u5bc6\u94a5\u56fa\u5b9a\u5b58\u50a8 \u00a7 \u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS AccessKey/AccessSecret/EndPoint \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662fJindoDistCp\u53ef\u4ee5\u5c06\u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> 6\u3001\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f \u00a7 \u4f7f\u7528preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --preserveMeta 7\u3001\u5176\u4ed6\u529f\u80fd \u00a7 \u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 * JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e * JindoDistCp \u4f7f\u7528 CMS \u8fdb\u884c\u544a\u8b66","title":"\u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS \u4e0a"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#hdfs-oss","text":"","title":"\u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS \u4e0a"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#_1","text":"\u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\uff0c\u4e5f\u53ef \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988","title":"\u4f7f\u7528\u524d\u987b\u77e5"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#1-oss","text":"\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06 HDFS \u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS \u4e0a hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --parallelism 10 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src HDFS \u7684\u6e90\u8def\u5f84\u3002 /data --dest OSS \u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket/ --hadoopConf \u6307\u5b9a OSS Access Key ID , Access Key Secret , Endpoint * \u914d\u7f6e OSS \u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS \u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret * \u914d\u7f6e OSS \u7684 Endpoint \u4fe1\u606f: fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10","title":"1\u3001\u62f7\u8d1d\u6570\u636e\u5230 OSS \u4e0a"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#2","text":"\u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53eaCopy\u5269\u4e0b\u672aCopy\u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a","title":"2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#-update","text":"\u4f7f\u7528 --update \u547d\u4ee4\u65f6\uff0c\u9ed8\u8ba4\u5f00\u542f checksum \u6bd4\u8f83\uff0c\u4e5f\u53ef\u901a\u8fc7 --disableChecksum \u5173\u95ed\u3002 \u5f00\u542f\u65f6\uff0c\u6bd4\u8f83\u7684\u65b9\u5f0f\u662f\uff0c\u4ece hdfs \u4e2d\u83b7\u53d6\u7684 checksum\uff0c\u5224\u65ad\u4e0e\u4e0a\u6b21\u62f7\u8d1d\u65f6\u8bb0\u5f55\u5728 OSS \u4e2d\u7684 checksum \u662f\u5426\u76f8\u540c\u3002\u56e0\u6b64\u4ec5\u652f\u6301\u6bd4\u8f83\u901a\u8fc7 3.4.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u62f7\u8d1d\u5f97\u5230\u7684\u6587\u4ef6\uff0c\u5982\u5e0c\u671b\u589e\u91cf\u6bd4\u8f83\u8001\u7248\u672c\u62f7\u8d1d\u5f97\u5230\u7684\u6587\u4ef6\uff0c\u63a8\u8350\u5173\u95ed checksum \u6bd4\u8f83\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --update --parallelism 20 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) \u5173\u95ed\u65f6\uff0c\u4ec5\u5bf9\u6587\u4ef6\u540d\u548c\u6587\u4ef6\u5927\u5c0f\u505a\u6bd4\u8f83\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --update --disableChecksum --parallelism 20 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) --disableChecksum\uff1a\u8df3\u8fc7\u4f20\u8f93\u65f6\u5bf9 checksum \u7684\u8ba1\u7b97\u548c\u68c0\u67e5","title":"\u4f7f\u7528 --update \u547d\u4ee4"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#3","text":"\u5982\u60a8\u60f3\u5bf9\u5199\u5165\u5230 OSS \u4e0a\u7684\u6587\u4ef6\u8fdb\u884c\u51b7\u5907\uff0c\u5982\u8f6c\u5316\u6210\u51b7\u5f52\u6863\uff08coldArchive\uff09\u3001\u5f52\u6863\uff08archive\uff09\u548c\u4f4e\u9891\uff08ia\uff09\u6587\u4ef6\uff0c\u53ef\u5229\u7528 JindoDistCp \u76f4\u63a5\u8fdb\u884c\u8be5\u6d41\u7a0b","title":"3\u3001\u6587\u4ef6\u51b7\u5907\u4efd"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#_2","text":"hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy coldArchive --parallelism 10 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) --policy coldArchive\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u51b7\u5f52\u6863\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e\uff0c\u51b7\u5f52\u6863\u76ee\u524d\u53ea\u5728\u90e8\u5206region\u53ef\u7528\uff0c\u5177\u4f53\u53c2\u89c1 OSS\u5b58\u50a8\u7c7b\u578b\u4ecb\u7ecd","title":"\u5199\u5165\u51b7\u5f52\u6863\u6587\u4ef6"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#_3","text":"hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy archive --parallelism 10 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) --policy archive\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u5f52\u6863\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e","title":"\u5199\u5165\u5f52\u6863\u6587\u4ef6"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#_4","text":"hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy ia --parallelism 10 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) --policy ia\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u4f4e\u9891\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e","title":"\u5199\u5165\u4f4e\u9891\u6587\u4ef6"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#4yarn","text":"\u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 (ECS \u73af\u5883\u63a8\u8350\u4f7f\u7528\u5185\u7f51 ossEndPoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com) --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB","title":"4\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#5","text":"\u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS AccessKey/AccessSecret/EndPoint \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662fJindoDistCp\u53ef\u4ee5\u5c06\u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration>","title":"5\u3001\u514d\u5bc6\u53ca\u5bc6\u94a5\u56fa\u5b9a\u5b58\u50a8"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#6","text":"\u4f7f\u7528preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket/ --preserveMeta","title":"6\u3001\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_hdfs_to_oss/#7","text":"\u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 * JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e * JindoDistCp \u4f7f\u7528 CMS \u8fdb\u884c\u544a\u8b66","title":"7\u3001\u5176\u4ed6\u529f\u80fd"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/","text":"\u6570\u636e\u5728 OSS \u4e0d\u540c bucket \u4e4b\u95f4\u8fc1\u79fb \u00a7 \u4f7f\u7528\u524d\u987b\u77e5 \u00a7 \u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\uff0c\u4e5f\u53ef \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988 1\u3001\u62f7\u8d1d\u6570\u636e\u5230 OSS \u4e0a \u00a7 \u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06 OSS \u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS \u4e0a\u3002 \u5982\u679c\u60a8\u7684\u4e24\u4e2a\u8981\u62f7\u8d1d\u7684 bucket \u90fd\u5728\u540c\u4e00\u4e2a region \u4e0b\uff0c\u4e14\u53ef\u7528\u540c\u4e00\u4e2a ossKey \u548c ossSecret \u8fdb\u884c\u8bfb\u5199\uff0c\u60a8\u53ef\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src OSS \u7684\u6e90\u8def\u5f84\u3002 oss://srcBucket/ --dest OSS \u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket/ --hadoopConf \u6307\u5b9a OSS Access Key ID , Access Key Secret , Endpoint * \u914d\u7f6e OSS \u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS \u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret * \u914d\u7f6e OSS \u7684 Endpoint \u4fe1\u606f: fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10 \u5982\u679c\u60a8\u7684 src \u548c dest \u7684 bucket \u4e0d\u5728\u540c\u4e00\u4e2a region \u6216\u8005\u4e0d\u80fd\u4f7f\u7528\u540c\u4e00\u4e2a ossKey \u548c ossSecret \u8fdb\u884c\u8bfb\u5199\uff0c\u90a3\u4e48\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcbucket.oss-cn-xxx.aliyuncs.com/ --dest oss://destBucket.oss-cn-xxx.aliyuncs.com/ --hadoopConf fs.oss.bucket.srcbucket.accessKeyId=yourkey --hadoopConf fs.oss.bucket.srcbucket.accessKeySecret=yoursecret --hadoopConf fs.oss.bucket.srcbucket.endpoint=oss-cn-xxx.aliyuncs.com --hadoopConf fs.oss.bucket.destBucket.accessKeyId=yourkey --hadoopConf fs.oss.bucket.destBucket.accessKeySecret=yoursecret --hadoopConf fs.oss.bucket.destBucket.endpoint=oss-cn-xxx.aliyuncs.com --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --hadoopConf \u7ed9 src \u548c dest \u7684 bucket \u6307\u5b9a\u4e0d\u540c\u7684 Access Key ID , Access Key Secret , Endpoint * \u914d\u7f6e XXX bucket \u7684 AccessKeyId: --hadoopConf fs.oss.bucket.XXX.accessKeyId=yourkey * \u914d\u7f6e XXX bucket \u7684 \u7684 AccessKeySecret: --hadoopConf fs.oss.bucket.XXX.accessKeySecret=yoursecret * \u914d\u7f6e XXX bucket \u7684 Endpoint \u4fe1\u606f: fs.oss.bucket.XXX.endpoint=oss-cn-xxx.aliyuncs.com 2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6 \u00a7 \u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53eaCopy\u5269\u4e0b\u672aCopy\u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a \u4f7f\u7528 --update \u547d\u4ee4\uff0c\u83b7\u5f97\u589e\u91cf\u7684\u6587\u4ef6\u5217\u8868 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --update --parallelism 20 3\u3001\u6587\u4ef6\u51b7\u5907\u4efd \u00a7 \u5982\u60a8\u60f3\u5bf9\u5199\u5165\u5230 OSS \u4e0a\u7684\u6587\u4ef6\u8fdb\u884c\u51b7\u5907\uff0c\u5982\u8f6c\u5316\u6210\u51b7\u5f52\u6863\uff08coldArchive\uff09\u3001\u5f52\u6863\uff08archive\uff09\u548c\u4f4e\u9891\uff08ia\uff09\u6587\u4ef6\uff0c\u53ef\u5229\u7528 JindoDistCp \u76f4\u63a5\u8fdb\u884c\u8be5\u6d41\u7a0b \u5199\u5165\u51b7\u5f52\u6863\u6587\u4ef6 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy coldArchive --parallelism 10 --policy coldArchive\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u51b7\u5f52\u6863\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e\uff0c\u51b7\u5f52\u6863\u76ee\u524d\u53ea\u5728\u90e8\u5206region\u53ef\u7528\uff0c\u5177\u4f53\u53c2\u89c1 OSS\u5b58\u50a8\u7c7b\u578b\u4ecb\u7ecd \u5199\u5165\u5f52\u6863\u6587\u4ef6 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy archive --parallelism 10 --policy archive\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u5f52\u6863\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e \u5199\u5165\u4f4e\u9891\u6587\u4ef6 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy ia --parallelism 10 --policy ia\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u4f4e\u9891\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e 4\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9 \u00a7 \u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB 5\u3001\u514d\u5bc6\u53ca\u5bc6\u94a5\u56fa\u5b9a\u5b58\u50a8 \u00a7 \u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS AccessKey/AccessSecret/EndPoint \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662fJindoDistCp\u53ef\u4ee5\u5c06 \u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> 6\u3001\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f \u00a7 \u4f7f\u7528preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --preserveMeta 7\u3001\u5176\u4ed6\u529f\u80fd \u00a7 \u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 * JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e","title":"\u6570\u636e\u5728 OSS \u4e0d\u540c bucket \u4e4b\u95f4\u8fc1\u79fb"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#oss-bucket","text":"","title":"\u6570\u636e\u5728 OSS \u4e0d\u540c bucket \u4e4b\u95f4\u8fc1\u79fb"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#_1","text":"\u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\uff0c\u4e5f\u53ef \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988","title":"\u4f7f\u7528\u524d\u987b\u77e5"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#1-oss","text":"\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06 OSS \u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS \u4e0a\u3002 \u5982\u679c\u60a8\u7684\u4e24\u4e2a\u8981\u62f7\u8d1d\u7684 bucket \u90fd\u5728\u540c\u4e00\u4e2a region \u4e0b\uff0c\u4e14\u53ef\u7528\u540c\u4e00\u4e2a ossKey \u548c ossSecret \u8fdb\u884c\u8bfb\u5199\uff0c\u60a8\u53ef\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src OSS \u7684\u6e90\u8def\u5f84\u3002 oss://srcBucket/ --dest OSS \u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket/ --hadoopConf \u6307\u5b9a OSS Access Key ID , Access Key Secret , Endpoint * \u914d\u7f6e OSS \u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS \u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret * \u914d\u7f6e OSS \u7684 Endpoint \u4fe1\u606f: fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10 \u5982\u679c\u60a8\u7684 src \u548c dest \u7684 bucket \u4e0d\u5728\u540c\u4e00\u4e2a region \u6216\u8005\u4e0d\u80fd\u4f7f\u7528\u540c\u4e00\u4e2a ossKey \u548c ossSecret \u8fdb\u884c\u8bfb\u5199\uff0c\u90a3\u4e48\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcbucket.oss-cn-xxx.aliyuncs.com/ --dest oss://destBucket.oss-cn-xxx.aliyuncs.com/ --hadoopConf fs.oss.bucket.srcbucket.accessKeyId=yourkey --hadoopConf fs.oss.bucket.srcbucket.accessKeySecret=yoursecret --hadoopConf fs.oss.bucket.srcbucket.endpoint=oss-cn-xxx.aliyuncs.com --hadoopConf fs.oss.bucket.destBucket.accessKeyId=yourkey --hadoopConf fs.oss.bucket.destBucket.accessKeySecret=yoursecret --hadoopConf fs.oss.bucket.destBucket.endpoint=oss-cn-xxx.aliyuncs.com --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --hadoopConf \u7ed9 src \u548c dest \u7684 bucket \u6307\u5b9a\u4e0d\u540c\u7684 Access Key ID , Access Key Secret , Endpoint * \u914d\u7f6e XXX bucket \u7684 AccessKeyId: --hadoopConf fs.oss.bucket.XXX.accessKeyId=yourkey * \u914d\u7f6e XXX bucket \u7684 \u7684 AccessKeySecret: --hadoopConf fs.oss.bucket.XXX.accessKeySecret=yoursecret * \u914d\u7f6e XXX bucket \u7684 Endpoint \u4fe1\u606f: fs.oss.bucket.XXX.endpoint=oss-cn-xxx.aliyuncs.com","title":"1\u3001\u62f7\u8d1d\u6570\u636e\u5230 OSS \u4e0a"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#2","text":"\u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53eaCopy\u5269\u4e0b\u672aCopy\u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a","title":"2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#-update","text":"hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --update --parallelism 20","title":"\u4f7f\u7528 --update \u547d\u4ee4\uff0c\u83b7\u5f97\u589e\u91cf\u7684\u6587\u4ef6\u5217\u8868"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#3","text":"\u5982\u60a8\u60f3\u5bf9\u5199\u5165\u5230 OSS \u4e0a\u7684\u6587\u4ef6\u8fdb\u884c\u51b7\u5907\uff0c\u5982\u8f6c\u5316\u6210\u51b7\u5f52\u6863\uff08coldArchive\uff09\u3001\u5f52\u6863\uff08archive\uff09\u548c\u4f4e\u9891\uff08ia\uff09\u6587\u4ef6\uff0c\u53ef\u5229\u7528 JindoDistCp \u76f4\u63a5\u8fdb\u884c\u8be5\u6d41\u7a0b","title":"3\u3001\u6587\u4ef6\u51b7\u5907\u4efd"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#_2","text":"hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy coldArchive --parallelism 10 --policy coldArchive\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u51b7\u5f52\u6863\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e\uff0c\u51b7\u5f52\u6863\u76ee\u524d\u53ea\u5728\u90e8\u5206region\u53ef\u7528\uff0c\u5177\u4f53\u53c2\u89c1 OSS\u5b58\u50a8\u7c7b\u578b\u4ecb\u7ecd","title":"\u5199\u5165\u51b7\u5f52\u6863\u6587\u4ef6"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#_3","text":"hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy archive --parallelism 10 --policy archive\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u5f52\u6863\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e","title":"\u5199\u5165\u5f52\u6863\u6587\u4ef6"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#_4","text":"hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --policy ia --parallelism 10 --policy ia\uff1a\u8868\u793a\u5199\u5165\u5230 OSS \u6587\u4ef6\u4ee5\u4f4e\u9891\u6587\u4ef6\u5f62\u5f0f\u5b58\u653e","title":"\u5199\u5165\u4f4e\u9891\u6587\u4ef6"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#4yarn","text":"\u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf fs.oss.endpoint=oss-cn-xxx.aliyuncs.com --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB","title":"4\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#5","text":"\u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS AccessKey/AccessSecret/EndPoint \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662fJindoDistCp\u53ef\u4ee5\u5c06 \u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration>","title":"5\u3001\u514d\u5bc6\u53ca\u5bc6\u94a5\u56fa\u5b9a\u5b58\u50a8"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#6","text":"\u4f7f\u7528preserveMeta\u53c2\u6570, \u4f7f\u5f97\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src oss://srcBucket/ --dest oss://destBucket/ --preserveMeta","title":"6\u3001\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f"},{"location":"backup/oss/jindo_distcp/jindo_distcp_on_oss_to_oss/#7","text":"\u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 * JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e","title":"7\u3001\u5176\u4ed6\u529f\u80fd"},{"location":"backup/oss/jindo_distcp/jindo_distcp_overview/","text":"JindoDistCp \u6570\u636e\u8fc1\u79fb \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 JDK 1.8\u53ca\u4ee5\u4e0a Hadoop 2.3+\u7248\u672c\uff0c\u8bf7\u4e0b\u8f7d\u6700\u65b0\u7248\u7684 jindo-distcp-tool-x.x.x.jar \uff0c\u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz \u5185\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 tools/ \u76ee\u5f55\u4e0b\u627e\u5230\uff0c \u4e0b\u8f7d\u9875\u9762 \u3002 \u4ec0\u4e48\u662f JindoDistCp \u00a7 \u968f\u7740\u963f\u91cc\u4e91 JindoSDK \u7684\u5168\u9762\u653e\u5f00\u4f7f\u7528\uff0c\u57fa\u4e8e JindoSDK \u7684\u963f\u91cc\u4e91\u6570\u636e\u8fc1\u79fb\u5229\u5668 JindoDistCp \u73b0\u5728\u4e5f\u5168\u9762\u9762\u5411\u7528\u6237\u5f00\u653e\u4f7f\u7528\u3002JindoDistCp \u662f\u963f\u91cc\u4e91\u6570\u636e\u6e56\u5b58\u50a8\u56e2\u961f\u5f00\u53d1\u7684\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u7684\u5de5\u5177\u3002\u5176\u4f7f\u7528 MapReduce \u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3a map/reduce \u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002\u76ee\u524d\u5168\u91cf\u652f\u6301 HDFS/OSS/S3 \u4e4b\u95f4\u7684\u7684\u6570\u636e\u62f7\u8d1d\u573a\u666f\uff0c\u63d0\u4f9b\u591a\u79cd\u4e2a\u6027\u5316\u62f7\u8d1d\u53c2\u6570\u548c\u591a\u79cd\u62f7\u8d1d\u7b56\u7565\u3002\u91cd\u70b9\u4f18\u5316 HDFS \u5230 OSS \u7684\u6570\u636e\u62f7\u8d1d\uff0c\u901a\u8fc7\u5b9a\u5236\u5316 CopyCommitter\uff0c\u5b9e\u73b0 No-Rename \u62f7\u8d1d\uff0c\u5e76\u4fdd\u8bc1\u6570\u636e\u62f7\u8d1d\u843d\u5730\u7684\u4e00\u81f4\u6027\u3002\u529f\u80fd\u5168\u91cf\u5bf9\u9f50 S3 DistCp \u548c HDFS DistCp\uff0c\u6027\u80fd\u8f83 HDFS DistCp \u6709\u8f83\u5927\u63d0\u5347\uff0c\u76ee\u6807\u63d0\u4f9b\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177\u3002 JindoDistCp \u4f7f\u7528\u573a\u666f \u00a7 \u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS \u4e0a \u6570\u636e\u5728 OSS \u4e0d\u540c bucket \u4e4b\u95f4\u8fc1\u79fb JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u00a7 \u6211\u4eec\u603b\u7ed3\u4e86\u4e00\u4e9b\u5728\u4e0d\u540c\u73af\u5883\u4f7f\u7528 JindoDistCp \u7528\u6237\u53ef\u80fd\u4f1a\u9047\u5230\u7684\u4e00\u4e9b\u95ee\u9898\uff0c\u60a8\u53ef\u4ee5\u53c2\u7167\u4e0b\u6587\u8fdb\u884c\u5e38\u89c1\u95ee\u9898\u6392\u67e5\u548c\u89e3\u51b3 * JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 JindoDistCp \u7cfb\u5217\u6587\u7ae0 \u00a7 \u300a\u518d\u51fa\u738b\u724c\uff1a\u963f\u91cc\u4e91 JindoDistCp \u5168\u9762\u5f00\u653e\u4f7f\u7528\uff0c\u6210\u4e3a\u963f\u91cc\u4e91\u6570\u636e\u8fc1\u79fb\u5229\u5668\u300b JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e (\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177) JindoDistCp API \u4f7f\u7528\u8bf4\u660e \u00a7 * JindoDistCp API \u4f7f\u7528\u8bf4\u660e","title":"JindoDistCp \u6570\u636e\u8fc1\u79fb"},{"location":"backup/oss/jindo_distcp/jindo_distcp_overview/#jindodistcp","text":"","title":"JindoDistCp \u6570\u636e\u8fc1\u79fb"},{"location":"backup/oss/jindo_distcp/jindo_distcp_overview/#_1","text":"JDK 1.8\u53ca\u4ee5\u4e0a Hadoop 2.3+\u7248\u672c\uff0c\u8bf7\u4e0b\u8f7d\u6700\u65b0\u7248\u7684 jindo-distcp-tool-x.x.x.jar \uff0c\u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz \u5185\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 tools/ \u76ee\u5f55\u4e0b\u627e\u5230\uff0c \u4e0b\u8f7d\u9875\u9762 \u3002","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/oss/jindo_distcp/jindo_distcp_overview/#jindodistcp_1","text":"\u968f\u7740\u963f\u91cc\u4e91 JindoSDK \u7684\u5168\u9762\u653e\u5f00\u4f7f\u7528\uff0c\u57fa\u4e8e JindoSDK \u7684\u963f\u91cc\u4e91\u6570\u636e\u8fc1\u79fb\u5229\u5668 JindoDistCp \u73b0\u5728\u4e5f\u5168\u9762\u9762\u5411\u7528\u6237\u5f00\u653e\u4f7f\u7528\u3002JindoDistCp \u662f\u963f\u91cc\u4e91\u6570\u636e\u6e56\u5b58\u50a8\u56e2\u961f\u5f00\u53d1\u7684\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u7684\u5de5\u5177\u3002\u5176\u4f7f\u7528 MapReduce \u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3a map/reduce \u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002\u76ee\u524d\u5168\u91cf\u652f\u6301 HDFS/OSS/S3 \u4e4b\u95f4\u7684\u7684\u6570\u636e\u62f7\u8d1d\u573a\u666f\uff0c\u63d0\u4f9b\u591a\u79cd\u4e2a\u6027\u5316\u62f7\u8d1d\u53c2\u6570\u548c\u591a\u79cd\u62f7\u8d1d\u7b56\u7565\u3002\u91cd\u70b9\u4f18\u5316 HDFS \u5230 OSS \u7684\u6570\u636e\u62f7\u8d1d\uff0c\u901a\u8fc7\u5b9a\u5236\u5316 CopyCommitter\uff0c\u5b9e\u73b0 No-Rename \u62f7\u8d1d\uff0c\u5e76\u4fdd\u8bc1\u6570\u636e\u62f7\u8d1d\u843d\u5730\u7684\u4e00\u81f4\u6027\u3002\u529f\u80fd\u5168\u91cf\u5bf9\u9f50 S3 DistCp \u548c HDFS DistCp\uff0c\u6027\u80fd\u8f83 HDFS DistCp \u6709\u8f83\u5927\u63d0\u5347\uff0c\u76ee\u6807\u63d0\u4f9b\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177\u3002","title":"\u4ec0\u4e48\u662f JindoDistCp"},{"location":"backup/oss/jindo_distcp/jindo_distcp_overview/#jindodistcp_2","text":"\u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS \u4e0a \u6570\u636e\u5728 OSS \u4e0d\u540c bucket \u4e4b\u95f4\u8fc1\u79fb","title":"JindoDistCp \u4f7f\u7528\u573a\u666f"},{"location":"backup/oss/jindo_distcp/jindo_distcp_overview/#jindodistcp_3","text":"\u6211\u4eec\u603b\u7ed3\u4e86\u4e00\u4e9b\u5728\u4e0d\u540c\u73af\u5883\u4f7f\u7528 JindoDistCp \u7528\u6237\u53ef\u80fd\u4f1a\u9047\u5230\u7684\u4e00\u4e9b\u95ee\u9898\uff0c\u60a8\u53ef\u4ee5\u53c2\u7167\u4e0b\u6587\u8fdb\u884c\u5e38\u89c1\u95ee\u9898\u6392\u67e5\u548c\u89e3\u51b3 * JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357","title":"JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357"},{"location":"backup/oss/jindo_distcp/jindo_distcp_overview/#jindodistcp_4","text":"\u300a\u518d\u51fa\u738b\u724c\uff1a\u963f\u91cc\u4e91 JindoDistCp \u5168\u9762\u5f00\u653e\u4f7f\u7528\uff0c\u6210\u4e3a\u963f\u91cc\u4e91\u6570\u636e\u8fc1\u79fb\u5229\u5668\u300b JindoDistCp \u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u7684\u4f7f\u7528\u8bf4\u660e (\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177)","title":"JindoDistCp \u7cfb\u5217\u6587\u7ae0"},{"location":"backup/oss/jindo_distcp/jindo_distcp_overview/#jindodistcp-api","text":"* JindoDistCp API \u4f7f\u7528\u8bf4\u660e","title":"JindoDistCp API \u4f7f\u7528\u8bf4\u660e"},{"location":"backup/oss/jindo_fuse/jindo_fuse_benchmark/","text":"Jindo Fuse \u6027\u80fd\u6d4b\u8bd5 \u00a7 \u6d4b\u8bd5\u73af\u5883 \u00a7 CPU: 80\u6838 | \u5185\u5b58\uff1a 128G \u7f51\u7edc\u89c4\u683c\uff1a25Gbps ECS\u89c4\u683c\uff1a ecs.hfc6.20xlarge \u6d4b\u8bd5\u6570\u636e\u96c6 \u00a7 \u6570\u636e\u96c6A: \u5927\u6587\u4ef6\u96c6\u5408\uff0c\u6bcf\u4e2a\u6587\u4ef6140MB\uff0c\u603b\u5171140GB \u6570\u636e\u96c6B: \u5c0f\u6587\u4ef6\u96c6\u5408\uff0c\u6bcf\u4e2a\u6587\u4ef623KB\uff0c\u603b\u51711w\u4e2a \u6d4b\u8bd5\u7ed3\u679c \u00a7 \u5355\u7ebf\u7a0b\u6d4b\u8bd5 \u6570\u636e\u96c6 jindo-fuse ossfs OSS2/Python \u6570\u636e\u96c6A 191 MB/s 72 MB/s 17 MB/s \u6570\u636e\u96c6B 1.16 MB/s 1.25 MB/s 2.1 MB/s \u591a\u7ebf\u7a0b\u6d4b\u8bd5\uff0810\u4e2a\u7ebf\u7a0b\uff09 \u6570\u636e\u96c6 jindo-fuse ossfs OSS2/Python \u6570\u636e\u96c6A 1693MB/s 686 MB/s 162 MB/s \u6570\u636e\u96c6B 18 MB/s 10 MB/s 9 MB/s \u7ed3\u679c\u5206\u6790 \u00a7 jindo-fuse \u5bf9\u4e8e\u5927\u6587\u4ef6 \u6709\u9884\u8bfb\uff0c\u56e0\u6b64\u6bd4\u8f83\u5feb\u3002 OSS2 \u5bf9\u5c0f\u6587\u4ef6\u6bd4\u8f83\u5feb\uff0c\u5c0f\u6587\u4ef6 fuse \u65e0\u6cd5\u505a\u9884\u8bfb\uff0cOSS2 \u76f4\u63a5 http \u8bf7\u6c42\u6ca1\u6709\u989d\u5916\u5f00\u9500\uff0c\u6240\u4ee5\u6bd4\u8f83\u5feb\u3002 \u5c0f\u6587\u4ef6\u7ecf\u8fc7 fuse \u6709\u989d\u5916\u5f00\u9500\uff0c \u56e0\u6b64 jindo-fuse\u3001ossfs \u901f\u5ea6\u6bd4 OSS2 \u5dee\u3002 OSS2 \u5bf9\u5355\u8fdb\u7a0b\u591a\u7ebf\u7a0b\u6548\u679c\u4e0d\u597d\u3002 \u6211\u4eec\u53ef\u4ee5\u63a8\u6d4b OSS2 \u591a\u8fdb\u7a0b\u6210\u7ee9 2.1MB/s *10 = 21MB/s\u3002 ossfs \u5bf9\u5c0f\u6587\u4ef6\u56e0\u4e3a\u6709\u989d\u5916\u4e00\u6b21get_attr\u64cd\u4f5c\uff0c\u5373\u4f7f\u8c03\u5927 max_stat_cache_size=500000 \u6548\u679c\u4e5f\u4e0d\u7406\u60f3\uff0c\u76f8\u5f53\u4e8e get_attr + get_object \u4e24\u6b21 rpc\uff0c\u56e0\u6b64\u901f\u5ea6\u662f\u53e6\u5916\u4e8c\u8005\u7684\u4e00\u534a\uff0c20MB/s /2 = 10MB/s \u7b26\u5408\u9884\u671f\u3002","title":"Jindo Fuse \u6027\u80fd\u6d4b\u8bd5"},{"location":"backup/oss/jindo_fuse/jindo_fuse_benchmark/#jindo-fuse","text":"","title":"Jindo Fuse \u6027\u80fd\u6d4b\u8bd5"},{"location":"backup/oss/jindo_fuse/jindo_fuse_benchmark/#_1","text":"CPU: 80\u6838 | \u5185\u5b58\uff1a 128G \u7f51\u7edc\u89c4\u683c\uff1a25Gbps ECS\u89c4\u683c\uff1a ecs.hfc6.20xlarge","title":"\u6d4b\u8bd5\u73af\u5883"},{"location":"backup/oss/jindo_fuse/jindo_fuse_benchmark/#_2","text":"\u6570\u636e\u96c6A: \u5927\u6587\u4ef6\u96c6\u5408\uff0c\u6bcf\u4e2a\u6587\u4ef6140MB\uff0c\u603b\u5171140GB \u6570\u636e\u96c6B: \u5c0f\u6587\u4ef6\u96c6\u5408\uff0c\u6bcf\u4e2a\u6587\u4ef623KB\uff0c\u603b\u51711w\u4e2a","title":"\u6d4b\u8bd5\u6570\u636e\u96c6"},{"location":"backup/oss/jindo_fuse/jindo_fuse_benchmark/#_3","text":"\u5355\u7ebf\u7a0b\u6d4b\u8bd5 \u6570\u636e\u96c6 jindo-fuse ossfs OSS2/Python \u6570\u636e\u96c6A 191 MB/s 72 MB/s 17 MB/s \u6570\u636e\u96c6B 1.16 MB/s 1.25 MB/s 2.1 MB/s \u591a\u7ebf\u7a0b\u6d4b\u8bd5\uff0810\u4e2a\u7ebf\u7a0b\uff09 \u6570\u636e\u96c6 jindo-fuse ossfs OSS2/Python \u6570\u636e\u96c6A 1693MB/s 686 MB/s 162 MB/s \u6570\u636e\u96c6B 18 MB/s 10 MB/s 9 MB/s","title":"\u6d4b\u8bd5\u7ed3\u679c"},{"location":"backup/oss/jindo_fuse/jindo_fuse_benchmark/#_4","text":"jindo-fuse \u5bf9\u4e8e\u5927\u6587\u4ef6 \u6709\u9884\u8bfb\uff0c\u56e0\u6b64\u6bd4\u8f83\u5feb\u3002 OSS2 \u5bf9\u5c0f\u6587\u4ef6\u6bd4\u8f83\u5feb\uff0c\u5c0f\u6587\u4ef6 fuse \u65e0\u6cd5\u505a\u9884\u8bfb\uff0cOSS2 \u76f4\u63a5 http \u8bf7\u6c42\u6ca1\u6709\u989d\u5916\u5f00\u9500\uff0c\u6240\u4ee5\u6bd4\u8f83\u5feb\u3002 \u5c0f\u6587\u4ef6\u7ecf\u8fc7 fuse \u6709\u989d\u5916\u5f00\u9500\uff0c \u56e0\u6b64 jindo-fuse\u3001ossfs \u901f\u5ea6\u6bd4 OSS2 \u5dee\u3002 OSS2 \u5bf9\u5355\u8fdb\u7a0b\u591a\u7ebf\u7a0b\u6548\u679c\u4e0d\u597d\u3002 \u6211\u4eec\u53ef\u4ee5\u63a8\u6d4b OSS2 \u591a\u8fdb\u7a0b\u6210\u7ee9 2.1MB/s *10 = 21MB/s\u3002 ossfs \u5bf9\u5c0f\u6587\u4ef6\u56e0\u4e3a\u6709\u989d\u5916\u4e00\u6b21get_attr\u64cd\u4f5c\uff0c\u5373\u4f7f\u8c03\u5927 max_stat_cache_size=500000 \u6548\u679c\u4e5f\u4e0d\u7406\u60f3\uff0c\u76f8\u5f53\u4e8e get_attr + get_object \u4e24\u6b21 rpc\uff0c\u56e0\u6b64\u901f\u5ea6\u662f\u53e6\u5916\u4e8c\u8005\u7684\u4e00\u534a\uff0c20MB/s /2 = 10MB/s \u7b26\u5408\u9884\u671f\u3002","title":"\u7ed3\u679c\u5206\u6790"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/","text":"\u4f7f\u7528 JindoFuse \u8bbf\u95ee OSS \u00a7 OSS \u901a\u8fc7 JindoFuse \u63d0\u4f9b POSIX \u652f\u6301\u3002JindoFuse \u53ef\u4ee5\u628a OSS \u4e0a\u7684\u6587\u4ef6\u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c OSS \u4e0a\u7684\u6587\u4ef6\u3002 \u57fa\u672c\u4f7f\u7528 \u00a7 \u73af\u5883\u51c6\u5907 \u00a7 \u652f\u6301 libfuse 3.7 \u4ee5\u4e0a\u7248\u672c\uff0c\u63a8\u8350\u4f7f\u7528\u6700\u65b0\u7684 libfuse \u7248\u672c\u4ee5\u83b7\u5f97\u6700\u4f73\u6027\u80fd\u3002 # build fuse required meson & ninja, for debian: apt install -y pkg-config meson ninja-build sudo yum install -y meson ninja-build # compile fuse required newer g++ (only CentOS) g++ -v # compile & install libfuse wget https://github.com/libfuse/libfuse/releases/download/fuse-3.11.0/fuse-3.11.0.tar.xz xz -d fuse-3.11.0.tar.xz tar xf fuse-3.11.0.tar cd fuse-3.11.0/ mkdir build; cd build meson .. sudo ninja install \u914d\u7f6e\u5ba2\u6237\u7aef \u00a7 \u914d\u7f6e\u76ee\u5f55 \u00a7 \u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf \u914d\u7f6e\u6587\u4ef6 \u00a7 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95eeOSS\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> \u514d\u5bc6\u8bbf\u95ee \u00a7 \u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7\u89d2\u8272\u6388\u6743\u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE \u6302\u8f7d JindoFuse \u00a7 \u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002 \u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 mkdir -p <mount_point> \u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a \u00a7 jindo-fuse <mount_point> -ouri=[<oss_path>] -ouri\u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684oss\u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3aBucket\u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002 \u8bbf\u95ee JindoFuse \u00a7 \u5982\u679c\u5c06 OSS \u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1 ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/ \u5378\u8f7d JindoFuse \u00a7 \u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point> \u81ea\u52a8\u5378\u8f7d JindoFuse \u00a7 \u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u7279\u6027\u652f\u6301 \u00a7 \u76ee\u524d JindoFuse \u5df2\u7ecf\u652f\u6301\u4ee5\u4e0b POSIX API\uff1a \u7279\u6027 \u8bf4\u660e getattr() \u67e5\u8be2\u6587\u4ef6\u5c5e\u6027\uff0c\u7c7b\u4f3c ls mkdir() \u521b\u5efa\u76ee\u5f55\uff0c\u7c7b\u4f3c mkdir rmdir() \u5220\u9664\u76ee\u5f55\uff0c\u7c7b\u4f3c rm -rf unlink() \u5220\u9664\u6587\u4ef6\uff0c\u7c7b\u4f3c unlink rename() \u91cd\u547d\u540d\uff0c\u7c7b\u4f3c mv read() \u987a\u5e8f\u8bfb\u53d6 pread() \u968f\u673a\u8bfb write() \u987a\u5e8f\u5199 flush() \u5237\u65b0\u5185\u5b58\u5230\u5185\u6838\u7f13\u51b2\u533a\uff08\u4ec5\u652f\u6301\u4ee5\u6700\u8ffd\u52a0\u5199\u65b9\u5f0f\u6253\u5f00\u7684\u6587\u4ef6\uff09 fsync() \u5237\u65b0\u5185\u5b58\u5230\u78c1\u76d8\uff08\u4ec5\u652f\u6301\u4ee5\u8ffd\u52a0\u5199\u65b9\u5f0f\u6253\u5f00\u7684\u6587\u4ef6\uff09 release() \u5173\u95ed\u6587\u4ef6 readdir() \u8bfb\u53d6\u76ee\u5f55 create() \u521b\u5efa\u6587\u4ef6 open() O_APPEND \u901a\u8fc7\u8ffd\u52a0\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 open() O_TRUNC \u901a\u8fc7\u8986\u76d6\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 access() \u67e5\u8be2\u6587\u4ef6\u6743\u9650 \u9ad8\u9636\u4f7f\u7528 \u00a7 \u6302\u8f7d\u9009\u9879 \u00a7 \u53c2\u6570\u540d\u79f0 \u5fc5\u9009 \u53c2\u6570\u8bf4\u660e \u4f7f\u7528\u8303\u4f8b uri \u2713 \u914d\u7f6e\u9700\u8981\u6620\u5c04\u7684 oss \u8def\u5f84\u3002\u8def\u5f84\u53ef\u4ee5\u662f\u6839\u76ee\u5f55\uff0c\u4e5f\u53ef\u4ee5\u662f\u5b50\u76ee\u5f55\u3002\u4f8b\u5982\uff1aoss://bucket/ \u6216 oss://bucket/subdir\u3002 -ouri=oss://bucket/ f \u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u9ed8\u8ba4\u4f7f\u7528\u5b88\u62a4\u8fdb\u7a0b\u65b9\u5f0f\u540e\u53f0\u542f\u52a8\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -f d \u4f7f\u7528 Debug \u6a21\u5f0f\uff0c\u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -d auto_unmount fuse\u8fdb\u7a0b\u9000\u51fa\u540e\u81ea\u52a8umount\u6302\u8f7d\u8282\u70b9\u3002 -oauto_unmount ro \u53ea\u8bfb\u6302\u8f7d\uff0c\u542f\u7528\u53c2\u6570\u540e\u4e0d\u5141\u8bb8\u5199\u64cd\u4f5c\u3002 -oro direct_io \u5f00\u542f\u540e\uff0c\u8bfb\u5199\u6587\u4ef6\u53ef\u4ee5\u7ed5\u8fc7page cache\u3002 -odirect_io kernel_cache \u5f00\u542f\u540e\uff0c\u5229\u7528\u5185\u6838\u7f13\u5b58\u4f18\u5316\u8bfb\u6027\u80fd\u3002 -okernel_cache auto_cache \u9ed8\u8ba4\u5f00\u542f\uff0c\u4e0ekernel_cache \u4e8c\u9009\u4e00\uff0c\u4e0ekernel_cache\u4e0d\u540c\u7684\u662f\uff0c\u5982\u679c\u6587\u4ef6\u5927\u5c0f\u6216\u4fee\u6539\u65f6\u95f4\u53d1\u751f\u53d8\u5316\uff0c\u7f13\u5b58\u5c31\u4f1a\u5931\u6548\u3002 entry_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u76ee\u5f55\u6761\u76ee\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oentry_timeout=60 attr_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oattr_timeout=60 negative_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u5931\u8d25\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -onegative_timeout=0 jindo_entry_size \u9ed8\u8ba4\u503c\uff0c5000\u3002\u76ee\u5f55\u6761\u76ee\u7f13\u5b58\u6570\u91cf\uff0c\u7528\u4e8e\u4f18\u5316readdir\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -ojindo_entry_size=5000 jindo_attr_size \u9ed8\u8ba4\u503c\uff0c50000\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u6570\u91cf\uff0c\u7528\u4e8e\u4f18\u5316getattr\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -ojindo_attr_sizet=50000 max_idle_threads \u9ed8\u8ba4\u503c\uff0c10\u3002\u6700\u5927\u7a7a\u95f2\u7ebf\u7a0b\u6570\u3002 -omax_idle_threads=10 metrics_port \u9ed8\u8ba4\u503c\uff0c9090\u3002\u5f00\u542fhttp\u7aef\u53e3\uff0c\u8f93\u51fametrics\uff0c\u5982http://localhost:9090/brpc_metrics -ometrics_port=9090 enable_pread \u5f00\u542f\u540e\uff0c\u4f7f\u7528pread\u63a5\u53e3\u8bfb\u53d6\u6587\u4ef6\u3002 -oenable_pread \u914d\u7f6e\u9009\u9879 \u00a7 \u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u914d\u7f6e\u8282 \u8bf4\u660e logger.dir /tmp/fuse-log common \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false common \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false common \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\u3002 \u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002 logger.verbose 0 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false common \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint jindosdk \u8bbf\u95ee OSS \u7684\u5730\u5740\uff0c\u5982oss-cn-xxx.aliyuncs.com fs.oss.accessKeyId jindosdk \u8bbf\u95ee OSS \u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret jindosdk \u8bbf\u95ee OSS \u9700\u8981\u7684 accessKeySecret \u66f4\u591a jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<oss_path>] -ofs.oss.endpoint=[<your_endpoint>] -ofs.oss.accessKeyId=[<your_key_id>] -ofs.oss.accessKeySecret=[<your_key_secret>] \u5e38\u89c1\u95ee\u9898 \u00a7 Input/Output error \u00a7 \u4e0d\u50cf\u4f7f\u7528 JindoSDK \u8c03\u7528 API \u53ef\u4ee5\u83b7\u53d6\u66f4\u4e3a\u5177\u4f53\u7684 ErrorMsg\uff0cJindoFuse \u53ea\u80fd\u663e\u793a\u64cd\u4f5c\u7cfb\u7edf\u9884\u8bbe\u7684\u9519\u8bef\u4fe1\u606f\uff0c\u6bd4\u5982\u4ee5\u4e0b\u9519\u8bef\u5c31\u975e\u5e38\u5e38\u89c1\uff1a $ ls /mnt/oss/ ls: /mnt/oss/: Input/output error \u5982\u679c\u9700\u8981\u5b9a\u4f4d\u5177\u4f53\u7684\u9519\u8bef\u539f\u56e0\uff0c\u53ef\u4ee5\u6839\u636e JindoSDK \u914d\u7f6e\u4e2d\u7684 logger.dir\uff0c\u5728\u6307\u5b9a\u8def\u5f84\u4e0b\u7684 jindosdk.log \u6587\u4ef6\u4e2d\uff0c\u5bfb\u627e\u5177\u4f53\u7684\u9519\u8bef\u3002 \u4e0b\u9762\u5c55\u793a\u7684\u8fd9\u4e2a\u9519\u8bef\u5c31\u6765\u81ea jindosdk.log \uff0c\u6839\u636e\u62a5\u9519\u4fe1\u606f\u53ef\u89c1\uff0c\u8fd9\u4e2a\u9519\u8bef\u662f\u5e38\u89c1\u7684\u9274\u6743\u95ee\u9898\u3002 EMMDD HH:mm:ss jindofs_connectivity.cpp:13] Please check your Endpoint/Bucket/RoleArn. Failed test connectivity, operation: mkdir, errMsg: [RequestId]: 618B8183343EA53531C62B74 [HostId]: oss-cn-shanghai-internal.aliyuncs.com [ErrorMessage]: [E1010]HTTP/1.1 403 Forbidden ...","title":"\u4f7f\u7528 JindoFuse \u8bbf\u95ee OSS"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#jindofuse-oss","text":"OSS \u901a\u8fc7 JindoFuse \u63d0\u4f9b POSIX \u652f\u6301\u3002JindoFuse \u53ef\u4ee5\u628a OSS \u4e0a\u7684\u6587\u4ef6\u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c OSS \u4e0a\u7684\u6587\u4ef6\u3002","title":"\u4f7f\u7528 JindoFuse \u8bbf\u95ee OSS"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_1","text":"","title":"\u57fa\u672c\u4f7f\u7528"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_2","text":"\u652f\u6301 libfuse 3.7 \u4ee5\u4e0a\u7248\u672c\uff0c\u63a8\u8350\u4f7f\u7528\u6700\u65b0\u7684 libfuse \u7248\u672c\u4ee5\u83b7\u5f97\u6700\u4f73\u6027\u80fd\u3002 # build fuse required meson & ninja, for debian: apt install -y pkg-config meson ninja-build sudo yum install -y meson ninja-build # compile fuse required newer g++ (only CentOS) g++ -v # compile & install libfuse wget https://github.com/libfuse/libfuse/releases/download/fuse-3.11.0/fuse-3.11.0.tar.xz xz -d fuse-3.11.0.tar.xz tar xf fuse-3.11.0.tar cd fuse-3.11.0/ mkdir build; cd build meson .. sudo ninja install","title":"\u73af\u5883\u51c6\u5907"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_3","text":"","title":"\u914d\u7f6e\u5ba2\u6237\u7aef"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_4","text":"\u914d\u7f6e JINDOSDK_CONF_DIR \u6307\u5b9a\u914d\u7f6e\u6587\u4ef6\u6240\u5728\u76ee\u5f55 \u4ee5\u914d\u7f6e\u6587\u4ef6 jindosdk.cfg \u5728 /usr/lib/jindosdk-4.6.12/conf \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_CONF_DIR=/usr/lib/jindosdk-4.6.12/conf","title":"\u914d\u7f6e\u76ee\u5f55"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_5","text":"\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95eeOSS\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret>","title":"\u914d\u7f6e\u6587\u4ef6"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_6","text":"\u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7\u89d2\u8272\u6388\u6743\u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE","title":"\u514d\u5bc6\u8bbf\u95ee"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#jindofuse","text":"\u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002","title":"\u6302\u8f7d JindoFuse"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_7","text":"mkdir -p <mount_point>","title":"\u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#fuse","text":"jindo-fuse <mount_point> -ouri=[<oss_path>] -ouri\u9700\u914d\u7f6e\u4e3a\u5f85\u6620\u5c04\u7684oss\u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3aBucket\u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002 \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 \u3002","title":"\u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#jindofuse_1","text":"\u5982\u679c\u5c06 OSS \u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1 ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/","title":"\u8bbf\u95ee JindoFuse"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#jindofuse_2","text":"\u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point>","title":"\u5378\u8f7d JindoFuse"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#jindofuse_3","text":"\u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002","title":"\u81ea\u52a8\u5378\u8f7d JindoFuse"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_8","text":"\u76ee\u524d JindoFuse \u5df2\u7ecf\u652f\u6301\u4ee5\u4e0b POSIX API\uff1a \u7279\u6027 \u8bf4\u660e getattr() \u67e5\u8be2\u6587\u4ef6\u5c5e\u6027\uff0c\u7c7b\u4f3c ls mkdir() \u521b\u5efa\u76ee\u5f55\uff0c\u7c7b\u4f3c mkdir rmdir() \u5220\u9664\u76ee\u5f55\uff0c\u7c7b\u4f3c rm -rf unlink() \u5220\u9664\u6587\u4ef6\uff0c\u7c7b\u4f3c unlink rename() \u91cd\u547d\u540d\uff0c\u7c7b\u4f3c mv read() \u987a\u5e8f\u8bfb\u53d6 pread() \u968f\u673a\u8bfb write() \u987a\u5e8f\u5199 flush() \u5237\u65b0\u5185\u5b58\u5230\u5185\u6838\u7f13\u51b2\u533a\uff08\u4ec5\u652f\u6301\u4ee5\u6700\u8ffd\u52a0\u5199\u65b9\u5f0f\u6253\u5f00\u7684\u6587\u4ef6\uff09 fsync() \u5237\u65b0\u5185\u5b58\u5230\u78c1\u76d8\uff08\u4ec5\u652f\u6301\u4ee5\u8ffd\u52a0\u5199\u65b9\u5f0f\u6253\u5f00\u7684\u6587\u4ef6\uff09 release() \u5173\u95ed\u6587\u4ef6 readdir() \u8bfb\u53d6\u76ee\u5f55 create() \u521b\u5efa\u6587\u4ef6 open() O_APPEND \u901a\u8fc7\u8ffd\u52a0\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 open() O_TRUNC \u901a\u8fc7\u8986\u76d6\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 access() \u67e5\u8be2\u6587\u4ef6\u6743\u9650","title":"\u7279\u6027\u652f\u6301"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_9","text":"","title":"\u9ad8\u9636\u4f7f\u7528"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_10","text":"\u53c2\u6570\u540d\u79f0 \u5fc5\u9009 \u53c2\u6570\u8bf4\u660e \u4f7f\u7528\u8303\u4f8b uri \u2713 \u914d\u7f6e\u9700\u8981\u6620\u5c04\u7684 oss \u8def\u5f84\u3002\u8def\u5f84\u53ef\u4ee5\u662f\u6839\u76ee\u5f55\uff0c\u4e5f\u53ef\u4ee5\u662f\u5b50\u76ee\u5f55\u3002\u4f8b\u5982\uff1aoss://bucket/ \u6216 oss://bucket/subdir\u3002 -ouri=oss://bucket/ f \u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u9ed8\u8ba4\u4f7f\u7528\u5b88\u62a4\u8fdb\u7a0b\u65b9\u5f0f\u540e\u53f0\u542f\u52a8\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -f d \u4f7f\u7528 Debug \u6a21\u5f0f\uff0c\u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -d auto_unmount fuse\u8fdb\u7a0b\u9000\u51fa\u540e\u81ea\u52a8umount\u6302\u8f7d\u8282\u70b9\u3002 -oauto_unmount ro \u53ea\u8bfb\u6302\u8f7d\uff0c\u542f\u7528\u53c2\u6570\u540e\u4e0d\u5141\u8bb8\u5199\u64cd\u4f5c\u3002 -oro direct_io \u5f00\u542f\u540e\uff0c\u8bfb\u5199\u6587\u4ef6\u53ef\u4ee5\u7ed5\u8fc7page cache\u3002 -odirect_io kernel_cache \u5f00\u542f\u540e\uff0c\u5229\u7528\u5185\u6838\u7f13\u5b58\u4f18\u5316\u8bfb\u6027\u80fd\u3002 -okernel_cache auto_cache \u9ed8\u8ba4\u5f00\u542f\uff0c\u4e0ekernel_cache \u4e8c\u9009\u4e00\uff0c\u4e0ekernel_cache\u4e0d\u540c\u7684\u662f\uff0c\u5982\u679c\u6587\u4ef6\u5927\u5c0f\u6216\u4fee\u6539\u65f6\u95f4\u53d1\u751f\u53d8\u5316\uff0c\u7f13\u5b58\u5c31\u4f1a\u5931\u6548\u3002 entry_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u76ee\u5f55\u6761\u76ee\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oentry_timeout=60 attr_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oattr_timeout=60 negative_timeout \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u5931\u8d25\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -onegative_timeout=0 jindo_entry_size \u9ed8\u8ba4\u503c\uff0c5000\u3002\u76ee\u5f55\u6761\u76ee\u7f13\u5b58\u6570\u91cf\uff0c\u7528\u4e8e\u4f18\u5316readdir\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -ojindo_entry_size=5000 jindo_attr_size \u9ed8\u8ba4\u503c\uff0c50000\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u6570\u91cf\uff0c\u7528\u4e8e\u4f18\u5316getattr\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -ojindo_attr_sizet=50000 max_idle_threads \u9ed8\u8ba4\u503c\uff0c10\u3002\u6700\u5927\u7a7a\u95f2\u7ebf\u7a0b\u6570\u3002 -omax_idle_threads=10 metrics_port \u9ed8\u8ba4\u503c\uff0c9090\u3002\u5f00\u542fhttp\u7aef\u53e3\uff0c\u8f93\u51fametrics\uff0c\u5982http://localhost:9090/brpc_metrics -ometrics_port=9090 enable_pread \u5f00\u542f\u540e\uff0c\u4f7f\u7528pread\u63a5\u53e3\u8bfb\u53d6\u6587\u4ef6\u3002 -oenable_pread","title":"\u6302\u8f7d\u9009\u9879"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_11","text":"\u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u914d\u7f6e\u8282 \u8bf4\u660e logger.dir /tmp/fuse-log common \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false common \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false common \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\u3002 \u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002 logger.verbose 0 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false common \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint jindosdk \u8bbf\u95ee OSS \u7684\u5730\u5740\uff0c\u5982oss-cn-xxx.aliyuncs.com fs.oss.accessKeyId jindosdk \u8bbf\u95ee OSS \u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret jindosdk \u8bbf\u95ee OSS \u9700\u8981\u7684 accessKeySecret \u66f4\u591a jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 \u652f\u6301\u5c06 jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<oss_path>] -ofs.oss.endpoint=[<your_endpoint>] -ofs.oss.accessKeyId=[<your_key_id>] -ofs.oss.accessKeySecret=[<your_key_secret>]","title":"\u914d\u7f6e\u9009\u9879"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#_12","text":"","title":"\u5e38\u89c1\u95ee\u9898"},{"location":"backup/oss/jindo_fuse/jindo_fuse_on_oss/#inputoutput-error","text":"\u4e0d\u50cf\u4f7f\u7528 JindoSDK \u8c03\u7528 API \u53ef\u4ee5\u83b7\u53d6\u66f4\u4e3a\u5177\u4f53\u7684 ErrorMsg\uff0cJindoFuse \u53ea\u80fd\u663e\u793a\u64cd\u4f5c\u7cfb\u7edf\u9884\u8bbe\u7684\u9519\u8bef\u4fe1\u606f\uff0c\u6bd4\u5982\u4ee5\u4e0b\u9519\u8bef\u5c31\u975e\u5e38\u5e38\u89c1\uff1a $ ls /mnt/oss/ ls: /mnt/oss/: Input/output error \u5982\u679c\u9700\u8981\u5b9a\u4f4d\u5177\u4f53\u7684\u9519\u8bef\u539f\u56e0\uff0c\u53ef\u4ee5\u6839\u636e JindoSDK \u914d\u7f6e\u4e2d\u7684 logger.dir\uff0c\u5728\u6307\u5b9a\u8def\u5f84\u4e0b\u7684 jindosdk.log \u6587\u4ef6\u4e2d\uff0c\u5bfb\u627e\u5177\u4f53\u7684\u9519\u8bef\u3002 \u4e0b\u9762\u5c55\u793a\u7684\u8fd9\u4e2a\u9519\u8bef\u5c31\u6765\u81ea jindosdk.log \uff0c\u6839\u636e\u62a5\u9519\u4fe1\u606f\u53ef\u89c1\uff0c\u8fd9\u4e2a\u9519\u8bef\u662f\u5e38\u89c1\u7684\u9274\u6743\u95ee\u9898\u3002 EMMDD HH:mm:ss jindofs_connectivity.cpp:13] Please check your Endpoint/Bucket/RoleArn. Failed test connectivity, operation: mkdir, errMsg: [RequestId]: 618B8183343EA53531C62B74 [HostId]: oss-cn-shanghai-internal.aliyuncs.com [ErrorMessage]: [E1010]HTTP/1.1 403 Forbidden ...","title":"Input/Output error"},{"location":"backup/oss/jindo_fuse/jindo_fuse_overview/","text":"JindoFUSE \u4f7f\u7528 \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 \u76ee\u524d JindoFUSE \u53ea\u652f\u6301 Linux \u64cd\u4f5c\u7cfb\u7edf\u3002\u5927\u591a\u6570 Linux \u53d1\u884c\u7248\u5df2\u7ecf\u5185\u7f6e\u4e86 FUSE \u6a21\u5757\uff0c\u6ca1\u6709\u7684\u8bdd\u60a8\u9700\u8981\u5b89\u88c5\u6216\u8005\u7f16\u8bd1 FUSE \u6a21\u5757\u3002 \u4ec0\u4e48\u662f JindoFUSE \u00a7 FUSE \u662f Linux \u7cfb\u7edf\u5185\u6838\u63d0\u4f9b\u7684\u4e00\u79cd\u6302\u8f7d\u6587\u4ef6\u7cfb\u7edf\u7684\u65b9\u5f0f\u3002 \u901a\u8fc7 JindoFUSE \u5ba2\u6237\u7aef\uff0c\u5c06\u5bf9\u8c61\u5b58\u50a8 OSS \u670d\u52a1\u6216 DLS \u96c6\u7fa4\u4e0a\u7684\u6587\u4ef6\u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u4e00\u6837\u64cd\u4f5c OSS\u3001DLS \u96c6\u7fa4\u4e0a\u7684\u6587\u4ef6\u3002 \u4e3a\u4ec0\u4e48\u4f7f\u7528 JindoFUSE \u00a7 JindoFUSE \u57fa\u4e8e JindoSDK\uff0c\u5728\u8bbf\u95ee OSS \u7684\u80fd\u529b\u4e0a\uff0c\u76f8\u5bf9\u4e8e\u5f00\u6e90\u7684 ossfs\u5ba2\u6237\u7aef \u505a\u4e86\u5f88\u591a\u7684\u6027\u80fd\u4f18\u5316\u3002 JindoFUSE \u57fa\u4e8e C++ \u5b9e\u73b0\uff0c\u4e0d\u4f9d\u8d56 JVM\uff0c\u76f8\u5bf9\u4e8e HDFS \u6216 Alluxio \u7684 FUSE \u5b9e\u73b0\u8282\u7701\u4e86 JNI \u8c03\u7528\u7684\u5f00\u9500\u3002 \u5173\u4e8e JindoFUSE \u5728\u6027\u80fd\u4e0a\u7684\u4f18\u52bf\uff0c\u6211\u4eec\u505a\u4e86\u5bf9\u6bd4\u6d4b\u8bd5\uff0c\u53ef\u4ee5\u53c2\u8003 JindoFUSE \u6027\u80fd\u6d4b\u8bd5 \u3002 JindoFUSE \u4f7f\u7528 \u00a7 \u4f7f\u7528 JindoFUSE \u8bbf\u95ee OSS","title":"JindoFUSE \u4f7f\u7528"},{"location":"backup/oss/jindo_fuse/jindo_fuse_overview/#jindofuse","text":"","title":"JindoFUSE \u4f7f\u7528"},{"location":"backup/oss/jindo_fuse/jindo_fuse_overview/#_1","text":"\u76ee\u524d JindoFUSE \u53ea\u652f\u6301 Linux \u64cd\u4f5c\u7cfb\u7edf\u3002\u5927\u591a\u6570 Linux \u53d1\u884c\u7248\u5df2\u7ecf\u5185\u7f6e\u4e86 FUSE \u6a21\u5757\uff0c\u6ca1\u6709\u7684\u8bdd\u60a8\u9700\u8981\u5b89\u88c5\u6216\u8005\u7f16\u8bd1 FUSE \u6a21\u5757\u3002","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/oss/jindo_fuse/jindo_fuse_overview/#jindofuse_1","text":"FUSE \u662f Linux \u7cfb\u7edf\u5185\u6838\u63d0\u4f9b\u7684\u4e00\u79cd\u6302\u8f7d\u6587\u4ef6\u7cfb\u7edf\u7684\u65b9\u5f0f\u3002 \u901a\u8fc7 JindoFUSE \u5ba2\u6237\u7aef\uff0c\u5c06\u5bf9\u8c61\u5b58\u50a8 OSS \u670d\u52a1\u6216 DLS \u96c6\u7fa4\u4e0a\u7684\u6587\u4ef6\u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u4e00\u6837\u64cd\u4f5c OSS\u3001DLS \u96c6\u7fa4\u4e0a\u7684\u6587\u4ef6\u3002","title":"\u4ec0\u4e48\u662f JindoFUSE"},{"location":"backup/oss/jindo_fuse/jindo_fuse_overview/#jindofuse_2","text":"JindoFUSE \u57fa\u4e8e JindoSDK\uff0c\u5728\u8bbf\u95ee OSS \u7684\u80fd\u529b\u4e0a\uff0c\u76f8\u5bf9\u4e8e\u5f00\u6e90\u7684 ossfs\u5ba2\u6237\u7aef \u505a\u4e86\u5f88\u591a\u7684\u6027\u80fd\u4f18\u5316\u3002 JindoFUSE \u57fa\u4e8e C++ \u5b9e\u73b0\uff0c\u4e0d\u4f9d\u8d56 JVM\uff0c\u76f8\u5bf9\u4e8e HDFS \u6216 Alluxio \u7684 FUSE \u5b9e\u73b0\u8282\u7701\u4e86 JNI \u8c03\u7528\u7684\u5f00\u9500\u3002 \u5173\u4e8e JindoFUSE \u5728\u6027\u80fd\u4e0a\u7684\u4f18\u52bf\uff0c\u6211\u4eec\u505a\u4e86\u5bf9\u6bd4\u6d4b\u8bd5\uff0c\u53ef\u4ee5\u53c2\u8003 JindoFUSE \u6027\u80fd\u6d4b\u8bd5 \u3002","title":"\u4e3a\u4ec0\u4e48\u4f7f\u7528 JindoFUSE"},{"location":"backup/oss/jindo_fuse/jindo_fuse_overview/#jindofuse_3","text":"\u4f7f\u7528 JindoFUSE \u8bbf\u95ee OSS","title":"JindoFUSE \u4f7f\u7528"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/","text":"\u4f7f\u7528 Jindo Tensorflow Collector \u8bbf\u95ee OSS \u00a7 Jindo Tensorflow Collector \u652f\u6301\u4f7f\u7528 Tensoflow \u539f\u751f API \u652f\u6301\u8bbf\u95ee OSS\u3002 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u4fee\u6539\u914d\u7f6e \u00a7 2.1 \u5b89\u88c5 tensorflow \u00a7 \u76ee\u524d\u4ec5\u652f\u6301 tensorflow 1.15 \u6216 tensorflow 2.8 \u7248\u672c\uff0c\u4ee5\u4e0b\u4ee5 tensorflow 2.8 \u4e3a\u4f8b\u3002 pip3 install tensorflow==2.8 2.2 \u6dfb\u52a0\u73af\u5883\u53d8\u91cf \u00a7 2.2.1 JindoSDK \u73af\u5883\u53d8\u91cf \u00a7 export LD_LIBRARY_PATH=\"/path/to/jindosdk-x.x.x/lib/native\":$LD_LIBRARY_PATH export JINDOSDK_CONF_DIR=/path/to/jindosdk-x.x.x/conf 2.2.2 \u4f9d\u8d56\u7684 Tensorflow \u73af\u5883\u53d8\u91cf \u00a7 \u540c\u65f6\uff0cLD_LIBRARY_PATH \u9700\u8981\u6dfb\u52a0 tensorflow \u6240\u5728\u7684\u4f9d\u8d56\u8def\u5f84\uff0c\u4e0d\u540c\u7684 python \u73af\u5883\u5b89\u88c5\u8def\u5f84\u53ef\u80fd\u4e0d\u540c\uff0c\u5982\uff1a # tensorflow1.15 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow/\":$LD_LIBRARY_PATH # tensorflow2.8 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow/\":$LD_LIBRARY_PATH 2.3 \u914d\u7f6e\u6587\u4ef6 \u00a7 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \u3002\u53ef\u4ee5\u9009\u62e9\u56fa\u5b9a\u6216\u8005\u514d\u5bc6\u4e24\u79cd\u914d\u7f6e\u65b9\u5f0f\u3002 2.3.1 \u56fa\u5b9a AK \u8bbf\u95ee \u00a7 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/tfio-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95eeOSS\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> \u914d\u7f6e\u9009\u9879 \u00a7 \u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u914d\u7f6e\u8282 \u8bf4\u660e logger.dir /tmp/tfio-log common \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false common \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false common \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\u3002 \u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002 logger.verbose 0 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false common \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u5730\u5740\uff0c\u5982cn-xxx.oss-dls.aliyuncs.com fs.oss.accessKeyId jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeySecret \u66f4\u591a jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 2.3.2 \u514d\u5bc6\u8bbf\u95ee \u00a7 \u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7\u89d2\u8272\u6388\u6743\u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE 3. \u4f7f\u7528 Jindo Tensorflow \u00a7 \u4ee5\u52a0\u8f7d jindo-tensorflow2.8 \u4e3a\u4f8b 3.1 \u901a\u8fc7 load_library \u6dfb\u52a0 collector \u00a7 import tensorflow as tf tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\") 3.2 \u901a\u8fc7 gfile \u8bbf\u95ee oss-hdfs \u00a7 \u4e3a\u907f\u514d\u4e0e tfio \u4e2d\u7684 oss \u524d\u7f00\u51b2\u7a81\uff0cjindo-tensorflow \u4f7f\u7528 joss \u524d\u7f00\u3002 # \u76ee\u5f55\u64cd\u4f5c tf.io.gfile.mkdir(\"joss://<bucket>/test_mkdir\") content = tf.io.gfile.listdir(\"joss://<bucket>/test_mkdir\") # \u8bfb gmnist = tf.io.gfile.GFile(\"joss://<bucket>/mnist.npz\", \"rb\") gmnist_file = gmnist.read() # \u5199 fh = gfile.Open(f, mode=\"w\") content = \"file content\" fh.write(content) fh.close() 3.3 \u8bfb\u53d6 TFRecordDataset \u00a7 tfRecordOssPath = \"joss://<bucket>/test.tfrecord\" filenames = [tfRecordOssPath] raw_dataset = tf.data.TFRecordDataset(filenames) 3.4 \u5b8c\u6574\u793a\u4f8b \u00a7 import tensorflow as tf import numpy as np tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\") gmnist = tf.io.gfile.GFile(\"joss://<bucket>/mnist.npz\", 'rb') print(gmnist.size()) with np.load(gmnist) as f: x_train, y_train = f['x_train'], f['y_train'] x_test, y_test = f['x_test'], f['y_test'] model = tf.keras.models.Sequential([ tf.keras.layers.Flatten(input_shape=(28, 28)), tf.keras.layers.Dense(128, activation='relu'), tf.keras.layers.Dropout(0.2), tf.keras.layers.Dense(10, activation='softmax') ]) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) model.fit(x_train, y_train, epochs=5)","title":"\u4f7f\u7528 Jindo Tensorflow Collector \u8bbf\u95ee OSS"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#jindo-tensorflow-collector-oss","text":"Jindo Tensorflow Collector \u652f\u6301\u4f7f\u7528 Tensoflow \u539f\u751f API \u652f\u6301\u8bbf\u95ee OSS\u3002","title":"\u4f7f\u7528 Jindo Tensorflow Collector \u8bbf\u95ee OSS"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#2","text":"","title":"2. \u4fee\u6539\u914d\u7f6e"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#21-tensorflow","text":"\u76ee\u524d\u4ec5\u652f\u6301 tensorflow 1.15 \u6216 tensorflow 2.8 \u7248\u672c\uff0c\u4ee5\u4e0b\u4ee5 tensorflow 2.8 \u4e3a\u4f8b\u3002 pip3 install tensorflow==2.8","title":"2.1 \u5b89\u88c5 tensorflow"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#22","text":"","title":"2.2 \u6dfb\u52a0\u73af\u5883\u53d8\u91cf"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#221-jindosdk","text":"export LD_LIBRARY_PATH=\"/path/to/jindosdk-x.x.x/lib/native\":$LD_LIBRARY_PATH export JINDOSDK_CONF_DIR=/path/to/jindosdk-x.x.x/conf","title":"2.2.1 JindoSDK \u73af\u5883\u53d8\u91cf"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#222-tensorflow","text":"\u540c\u65f6\uff0cLD_LIBRARY_PATH \u9700\u8981\u6dfb\u52a0 tensorflow \u6240\u5728\u7684\u4f9d\u8d56\u8def\u5f84\uff0c\u4e0d\u540c\u7684 python \u73af\u5883\u5b89\u88c5\u8def\u5f84\u53ef\u80fd\u4e0d\u540c\uff0c\u5982\uff1a # tensorflow1.15 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow/\":$LD_LIBRARY_PATH # tensorflow2.8 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow/\":$LD_LIBRARY_PATH","title":"2.2.2 \u4f9d\u8d56\u7684 Tensorflow \u73af\u5883\u53d8\u91cf"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#23","text":"\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \u3002\u53ef\u4ee5\u9009\u62e9\u56fa\u5b9a\u6216\u8005\u514d\u5bc6\u4e24\u79cd\u914d\u7f6e\u65b9\u5f0f\u3002","title":"2.3 \u914d\u7f6e\u6587\u4ef6"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#231-ak","text":"\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/tfio-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95eeOSS\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret>","title":"2.3.1 \u56fa\u5b9a AK \u8bbf\u95ee"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#_1","text":"\u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u914d\u7f6e\u8282 \u8bf4\u660e logger.dir /tmp/tfio-log common \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false common \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false common \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\u3002 \u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002 logger.verbose 0 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false common \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u5730\u5740\uff0c\u5982cn-xxx.oss-dls.aliyuncs.com fs.oss.accessKeyId jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeySecret \u66f4\u591a jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002","title":"\u914d\u7f6e\u9009\u9879"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#232","text":"\u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7\u89d2\u8272\u6388\u6743\u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/fuse-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE","title":"2.3.2 \u514d\u5bc6\u8bbf\u95ee"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#3-jindo-tensorflow","text":"\u4ee5\u52a0\u8f7d jindo-tensorflow2.8 \u4e3a\u4f8b","title":"3. \u4f7f\u7528 Jindo Tensorflow"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#31-load_library-collector","text":"import tensorflow as tf tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\")","title":"3.1 \u901a\u8fc7 load_library \u6dfb\u52a0 collector"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#32-gfile-oss-hdfs","text":"\u4e3a\u907f\u514d\u4e0e tfio \u4e2d\u7684 oss \u524d\u7f00\u51b2\u7a81\uff0cjindo-tensorflow \u4f7f\u7528 joss \u524d\u7f00\u3002 # \u76ee\u5f55\u64cd\u4f5c tf.io.gfile.mkdir(\"joss://<bucket>/test_mkdir\") content = tf.io.gfile.listdir(\"joss://<bucket>/test_mkdir\") # \u8bfb gmnist = tf.io.gfile.GFile(\"joss://<bucket>/mnist.npz\", \"rb\") gmnist_file = gmnist.read() # \u5199 fh = gfile.Open(f, mode=\"w\") content = \"file content\" fh.write(content) fh.close()","title":"3.2 \u901a\u8fc7 gfile \u8bbf\u95ee oss-hdfs"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#33-tfrecorddataset","text":"tfRecordOssPath = \"joss://<bucket>/test.tfrecord\" filenames = [tfRecordOssPath] raw_dataset = tf.data.TFRecordDataset(filenames)","title":"3.3 \u8bfb\u53d6 TFRecordDataset"},{"location":"backup/oss/jindo_tensorflow/jindo_tensorflow_on_oss/#34","text":"import tensorflow as tf import numpy as np tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\") gmnist = tf.io.gfile.GFile(\"joss://<bucket>/mnist.npz\", 'rb') print(gmnist.size()) with np.load(gmnist) as f: x_train, y_train = f['x_train'], f['y_train'] x_test, y_test = f['x_test'], f['y_test'] model = tf.keras.models.Sequential([ tf.keras.layers.Flatten(input_shape=(28, 28)), tf.keras.layers.Dense(128, activation='relu'), tf.keras.layers.Dropout(0.2), tf.keras.layers.Dense(10, activation='softmax') ]) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) model.fit(x_train, y_train, epochs=5)","title":"3.4 \u5b8c\u6574\u793a\u4f8b"},{"location":"backup/oss/jindotable/jindotable_archive/","text":"JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a \u00a7 JindoTable \u63d0\u4f9b archiveTable \u548c unarchiveTable \u547d\u4ee4\uff0c\u53ef\u4ee5\u8fdb\u884c\u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a\u7684\u64cd\u4f5c\u3002\u672c\u6587\u4e3a\u60a8\u4ecb\u7ecd archiveTable \u548c unarchiveTable \u547d\u4ee4\u7684\u4f7f\u7528\u65b9\u6cd5\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5f85\u5f52\u6863\u7684\u6570\u636e\u5fc5\u987b\u662f\u8868\u6570\u636e\uff08\u53ef\u4ee5\u662f\u5206\u533a\u8868\u6216\u975e\u5206\u533a\u8868\uff09\uff0c\u4e14\u5df2\u7ecf\u4f4d\u4e8e\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8 OSS\u3002 \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 JindoSDK + \u963f\u91cc\u4e91 OSS \u5feb\u901f\u5165\u95e8 \u3002 archiveTable \u547d\u4ee4 \u00a7 archiveTable \u547d\u4ee4\u53ef\u4ee5\u5bf9 OSS \u4e0a\u7684\u8868\u6216\u5206\u533a\u8fdb\u884c\u5f52\u6863\u3002 \u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f \u00a7 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help archiveTable \u53c2\u6570\u8bf4\u660e \u00a7 jindotable -archiveTable -t <dbName.tableName> -i/-a/-ca [-c \"<condition>\" | -fullTable] [-b/-before <before days>] [-p/-parallel <parallelism>] [-mr/-mapReduce] [-e/-explain] [-w/-workingDir <working directory>][-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u5f85\u5f52\u6863\u7684\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a\u6570\u636e\u5e93\u540d.\u8868\u540d\u3002\u6570\u636e\u5e93\u548c\u8868\u540d\u4e4b\u95f4\u4ee5\u534a\u89d2\u53e5\u53f7\uff08.\uff09\u5206\u9694\u3002\u8868\u53ef\u4ee5\u662f\u5206\u533a\u8868\u6216\u975e\u5206\u533a\u8868\u3002 \u662f -i/-a/-ca \u76ee\u6807\u5b58\u50a8\u65b9\u5f0f\u3002\u652f\u6301\u5982\u4e0b\u65b9\u5f0f\uff1a-a\uff1a\u5f52\u6863\uff08Archive\uff09\u5b58\u50a8\u3002-i\uff1a\u4f4e\u9891 \uff08Infrequent Access\uff0cIA\uff09\u5b58\u50a8\u3002-ca\uff1a\u51b7\u5f52\u6863\uff08Code Archive\uff09\u5b58\u50a8\u3002\u5982\u679c\u4f7f\u7528-i\u5373\u8868\u793a\u4f4e\u9891\u5b58\u50a8\uff0c\u4f1a\u8df3\u8fc7\u5df2\u7ecf\u5904\u4e8e\u5f52\u6863\u5b58\u50a8\u7684\u6587\u4ef6\u3002 \u662f -c \"<condition>\" / -fullTable -fullTable \u548c -c \"<condition>\" \u53ea\u9700\u63d0\u4f9b\u4e00\u4e2a\uff0c\u5373\u8981\u4e48\u6307\u5b9a -c \"<condition>\" \uff0c\u8981\u4e48\u6307\u5b9a -fullTable \u3002\u6307\u5b9a -fullTable \u65f6\uff0c\u5219\u4e3a\u79fb\u52a8\u6574\u8868\uff0c\u65e2\u53ef\u4ee5\u662f\u975e\u5206\u533a\u8868\u4e5f\u53ef\u4ee5\u662f\u5206\u533a\u8868\u3002\u6307\u5b9a -c \"<condition>\" \u65f6\uff0c\u5219\u63d0\u4f9b\u4e86\u4e00\u4e2a\u8fc7\u6ee4\u6761\u4ef6\uff0c\u7528\u6765\u9009\u62e9\u5e0c\u671b\u79fb\u52a8\u7684\u5206\u533a\uff0c\u652f\u6301\u5e38\u89c1\u8fd0\u7b97\u7b26\uff0c\u4f8b\u5982\u5927\u4e8e\u53f7\uff08>\uff09\u3002\u4f8b\u5982\uff0c\u6570\u636e\u7c7b\u578b\u4e3aString\u7684\u5206\u533ads\uff0c\u5e0c\u671b\u5206\u533a\u540d\u5927\u4e8e 'd'\uff0c\u5219\u4ee3\u7801\u4e3a-c \" ds > 'd' \"\u3002 \u5426 -b/before <before days> \u53ea\u6709\u521b\u5efa\u65f6\u95f4\u8ddd\u79bb\u73b0\u5728\u8d85\u8fc7\u4e00\u5b9a\u5929\u6570\u7684\u8868\u6216\u5206\u533a\u624d\u4f1a\u88ab\u5f52\u6863\u3002 \u5426 -p/-parallel <parallelism> \u5f52\u6863\u64cd\u4f5c\u7684\u5e76\u884c\u5ea6\u3002 \u5426 -mr/-mapReduce \u4f7f\u7528Hadoop MapReduce\u800c\u975e\u672c\u5730\u591a\u7ebf\u7a0b\u6765\u5f52\u6863\u6570\u636e\u3002 \u5426 -e/-explain \u5982\u679c\u51fa\u73b0\u8be5\u9009\u9879\uff0c\u5219\u4e3a\u89e3\u91ca\uff08explain\uff09\u6a21\u5f0f\uff0c\u53ea\u4f1a\u663e\u793a\u5f85\u79fb\u52a8\u7684\u5206\u533a\u5217\u8868\uff0c\u800c\u4e0d\u4f1a\u771f\u6b63\u79fb\u52a8\u6570\u636e\u3002 \u5426 -w/-workingDir \u53ea\u5728MapReduce\u4f5c\u4e1a\u65f6\u4f7f\u7528\uff0c\u4e3aMapReduce\u4f5c\u4e1a\u7684\u5de5\u4f5c\u76ee\u5f55\u3002\u5fc5\u987b\u6709\u8bfb\u5199\u6743\u9650\uff0c\u5de5\u4f5c\u76ee\u5f55\u53ef\u4ee5\u975e\u7a7a\uff08\u4f5c\u4e1a\u6267\u884c\u8fc7\u7a0b\u4e2d\u4f1a\u521b\u5efa\u4e34\u65f6\u6587\u4ef6\uff0c\u6267\u884c\u5b8c\u6bd5\u4f1a\u6e05\u7406\u4e34\u65f6\u6587\u4ef6\uff09\u3002 \u5426 -l/-logDir <log directory> \u6307\u5b9aLog\u6587\u4ef6\u7684\u76ee\u5f55\u3002 \u5426 unarchiveTable \u547d\u4ee4 \u00a7 unarchiveTable \u547d\u4ee4\u4e0e archiveTable \u547d\u4ee4\u683c\u5f0f\u57fa\u672c\u4e00\u81f4\uff0c\u4f46\u6548\u679c\u76f8\u53cd\u3002unarchiveTable \u547d\u4ee4\u53ef\u4ee5\u5bf9 OSS \u4e0a\u7684\u8868\u6216\u5206\u533a\u8fdb\u884c\u89e3\u51bb\u3002 \u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f \u00a7 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help unarchiveTable \u53c2\u6570\u8bf4\u660e \u00a7 jindotable -unarchiveTable -t <dbName.tableName> [-i/-a/-o/-cr] [-notWait] [-c \"<condition>\" | -fullTable] [-d/-restoreDays <restore days>] [-b/-before <before days>] [-p/-parallel <parallelism>] [-mr/-mapReduce] [-e/-explain] [-w/-workingDir <working directory>][-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u5f85\u89e3\u51bb\u7684\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a\u6570\u636e\u5e93\u540d.\u8868\u540d\u3002\u6570\u636e\u5e93\u548c\u8868\u540d\u4e4b\u95f4\u4ee5\u534a\u89d2\u53e5\u53f7\uff08.\uff09\u5206\u9694\u3002\u8868\u53ef\u4ee5\u662f\u5206\u533a\u8868\u6216\u975e\u5206\u533a\u8868\u3002 \u662f -i/-a/-o/-cr \u5982\u679c\u4e0d\u6307\u5b9a\u53c2\u6570\uff0c\u5219\u8f6c\u6362\u5b58\u50a8\u683c\u5f0f\u4e3a\u6807\u51c6\uff08Standard\uff09\u5b58\u50a8\u3002\u652f\u6301\u5982\u4e0b\u65b9\u5f0f\uff1a-i\uff1a\u5219\u8f6c\u6362\u5b58\u50a8\u683c\u5f0f\u4e3a\u4f4e\u9891\uff08Infrequent Access\uff0cIA\uff09\u5b58\u50a8\uff0c\u539f\u672c\u4e3a\u6807\u51c6\u5b58\u50a8\u7684\u6587\u4ef6\u88ab\u8df3\u8fc7\u3002-a\uff1a\u5219\u8f6c\u6362\u5b58\u50a8\u683c\u5f0f\u5f52\u6863\uff08Archive\uff09\u5b58\u50a8\u3002-o: \u4ec5\u505a\u89e3\u51bb\uff08Restore\uff09\u64cd\u4f5c\u3002\u539f\u672c\u4e3a\u6807\u51c6\u5b58\u50a8\u6216\u4f4e\u9891\u5b58\u50a8\u7684\u6587\u4ef6\u5747\u88ab\u8df3\u8fc7\u3002\u5df2\u7ecf\u5904\u4e8e\u89e3\u51bb\u72b6\u6001\u7684\u6587\u4ef6\u4e5f\u4f1a\u88ab\u8df3\u8fc7\uff0c\u5373\u4e0d\u4f1a\u91cd\u590d\u89e3\u51bb\u3002-cr\uff1a\u68c0\u67e5\u662f\u5426\u6240\u6709 restore \u4efb\u52a1\u5df2\u5b8c\u6210\u3002 \u5426 -notWait \u53ea\u5bf9 Restore \u64cd\u4f5c\u751f\u6548\u3002\u5982\u679c\u51fa\u73b0\u8be5\u9009\u9879\uff0c\u5219\u4e0d\u4f1a\u7b49 OSS \u670d\u52a1\u7aef\u5b8c\u6210 Restore \u64cd\u4f5c\u3002\u5426\u5219\uff0c\u4f1a\u7b49 Restore \u64cd\u4f5c\u5b8c\u6210\u6216\u8005\u8d85\u65f6\uff0810\u5206\u949f\uff09\u624d\u9000\u51fa \u5426 -c \"<condition>\" / -fullTable -fullTable \u548c -c \"<condition>\" \u53ea\u9700\u63d0\u4f9b\u4e00\u4e2a\uff0c\u5373\u8981\u4e48\u6307\u5b9a -c \"<condition>\" \uff0c\u8981\u4e48\u6307\u5b9a -fullTable \u3002\u6307\u5b9a -fullTable \u65f6\uff0c\u5219\u4e3a\u79fb\u52a8\u6574\u8868\uff0c\u65e2\u53ef\u4ee5\u662f\u975e\u5206\u533a\u8868\u4e5f\u53ef\u4ee5\u662f\u5206\u533a\u8868\u3002\u6307\u5b9a -c \"<condition>\" \u65f6\uff0c\u5219\u63d0\u4f9b\u4e86\u4e00\u4e2a\u8fc7\u6ee4\u6761\u4ef6\uff0c\u7528\u6765\u9009\u62e9\u5e0c\u671b\u79fb\u52a8\u7684\u5206\u533a\uff0c\u652f\u6301\u5e38\u89c1\u8fd0\u7b97\u7b26\uff0c\u4f8b\u5982\u5927\u4e8e\u53f7\uff08>\uff09\u3002\u4f8b\u5982\uff0c\u6570\u636e\u7c7b\u578b\u4e3aString\u7684\u5206\u533ads\uff0c\u5e0c\u671b\u5206\u533a\u540d\u5927\u4e8e 'd'\uff0c\u5219\u4ee3\u7801\u4e3a-c \" ds > 'd' \"\u3002 \u5426 -d/-restoreDays <restore days> \u5f53\u6d89\u53ca\u4ec5\u89e3\u51bb\uff08Restore\uff09\u64cd\u4f5c\u65f6\uff0c\u7ef4\u6301\u89e3\u51bb\u72b6\u6001\u7684\u5929\u6570\uff0c\u9ed8\u8ba4\u4e3a\u4e00\u5929\u3002\u4f8b\u5982\uff0c\u5bf9\u4e00\u4e2a\u51b7\u5f52\u6863\u7684\u8868\u505a\u4ec5\u89e3\u51bb\u64cd\u4f5c\u65f6\uff08-o \u53c2\u6570\uff09\uff0c\u8be5\u53c2\u6570\u5c06\u4ea7\u751f\u4f5c\u7528\uff1b\u5bf9\u4e00\u4e2a\u4ece\u672a\u89e3\u51bb\u8fc7\u7684\u5f52\u6863\u72b6\u6001\u7684\u8868\u8f6c\u4e3a\u6807\u51c6\u5b58\u50a8\u65f6\uff0c\u8be5\u53c2\u6570\u5c06\u5728\u4e2d\u95f4\u73af\u8282\u53d1\u6325\u4f5c\u7528\uff1b\u800c\u5bf9\u4e00\u4e2a\u4f4e\u9891\u72b6\u6001\u7684\u8868\u8f6c\u4e3a\u6807\u51c6\u5b58\u50a8\u65f6\uff0c\u8be5\u53c2\u6570\u4e0d\u4f1a\u4ea7\u751f\u4f5c\u7528\u3002 \u5426 -b/-before <before days> \u53ea\u6709\u521b\u5efa\u65f6\u95f4\u8ddd\u79bb\u73b0\u5728\u8d85\u8fc7\u4e00\u5b9a\u5929\u6570\u7684\u8868\u6216\u5206\u533a\u624d\u4f1a\u88ab\u89e3\u51bb\u3002 \u5426 -p/-parallel <parallelism> \u89e3\u51bb\u64cd\u4f5c\u7684\u5e76\u884c\u5ea6\u3002 \u5426 -mr/-mapReduce \u4f7f\u7528Hadoop MapReduce\u800c\u975e\u672c\u5730\u591a\u7ebf\u7a0b\u6765\u89e3\u51bb\u6570\u636e\u3002 \u5426 -e/-explain \u5982\u679c\u51fa\u73b0\u8be5\u9009\u9879\uff0c\u5219\u4e3a\u89e3\u91ca\uff08explain\uff09\u6a21\u5f0f\uff0c\u53ea\u4f1a\u663e\u793a\u5f85\u79fb\u52a8\u7684\u5206\u533a\u5217\u8868\uff0c\u800c\u4e0d\u4f1a\u771f\u6b63\u79fb\u52a8\u6570\u636e\u3002 \u5426 -w/-workingDir \u53ea\u5728MapReduce\u4f5c\u4e1a\u65f6\u4f7f\u7528\uff0c\u4e3aMapReduce\u4f5c\u4e1a\u7684\u5de5\u4f5c\u76ee\u5f55\u3002\u5fc5\u987b\u6709\u8bfb\u5199\u6743\u9650\uff0c\u5de5\u4f5c\u76ee\u5f55\u53ef\u4ee5\u975e\u7a7a\uff08\u4f5c\u4e1a\u6267\u884c\u8fc7\u7a0b\u4e2d\u4f1a\u521b\u5efa\u4e34\u65f6\u6587\u4ef6\uff0c\u6267\u884c\u5b8c\u6bd5\u4f1a\u6e05\u7406\u4e34\u65f6\u6587\u4ef6\uff09\u3002 \u5426 -l/-logDir <log directory> \u6307\u5b9aLog\u6587\u4ef6\u7684\u76ee\u5f55\u3002 \u5426","title":"JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a"},{"location":"backup/oss/jindotable/jindotable_archive/#jindotable-oss","text":"JindoTable \u63d0\u4f9b archiveTable \u548c unarchiveTable \u547d\u4ee4\uff0c\u53ef\u4ee5\u8fdb\u884c\u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a\u7684\u64cd\u4f5c\u3002\u672c\u6587\u4e3a\u60a8\u4ecb\u7ecd archiveTable \u548c unarchiveTable \u547d\u4ee4\u7684\u4f7f\u7528\u65b9\u6cd5\u3002","title":"JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a"},{"location":"backup/oss/jindotable/jindotable_archive/#_1","text":"\u5f85\u5f52\u6863\u7684\u6570\u636e\u5fc5\u987b\u662f\u8868\u6570\u636e\uff08\u53ef\u4ee5\u662f\u5206\u533a\u8868\u6216\u975e\u5206\u533a\u8868\uff09\uff0c\u4e14\u5df2\u7ecf\u4f4d\u4e8e\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8 OSS\u3002 \u5df2\u90e8\u7f72 JindoSDK \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 JindoSDK + \u963f\u91cc\u4e91 OSS \u5feb\u901f\u5165\u95e8 \u3002","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/oss/jindotable/jindotable_archive/#archivetable","text":"archiveTable \u547d\u4ee4\u53ef\u4ee5\u5bf9 OSS \u4e0a\u7684\u8868\u6216\u5206\u533a\u8fdb\u884c\u5f52\u6863\u3002","title":"archiveTable \u547d\u4ee4"},{"location":"backup/oss/jindotable/jindotable_archive/#_2","text":"\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help archiveTable","title":"\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f"},{"location":"backup/oss/jindotable/jindotable_archive/#_3","text":"jindotable -archiveTable -t <dbName.tableName> -i/-a/-ca [-c \"<condition>\" | -fullTable] [-b/-before <before days>] [-p/-parallel <parallelism>] [-mr/-mapReduce] [-e/-explain] [-w/-workingDir <working directory>][-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u5f85\u5f52\u6863\u7684\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a\u6570\u636e\u5e93\u540d.\u8868\u540d\u3002\u6570\u636e\u5e93\u548c\u8868\u540d\u4e4b\u95f4\u4ee5\u534a\u89d2\u53e5\u53f7\uff08.\uff09\u5206\u9694\u3002\u8868\u53ef\u4ee5\u662f\u5206\u533a\u8868\u6216\u975e\u5206\u533a\u8868\u3002 \u662f -i/-a/-ca \u76ee\u6807\u5b58\u50a8\u65b9\u5f0f\u3002\u652f\u6301\u5982\u4e0b\u65b9\u5f0f\uff1a-a\uff1a\u5f52\u6863\uff08Archive\uff09\u5b58\u50a8\u3002-i\uff1a\u4f4e\u9891 \uff08Infrequent Access\uff0cIA\uff09\u5b58\u50a8\u3002-ca\uff1a\u51b7\u5f52\u6863\uff08Code Archive\uff09\u5b58\u50a8\u3002\u5982\u679c\u4f7f\u7528-i\u5373\u8868\u793a\u4f4e\u9891\u5b58\u50a8\uff0c\u4f1a\u8df3\u8fc7\u5df2\u7ecf\u5904\u4e8e\u5f52\u6863\u5b58\u50a8\u7684\u6587\u4ef6\u3002 \u662f -c \"<condition>\" / -fullTable -fullTable \u548c -c \"<condition>\" \u53ea\u9700\u63d0\u4f9b\u4e00\u4e2a\uff0c\u5373\u8981\u4e48\u6307\u5b9a -c \"<condition>\" \uff0c\u8981\u4e48\u6307\u5b9a -fullTable \u3002\u6307\u5b9a -fullTable \u65f6\uff0c\u5219\u4e3a\u79fb\u52a8\u6574\u8868\uff0c\u65e2\u53ef\u4ee5\u662f\u975e\u5206\u533a\u8868\u4e5f\u53ef\u4ee5\u662f\u5206\u533a\u8868\u3002\u6307\u5b9a -c \"<condition>\" \u65f6\uff0c\u5219\u63d0\u4f9b\u4e86\u4e00\u4e2a\u8fc7\u6ee4\u6761\u4ef6\uff0c\u7528\u6765\u9009\u62e9\u5e0c\u671b\u79fb\u52a8\u7684\u5206\u533a\uff0c\u652f\u6301\u5e38\u89c1\u8fd0\u7b97\u7b26\uff0c\u4f8b\u5982\u5927\u4e8e\u53f7\uff08>\uff09\u3002\u4f8b\u5982\uff0c\u6570\u636e\u7c7b\u578b\u4e3aString\u7684\u5206\u533ads\uff0c\u5e0c\u671b\u5206\u533a\u540d\u5927\u4e8e 'd'\uff0c\u5219\u4ee3\u7801\u4e3a-c \" ds > 'd' \"\u3002 \u5426 -b/before <before days> \u53ea\u6709\u521b\u5efa\u65f6\u95f4\u8ddd\u79bb\u73b0\u5728\u8d85\u8fc7\u4e00\u5b9a\u5929\u6570\u7684\u8868\u6216\u5206\u533a\u624d\u4f1a\u88ab\u5f52\u6863\u3002 \u5426 -p/-parallel <parallelism> \u5f52\u6863\u64cd\u4f5c\u7684\u5e76\u884c\u5ea6\u3002 \u5426 -mr/-mapReduce \u4f7f\u7528Hadoop MapReduce\u800c\u975e\u672c\u5730\u591a\u7ebf\u7a0b\u6765\u5f52\u6863\u6570\u636e\u3002 \u5426 -e/-explain \u5982\u679c\u51fa\u73b0\u8be5\u9009\u9879\uff0c\u5219\u4e3a\u89e3\u91ca\uff08explain\uff09\u6a21\u5f0f\uff0c\u53ea\u4f1a\u663e\u793a\u5f85\u79fb\u52a8\u7684\u5206\u533a\u5217\u8868\uff0c\u800c\u4e0d\u4f1a\u771f\u6b63\u79fb\u52a8\u6570\u636e\u3002 \u5426 -w/-workingDir \u53ea\u5728MapReduce\u4f5c\u4e1a\u65f6\u4f7f\u7528\uff0c\u4e3aMapReduce\u4f5c\u4e1a\u7684\u5de5\u4f5c\u76ee\u5f55\u3002\u5fc5\u987b\u6709\u8bfb\u5199\u6743\u9650\uff0c\u5de5\u4f5c\u76ee\u5f55\u53ef\u4ee5\u975e\u7a7a\uff08\u4f5c\u4e1a\u6267\u884c\u8fc7\u7a0b\u4e2d\u4f1a\u521b\u5efa\u4e34\u65f6\u6587\u4ef6\uff0c\u6267\u884c\u5b8c\u6bd5\u4f1a\u6e05\u7406\u4e34\u65f6\u6587\u4ef6\uff09\u3002 \u5426 -l/-logDir <log directory> \u6307\u5b9aLog\u6587\u4ef6\u7684\u76ee\u5f55\u3002 \u5426","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"backup/oss/jindotable/jindotable_archive/#unarchivetable","text":"unarchiveTable \u547d\u4ee4\u4e0e archiveTable \u547d\u4ee4\u683c\u5f0f\u57fa\u672c\u4e00\u81f4\uff0c\u4f46\u6548\u679c\u76f8\u53cd\u3002unarchiveTable \u547d\u4ee4\u53ef\u4ee5\u5bf9 OSS \u4e0a\u7684\u8868\u6216\u5206\u533a\u8fdb\u884c\u89e3\u51bb\u3002","title":"unarchiveTable \u547d\u4ee4"},{"location":"backup/oss/jindotable/jindotable_archive/#_4","text":"\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help unarchiveTable","title":"\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f"},{"location":"backup/oss/jindotable/jindotable_archive/#_5","text":"jindotable -unarchiveTable -t <dbName.tableName> [-i/-a/-o/-cr] [-notWait] [-c \"<condition>\" | -fullTable] [-d/-restoreDays <restore days>] [-b/-before <before days>] [-p/-parallel <parallelism>] [-mr/-mapReduce] [-e/-explain] [-w/-workingDir <working directory>][-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u5f85\u89e3\u51bb\u7684\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a\u6570\u636e\u5e93\u540d.\u8868\u540d\u3002\u6570\u636e\u5e93\u548c\u8868\u540d\u4e4b\u95f4\u4ee5\u534a\u89d2\u53e5\u53f7\uff08.\uff09\u5206\u9694\u3002\u8868\u53ef\u4ee5\u662f\u5206\u533a\u8868\u6216\u975e\u5206\u533a\u8868\u3002 \u662f -i/-a/-o/-cr \u5982\u679c\u4e0d\u6307\u5b9a\u53c2\u6570\uff0c\u5219\u8f6c\u6362\u5b58\u50a8\u683c\u5f0f\u4e3a\u6807\u51c6\uff08Standard\uff09\u5b58\u50a8\u3002\u652f\u6301\u5982\u4e0b\u65b9\u5f0f\uff1a-i\uff1a\u5219\u8f6c\u6362\u5b58\u50a8\u683c\u5f0f\u4e3a\u4f4e\u9891\uff08Infrequent Access\uff0cIA\uff09\u5b58\u50a8\uff0c\u539f\u672c\u4e3a\u6807\u51c6\u5b58\u50a8\u7684\u6587\u4ef6\u88ab\u8df3\u8fc7\u3002-a\uff1a\u5219\u8f6c\u6362\u5b58\u50a8\u683c\u5f0f\u5f52\u6863\uff08Archive\uff09\u5b58\u50a8\u3002-o: \u4ec5\u505a\u89e3\u51bb\uff08Restore\uff09\u64cd\u4f5c\u3002\u539f\u672c\u4e3a\u6807\u51c6\u5b58\u50a8\u6216\u4f4e\u9891\u5b58\u50a8\u7684\u6587\u4ef6\u5747\u88ab\u8df3\u8fc7\u3002\u5df2\u7ecf\u5904\u4e8e\u89e3\u51bb\u72b6\u6001\u7684\u6587\u4ef6\u4e5f\u4f1a\u88ab\u8df3\u8fc7\uff0c\u5373\u4e0d\u4f1a\u91cd\u590d\u89e3\u51bb\u3002-cr\uff1a\u68c0\u67e5\u662f\u5426\u6240\u6709 restore \u4efb\u52a1\u5df2\u5b8c\u6210\u3002 \u5426 -notWait \u53ea\u5bf9 Restore \u64cd\u4f5c\u751f\u6548\u3002\u5982\u679c\u51fa\u73b0\u8be5\u9009\u9879\uff0c\u5219\u4e0d\u4f1a\u7b49 OSS \u670d\u52a1\u7aef\u5b8c\u6210 Restore \u64cd\u4f5c\u3002\u5426\u5219\uff0c\u4f1a\u7b49 Restore \u64cd\u4f5c\u5b8c\u6210\u6216\u8005\u8d85\u65f6\uff0810\u5206\u949f\uff09\u624d\u9000\u51fa \u5426 -c \"<condition>\" / -fullTable -fullTable \u548c -c \"<condition>\" \u53ea\u9700\u63d0\u4f9b\u4e00\u4e2a\uff0c\u5373\u8981\u4e48\u6307\u5b9a -c \"<condition>\" \uff0c\u8981\u4e48\u6307\u5b9a -fullTable \u3002\u6307\u5b9a -fullTable \u65f6\uff0c\u5219\u4e3a\u79fb\u52a8\u6574\u8868\uff0c\u65e2\u53ef\u4ee5\u662f\u975e\u5206\u533a\u8868\u4e5f\u53ef\u4ee5\u662f\u5206\u533a\u8868\u3002\u6307\u5b9a -c \"<condition>\" \u65f6\uff0c\u5219\u63d0\u4f9b\u4e86\u4e00\u4e2a\u8fc7\u6ee4\u6761\u4ef6\uff0c\u7528\u6765\u9009\u62e9\u5e0c\u671b\u79fb\u52a8\u7684\u5206\u533a\uff0c\u652f\u6301\u5e38\u89c1\u8fd0\u7b97\u7b26\uff0c\u4f8b\u5982\u5927\u4e8e\u53f7\uff08>\uff09\u3002\u4f8b\u5982\uff0c\u6570\u636e\u7c7b\u578b\u4e3aString\u7684\u5206\u533ads\uff0c\u5e0c\u671b\u5206\u533a\u540d\u5927\u4e8e 'd'\uff0c\u5219\u4ee3\u7801\u4e3a-c \" ds > 'd' \"\u3002 \u5426 -d/-restoreDays <restore days> \u5f53\u6d89\u53ca\u4ec5\u89e3\u51bb\uff08Restore\uff09\u64cd\u4f5c\u65f6\uff0c\u7ef4\u6301\u89e3\u51bb\u72b6\u6001\u7684\u5929\u6570\uff0c\u9ed8\u8ba4\u4e3a\u4e00\u5929\u3002\u4f8b\u5982\uff0c\u5bf9\u4e00\u4e2a\u51b7\u5f52\u6863\u7684\u8868\u505a\u4ec5\u89e3\u51bb\u64cd\u4f5c\u65f6\uff08-o \u53c2\u6570\uff09\uff0c\u8be5\u53c2\u6570\u5c06\u4ea7\u751f\u4f5c\u7528\uff1b\u5bf9\u4e00\u4e2a\u4ece\u672a\u89e3\u51bb\u8fc7\u7684\u5f52\u6863\u72b6\u6001\u7684\u8868\u8f6c\u4e3a\u6807\u51c6\u5b58\u50a8\u65f6\uff0c\u8be5\u53c2\u6570\u5c06\u5728\u4e2d\u95f4\u73af\u8282\u53d1\u6325\u4f5c\u7528\uff1b\u800c\u5bf9\u4e00\u4e2a\u4f4e\u9891\u72b6\u6001\u7684\u8868\u8f6c\u4e3a\u6807\u51c6\u5b58\u50a8\u65f6\uff0c\u8be5\u53c2\u6570\u4e0d\u4f1a\u4ea7\u751f\u4f5c\u7528\u3002 \u5426 -b/-before <before days> \u53ea\u6709\u521b\u5efa\u65f6\u95f4\u8ddd\u79bb\u73b0\u5728\u8d85\u8fc7\u4e00\u5b9a\u5929\u6570\u7684\u8868\u6216\u5206\u533a\u624d\u4f1a\u88ab\u89e3\u51bb\u3002 \u5426 -p/-parallel <parallelism> \u89e3\u51bb\u64cd\u4f5c\u7684\u5e76\u884c\u5ea6\u3002 \u5426 -mr/-mapReduce \u4f7f\u7528Hadoop MapReduce\u800c\u975e\u672c\u5730\u591a\u7ebf\u7a0b\u6765\u89e3\u51bb\u6570\u636e\u3002 \u5426 -e/-explain \u5982\u679c\u51fa\u73b0\u8be5\u9009\u9879\uff0c\u5219\u4e3a\u89e3\u91ca\uff08explain\uff09\u6a21\u5f0f\uff0c\u53ea\u4f1a\u663e\u793a\u5f85\u79fb\u52a8\u7684\u5206\u533a\u5217\u8868\uff0c\u800c\u4e0d\u4f1a\u771f\u6b63\u79fb\u52a8\u6570\u636e\u3002 \u5426 -w/-workingDir \u53ea\u5728MapReduce\u4f5c\u4e1a\u65f6\u4f7f\u7528\uff0c\u4e3aMapReduce\u4f5c\u4e1a\u7684\u5de5\u4f5c\u76ee\u5f55\u3002\u5fc5\u987b\u6709\u8bfb\u5199\u6743\u9650\uff0c\u5de5\u4f5c\u76ee\u5f55\u53ef\u4ee5\u975e\u7a7a\uff08\u4f5c\u4e1a\u6267\u884c\u8fc7\u7a0b\u4e2d\u4f1a\u521b\u5efa\u4e34\u65f6\u6587\u4ef6\uff0c\u6267\u884c\u5b8c\u6bd5\u4f1a\u6e05\u7406\u4e34\u65f6\u6587\u4ef6\uff09\u3002 \u5426 -l/-logDir <log directory> \u6307\u5b9aLog\u6587\u4ef6\u7684\u76ee\u5f55\u3002 \u5426","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"backup/oss/jindotable/jindotable_data_migration/","text":"JindoTable \u6570\u4ed3\u8fc1\u79fb \u00a7 JindoTable \u63d0\u4f9b\u4e86\u9ad8\u6548\u4fbf\u6377\u7684\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u8fc1\u79fb\uff0c\u90e8\u7f72 JindoSDK\uff0c\u60a8\u5c31\u53ef\u4ee5\u4f7f\u7528\u8fd9\u4e9b\u529f\u80fd\u3002 \u4f7f\u7528 JindoTable \u5c06 Hive \u6570\u636e\u5206\u533a\u5f52\u6863\u548c\u5206\u5c42\u5230 OSS","title":"JindoTable \u6570\u4ed3\u8fc1\u79fb"},{"location":"backup/oss/jindotable/jindotable_data_migration/#jindotable","text":"JindoTable \u63d0\u4f9b\u4e86\u9ad8\u6548\u4fbf\u6377\u7684\u547d\u4ee4\u8fdb\u884c\u6570\u636e\u8fc1\u79fb\uff0c\u90e8\u7f72 JindoSDK\uff0c\u60a8\u5c31\u53ef\u4ee5\u4f7f\u7528\u8fd9\u4e9b\u529f\u80fd\u3002 \u4f7f\u7528 JindoTable \u5c06 Hive \u6570\u636e\u5206\u533a\u5f52\u6863\u548c\u5206\u5c42\u5230 OSS","title":"JindoTable \u6570\u4ed3\u8fc1\u79fb"},{"location":"backup/oss/jindotable/table_moveto/","text":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS \u00a7 HDFS \u53d7\u9650\u4e8e\u96c6\u7fa4\u89c4\u6a21\u548c\u6210\u672c\u56e0\u7d20\uff0c\u65e0\u6cd5\u65e0\u9650\u6269\u5c55\uff0c\u5bb9\u91cf\u5b58\u5728\u74f6\u9888\u3002\u4e91\u5382\u5546\u63d0\u4f9b\u7684\u5bf9\u8c61\u5b58\u50a8\u53ef\u4ee5\u4f5c\u4e3a HDFS \u7684\u66ff\u4ee3\u6216\u8865\u5145\uff0c\u6269\u5c55\u4e91\u4e0a Hadoop \u5e73\u53f0\u7684\u5b58\u50a8\u80fd\u529b\u3002 \u672c\u5de5\u5177\u7528\u4e8e\u628a Hive \u6570\u636e\u6839\u636e\u5206\u533a\u952e\u89c4\u5219\u7b5b\u9009\uff0c\u5728 HDFS \u548c OSS \u4e4b\u95f4\u8f6c\u79fb\u5206\u533a\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u90e8\u7f72 JindoSDK \u00a7 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 JindoSDK + \u963f\u91cc\u4e91 OSS \u5feb\u901f\u5165\u95e8 \u3002 \u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883 \u00a7 \u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e \u914d\u7f6e MoveTo \u5de5\u5177\u5728 HDFS \u4e0b\u7684\u9501\u76ee\u5f55\uff1a \u00a7 \u5728 Hadoop \u914d\u7f6e\u6587\u4ef6 core-site.xml \u6216 hdfs-site.xml \uff08\u4efb\u4e00\u5373\u53ef\uff0c\u5728 $HADOOP_CONF_DIR \u76ee\u5f55\u4e0b\uff09\u65b0\u589e\u914d\u7f6e\u9879\uff1a jindotable.moveto.tablelock.base.dir \u8be5\u914d\u7f6e\u7684\u503c\u5e94\u6307\u5411\u4e00\u4e2a HDFS \u76ee\u5f55\uff0c\u76ee\u7684\u662f\u5b58\u653e MoveTo \u5de5\u5177\u5728\u8fd0\u884c\u65f6\u81ea\u52a8\u521b\u5efa\u7684\u9501\u6587\u4ef6\u3002\u9700\u786e\u4fdd\u8be5\u76ee\u5f55\u53ea\u4f1a\u88ab MoveTo \u5de5\u5177\u8bbf\u95ee\uff0c\u5e76\u4e14\u6709\u8bbf\u95ee\u6743\u9650\u3002\u5982\u679c\u4e0d\u914d\u7f6e\uff0c\u5219\u4f7f\u7528\u7f3a\u7701\u503c hdfs:///tmp/jindotable-lock/ \uff0c\u65e0\u6743\u9650\u5219\u62a5\u9519\u3002 \u4f7f\u7528\u8bf4\u660e \u00a7 \u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f \u00a7 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help moveTo \u53c2\u6570\u8bf4\u660e \u00a7 jindotable -moveTo -t <dbName.tableName> -d <destination path> [-c \"<condition>\" | -fullTable] [-b/-before <before days>] [-p/-parallel <parallelism>] [-s/-storagePolicy <OSS storage policy>] [-o/-overWrite] [-r/-removeSource] [-skipTrash] [-e/-explain] [-q/-queue <yarn queue>] [-w/-workingDir <working directory>][-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u8981\u79fb\u52a8\u7684\u8868\u3002 \u662f -d <destination path> \u76ee\u6807\u8def\u5f84\uff0c\u4e3a\u8868\u7ea7\u522b\u7684\u8def\u5f84\uff0c\u5206\u533a\u8def\u5f84\u4f1a\u5728\u8fd9\u4e2a\u8def\u5f84\u4e0b\u81ea\u52a8\u521b\u5efa\u3002 \u662f -c \"<condition>\" / -fullTable \u5206\u533a\u8fc7\u6ee4\u6761\u4ef6\u8868\u8fbe\u5f0f\uff0c\u652f\u6301\u57fa\u672c\u8fd0\u7b97\u7b26\uff0c\u4e0d\u652f\u6301udf\u3002 \u5426 -b/before <before days> \u6839\u636e\u5206\u533a\u521b\u5efa\u65f6\u95f4\uff0c\u521b\u5efa\u65f6\u95f4\u8d85\u8fc7\u7ed9\u5b9a\u5929\u6570\u7684\u5206\u533a\u624d\u8fdb\u884c\u79fb\u52a8\u3002 \u5426 -p/-parallel <parallelism> \u6574\u4e2amoveTo\u4efb\u52a1\u7684\u6700\u5927task\u5e76\u53d1\u5ea6\uff0c\u9ed8\u8ba4\u4e3a1\u3002 \u5426 -s/-storagePolicy <OSS storage policy> \u62f7\u8d1d\u81f3 OSS \u65f6\u6570\u636e\u6587\u4ef6\u7684\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301 Standard\uff08\u9ed8\u8ba4\uff09\u3001IA\u3001Archive\u3001ColdArchive\u3002 \u5426 -o/-overWrite \u662f\u5426\u8986\u76d6\u6700\u7ec8\u76ee\u5f55\u3002\u5206\u533a\u7ea7\u522b\u8986\u76d6\uff0c\u4e0d\u4f1a\u8986\u76d6\u672c\u6b21\u79fb\u52a8\u4e0d\u6d89\u53ca\u7684\u5206\u533a\u3002 \u5426 -r/-removeSource \u662f\u5426\u5728\u79fb\u52a8\u5b8c\u6210\u540e\u5220\u9664\u6e90\u8def\u5f84\u3002 \u5426 -skipTrash \u5982\u679c\u5220\u9664\u6e90\u8def\u5f84\uff0c\u662f\u5426\u8df3\u8fc7Trash\u3002 \u5426 -e/-explain \u5982\u679c\u6307\u5b9aexplain\u6a21\u5f0f\uff0c\u4e0d\u4f1a\u89e6\u53d1\u5b9e\u9645\u64cd\u4f5c\uff0c\u4ec5\u6253\u5370\u4f1a\u540c\u6b65\u7684\u5206\u533a\u3002 \u5426 -q/-queue <yarn queue \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684yarn\u961f\u5217\u3002 \u5426 -w/-workingDir \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684\u5de5\u4f5c\u4e34\u65f6\u76ee\u5f55\u3002 \u5426 -l/-logDir <log directory> \u672c\u5730\u65e5\u5fd7\u76ee\u5f55\uff0c\u9ed8\u8ba4\u4e3a /tmp/<current user>/ \u5426 \u4f7f\u7528\u793a\u4f8b \u00a7 \u6709\u4e00\u4e2a HDFS \u4e0a\u7684 Hive \u5206\u533a\u8868\uff0c\u5982\u4e0b\u56fe\u6240\u793a\uff1a \u60f3\u628a bbb \u548c ccc \u5206\u533a\u79fb\u52a8\u5230 OSS\uff0c\u5148\u7528 explain \u6a21\u5f0f\u770b\u770b\u4f1a\u79fb\u52a8\u7684\u5206\u533a\u662f\u5426\u7b26\u5408\u9884\u671f\uff0c\u53c2\u6570\u4e3a -e \u6216 -explain\uff1a \u53bb\u6389\u53c2\u6570 -e\uff0c\u771f\u6b63\u79fb\u52a8\u5206\u533a\uff1a \u6267\u884c\u5b8c\u6210\u540e\uff0c\u68c0\u67e5\u6570\u636e\uff0c\u6570\u636e\u5df2\u7ecf\u5728 OSS\uff1a \u518d\u79fb\u56de HDFS\uff0c\u7ed3\u679c\u5931\u8d25\u4e86\uff0c\u4f1a\u663e\u793a\u5931\u8d25\u539f\u56e0\uff0c\u539f\u6765\u662f\u76ee\u6807\u76ee\u5f55\u975e\u7a7a\uff0c\u63d0\u793a\u53ef\u4ee5\u4f7f\u7528 -overWrite \u6e05\u7a7a\u76ee\u6807\u76ee\u5f55\uff1a \u4f7f\u7528 -overWrite \u5f3a\u5236\u8986\u76d6\uff1a \u5f02\u5e38\u72b6\u6001\u5904\u7406 \u00a7 \u4e3a\u4e86\u5c3d\u53ef\u80fd\u4fdd\u8bc1\u6570\u636e\u5b89\u5168\uff0c\u9632\u6b62\u6c61\u67d3\uff0c\u8be5\u547d\u4ee4\u4f1a\u81ea\u52a8\u68c0\u67e5\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u786e\u4fdd\u4e0d\u5b58\u5728\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u76ee\u5f55\u62f7\u8d1d\u3002 \u5982\u679c\u65e0\u6cd5\u6392\u9664\uff0c\u5219\u8be5\u8868\u6216\u5206\u533a\u7684\u62f7\u8d1d\u547d\u4ee4\u5c06\u6267\u884c\u5931\u8d25\u3002 \u6b64\u65f6\uff0c\u9700\u8981\u4e3b\u52a8\u5e72\u9884\uff0c\u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u6b63\u5728\u6267\u884c\u62f7\u8d1d\u4e4b\u540e\uff0c\u6e05\u7406\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u5219\u53ef\u4ee5\u91cd\u65b0\u6267\u884c\u8be5\u547d\u4ee4\u3002 \u5bf9\u4e8e\u975e\u5206\u533a\u8868\u7684\u62f7\u8d1d\u6216\u8fc1\u79fb\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u5373\u4e3a\u8868\u7ea7\u522b\u76ee\u5f55\uff1b\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u4e3a\u5f85\u62f7\u8d1d\u6216\u79fb\u52a8\u5206\u533a\u7684\u5206\u533a\u7ea7\u76ee\u5f55\uff0c \u53ea\u9700\u5bf9\u5f85\u62f7\u8d1d\u7684\u5206\u533a\u8fdb\u884c\u6e05\u7406\u3002 \u51fa\u73b0\u539f\u56e0 \u00a7 \u5982\u679c\u8be5\u547d\u4ee4\u88ab\u5f02\u5e38\u4e2d\u6b62\uff0c\u5219\u53ef\u80fd\u51fa\u73b0\u9700\u8981\u4e3b\u52a8\u5e72\u9884\u7684\u60c5\u51b5\u3002\u6b64\u65f6\u62f7\u8d1d\u8fd8\u672a\u5b8c\u6210\uff0c\u6e90\u6570\u636e\u4e0e\u8868\u7684\u5143\u4fe1\u606f\u5747\u672a\u6539\u53d8\uff0c \u6570\u636e\u4ecd\u5904\u4e8e\u5b89\u5168\u72b6\u6001\u3002\u5e38\u89c1\u7684\u5f02\u5e38\u4e2d\u6b62\u6709\u4e24\u79cd\u60c5\u51b5\uff1a * \u7528\u6237\u5728\u547d\u4ee4\u5c1a\u672a\u7ed3\u675f\u65f6\uff0c\u4e3b\u52a8\u6740\u6389\u547d\u4ee4\u8fdb\u7a0b * \u7531\u4e8e\u5185\u5b58\u6ea2\u51fa\u7b49\u5f02\u5e38\uff0c\u8fdb\u7a0b\u5f02\u5e38\u4e2d\u6b62 \u5982\u679c\u62f7\u8d1d\u8868\u6216\u5206\u533a\u65f6\u62f7\u8d1d\u5931\u8d25\u5e76\u663e\u793a Conflicts found \uff0c\u5219\u5c5e\u4e8e\u5f02\u5e38\u72b6\u6001\uff0c\u9700\u4e3b\u52a8\u5e72\u9884\u3002 \u5904\u7406\u65b9\u6cd5 \u00a7 \u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u7684\u76ee\u6807\u8def\u5f84\u62f7\u8d1d\u6570\u636e\u3002\u7279\u522b\u662f\uff0c\u786e\u4fdd\u6ca1\u6709 DistCp/JindoDistCp \u7b49\u5206\u5e03\u5f0f\u62f7\u8d1d\u547d\u4ee4\u3002 \u5220\u9664\u76ee\u7684\u5730\u76ee\u5f55\u3002\u5bf9\u4e8e\u975e\u5206\u533a\u8868\uff0c\u5220\u9664\u8868\u4e00\u7ea7\u76ee\u5f55\u3002\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u5220\u9664\u5b58\u5728\u51b2\u7a81\u7684\u5206\u533a\u7ea7\u76ee\u5f55\u3002\u5207\u52ff\u5220\u9664\u6e90\u76ee\u5f55\u3002","title":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS"},{"location":"backup/oss/jindotable/table_moveto/#jindotable-hive-oss","text":"HDFS \u53d7\u9650\u4e8e\u96c6\u7fa4\u89c4\u6a21\u548c\u6210\u672c\u56e0\u7d20\uff0c\u65e0\u6cd5\u65e0\u9650\u6269\u5c55\uff0c\u5bb9\u91cf\u5b58\u5728\u74f6\u9888\u3002\u4e91\u5382\u5546\u63d0\u4f9b\u7684\u5bf9\u8c61\u5b58\u50a8\u53ef\u4ee5\u4f5c\u4e3a HDFS \u7684\u66ff\u4ee3\u6216\u8865\u5145\uff0c\u6269\u5c55\u4e91\u4e0a Hadoop \u5e73\u53f0\u7684\u5b58\u50a8\u80fd\u529b\u3002 \u672c\u5de5\u5177\u7528\u4e8e\u628a Hive \u6570\u636e\u6839\u636e\u5206\u533a\u952e\u89c4\u5219\u7b5b\u9009\uff0c\u5728 HDFS \u548c OSS \u4e4b\u95f4\u8f6c\u79fb\u5206\u533a\u3002","title":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS"},{"location":"backup/oss/jindotable/table_moveto/#_1","text":"","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/oss/jindotable/table_moveto/#jindosdk","text":"\u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 JindoSDK + \u963f\u91cc\u4e91 OSS \u5feb\u901f\u5165\u95e8 \u3002","title":"\u5df2\u90e8\u7f72 JindoSDK"},{"location":"backup/oss/jindotable/table_moveto/#hadoop-hive","text":"\u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e","title":"\u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883"},{"location":"backup/oss/jindotable/table_moveto/#moveto-hdfs","text":"\u5728 Hadoop \u914d\u7f6e\u6587\u4ef6 core-site.xml \u6216 hdfs-site.xml \uff08\u4efb\u4e00\u5373\u53ef\uff0c\u5728 $HADOOP_CONF_DIR \u76ee\u5f55\u4e0b\uff09\u65b0\u589e\u914d\u7f6e\u9879\uff1a jindotable.moveto.tablelock.base.dir \u8be5\u914d\u7f6e\u7684\u503c\u5e94\u6307\u5411\u4e00\u4e2a HDFS \u76ee\u5f55\uff0c\u76ee\u7684\u662f\u5b58\u653e MoveTo \u5de5\u5177\u5728\u8fd0\u884c\u65f6\u81ea\u52a8\u521b\u5efa\u7684\u9501\u6587\u4ef6\u3002\u9700\u786e\u4fdd\u8be5\u76ee\u5f55\u53ea\u4f1a\u88ab MoveTo \u5de5\u5177\u8bbf\u95ee\uff0c\u5e76\u4e14\u6709\u8bbf\u95ee\u6743\u9650\u3002\u5982\u679c\u4e0d\u914d\u7f6e\uff0c\u5219\u4f7f\u7528\u7f3a\u7701\u503c hdfs:///tmp/jindotable-lock/ \uff0c\u65e0\u6743\u9650\u5219\u62a5\u9519\u3002","title":"\u914d\u7f6e MoveTo \u5de5\u5177\u5728 HDFS \u4e0b\u7684\u9501\u76ee\u5f55\uff1a"},{"location":"backup/oss/jindotable/table_moveto/#_2","text":"","title":"\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/oss/jindotable/table_moveto/#_3","text":"\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help moveTo","title":"\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f"},{"location":"backup/oss/jindotable/table_moveto/#_4","text":"jindotable -moveTo -t <dbName.tableName> -d <destination path> [-c \"<condition>\" | -fullTable] [-b/-before <before days>] [-p/-parallel <parallelism>] [-s/-storagePolicy <OSS storage policy>] [-o/-overWrite] [-r/-removeSource] [-skipTrash] [-e/-explain] [-q/-queue <yarn queue>] [-w/-workingDir <working directory>][-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u8981\u79fb\u52a8\u7684\u8868\u3002 \u662f -d <destination path> \u76ee\u6807\u8def\u5f84\uff0c\u4e3a\u8868\u7ea7\u522b\u7684\u8def\u5f84\uff0c\u5206\u533a\u8def\u5f84\u4f1a\u5728\u8fd9\u4e2a\u8def\u5f84\u4e0b\u81ea\u52a8\u521b\u5efa\u3002 \u662f -c \"<condition>\" / -fullTable \u5206\u533a\u8fc7\u6ee4\u6761\u4ef6\u8868\u8fbe\u5f0f\uff0c\u652f\u6301\u57fa\u672c\u8fd0\u7b97\u7b26\uff0c\u4e0d\u652f\u6301udf\u3002 \u5426 -b/before <before days> \u6839\u636e\u5206\u533a\u521b\u5efa\u65f6\u95f4\uff0c\u521b\u5efa\u65f6\u95f4\u8d85\u8fc7\u7ed9\u5b9a\u5929\u6570\u7684\u5206\u533a\u624d\u8fdb\u884c\u79fb\u52a8\u3002 \u5426 -p/-parallel <parallelism> \u6574\u4e2amoveTo\u4efb\u52a1\u7684\u6700\u5927task\u5e76\u53d1\u5ea6\uff0c\u9ed8\u8ba4\u4e3a1\u3002 \u5426 -s/-storagePolicy <OSS storage policy> \u62f7\u8d1d\u81f3 OSS \u65f6\u6570\u636e\u6587\u4ef6\u7684\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301 Standard\uff08\u9ed8\u8ba4\uff09\u3001IA\u3001Archive\u3001ColdArchive\u3002 \u5426 -o/-overWrite \u662f\u5426\u8986\u76d6\u6700\u7ec8\u76ee\u5f55\u3002\u5206\u533a\u7ea7\u522b\u8986\u76d6\uff0c\u4e0d\u4f1a\u8986\u76d6\u672c\u6b21\u79fb\u52a8\u4e0d\u6d89\u53ca\u7684\u5206\u533a\u3002 \u5426 -r/-removeSource \u662f\u5426\u5728\u79fb\u52a8\u5b8c\u6210\u540e\u5220\u9664\u6e90\u8def\u5f84\u3002 \u5426 -skipTrash \u5982\u679c\u5220\u9664\u6e90\u8def\u5f84\uff0c\u662f\u5426\u8df3\u8fc7Trash\u3002 \u5426 -e/-explain \u5982\u679c\u6307\u5b9aexplain\u6a21\u5f0f\uff0c\u4e0d\u4f1a\u89e6\u53d1\u5b9e\u9645\u64cd\u4f5c\uff0c\u4ec5\u6253\u5370\u4f1a\u540c\u6b65\u7684\u5206\u533a\u3002 \u5426 -q/-queue <yarn queue \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684yarn\u961f\u5217\u3002 \u5426 -w/-workingDir \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684\u5de5\u4f5c\u4e34\u65f6\u76ee\u5f55\u3002 \u5426 -l/-logDir <log directory> \u672c\u5730\u65e5\u5fd7\u76ee\u5f55\uff0c\u9ed8\u8ba4\u4e3a /tmp/<current user>/ \u5426","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"backup/oss/jindotable/table_moveto/#_5","text":"\u6709\u4e00\u4e2a HDFS \u4e0a\u7684 Hive \u5206\u533a\u8868\uff0c\u5982\u4e0b\u56fe\u6240\u793a\uff1a \u60f3\u628a bbb \u548c ccc \u5206\u533a\u79fb\u52a8\u5230 OSS\uff0c\u5148\u7528 explain \u6a21\u5f0f\u770b\u770b\u4f1a\u79fb\u52a8\u7684\u5206\u533a\u662f\u5426\u7b26\u5408\u9884\u671f\uff0c\u53c2\u6570\u4e3a -e \u6216 -explain\uff1a \u53bb\u6389\u53c2\u6570 -e\uff0c\u771f\u6b63\u79fb\u52a8\u5206\u533a\uff1a \u6267\u884c\u5b8c\u6210\u540e\uff0c\u68c0\u67e5\u6570\u636e\uff0c\u6570\u636e\u5df2\u7ecf\u5728 OSS\uff1a \u518d\u79fb\u56de HDFS\uff0c\u7ed3\u679c\u5931\u8d25\u4e86\uff0c\u4f1a\u663e\u793a\u5931\u8d25\u539f\u56e0\uff0c\u539f\u6765\u662f\u76ee\u6807\u76ee\u5f55\u975e\u7a7a\uff0c\u63d0\u793a\u53ef\u4ee5\u4f7f\u7528 -overWrite \u6e05\u7a7a\u76ee\u6807\u76ee\u5f55\uff1a \u4f7f\u7528 -overWrite \u5f3a\u5236\u8986\u76d6\uff1a","title":"\u4f7f\u7528\u793a\u4f8b"},{"location":"backup/oss/jindotable/table_moveto/#_6","text":"\u4e3a\u4e86\u5c3d\u53ef\u80fd\u4fdd\u8bc1\u6570\u636e\u5b89\u5168\uff0c\u9632\u6b62\u6c61\u67d3\uff0c\u8be5\u547d\u4ee4\u4f1a\u81ea\u52a8\u68c0\u67e5\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u786e\u4fdd\u4e0d\u5b58\u5728\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u76ee\u5f55\u62f7\u8d1d\u3002 \u5982\u679c\u65e0\u6cd5\u6392\u9664\uff0c\u5219\u8be5\u8868\u6216\u5206\u533a\u7684\u62f7\u8d1d\u547d\u4ee4\u5c06\u6267\u884c\u5931\u8d25\u3002 \u6b64\u65f6\uff0c\u9700\u8981\u4e3b\u52a8\u5e72\u9884\uff0c\u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u6b63\u5728\u6267\u884c\u62f7\u8d1d\u4e4b\u540e\uff0c\u6e05\u7406\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u5219\u53ef\u4ee5\u91cd\u65b0\u6267\u884c\u8be5\u547d\u4ee4\u3002 \u5bf9\u4e8e\u975e\u5206\u533a\u8868\u7684\u62f7\u8d1d\u6216\u8fc1\u79fb\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u5373\u4e3a\u8868\u7ea7\u522b\u76ee\u5f55\uff1b\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u4e3a\u5f85\u62f7\u8d1d\u6216\u79fb\u52a8\u5206\u533a\u7684\u5206\u533a\u7ea7\u76ee\u5f55\uff0c \u53ea\u9700\u5bf9\u5f85\u62f7\u8d1d\u7684\u5206\u533a\u8fdb\u884c\u6e05\u7406\u3002","title":"\u5f02\u5e38\u72b6\u6001\u5904\u7406"},{"location":"backup/oss/jindotable/table_moveto/#_7","text":"\u5982\u679c\u8be5\u547d\u4ee4\u88ab\u5f02\u5e38\u4e2d\u6b62\uff0c\u5219\u53ef\u80fd\u51fa\u73b0\u9700\u8981\u4e3b\u52a8\u5e72\u9884\u7684\u60c5\u51b5\u3002\u6b64\u65f6\u62f7\u8d1d\u8fd8\u672a\u5b8c\u6210\uff0c\u6e90\u6570\u636e\u4e0e\u8868\u7684\u5143\u4fe1\u606f\u5747\u672a\u6539\u53d8\uff0c \u6570\u636e\u4ecd\u5904\u4e8e\u5b89\u5168\u72b6\u6001\u3002\u5e38\u89c1\u7684\u5f02\u5e38\u4e2d\u6b62\u6709\u4e24\u79cd\u60c5\u51b5\uff1a * \u7528\u6237\u5728\u547d\u4ee4\u5c1a\u672a\u7ed3\u675f\u65f6\uff0c\u4e3b\u52a8\u6740\u6389\u547d\u4ee4\u8fdb\u7a0b * \u7531\u4e8e\u5185\u5b58\u6ea2\u51fa\u7b49\u5f02\u5e38\uff0c\u8fdb\u7a0b\u5f02\u5e38\u4e2d\u6b62 \u5982\u679c\u62f7\u8d1d\u8868\u6216\u5206\u533a\u65f6\u62f7\u8d1d\u5931\u8d25\u5e76\u663e\u793a Conflicts found \uff0c\u5219\u5c5e\u4e8e\u5f02\u5e38\u72b6\u6001\uff0c\u9700\u4e3b\u52a8\u5e72\u9884\u3002","title":"\u51fa\u73b0\u539f\u56e0"},{"location":"backup/oss/jindotable/table_moveto/#_8","text":"\u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u7684\u76ee\u6807\u8def\u5f84\u62f7\u8d1d\u6570\u636e\u3002\u7279\u522b\u662f\uff0c\u786e\u4fdd\u6ca1\u6709 DistCp/JindoDistCp \u7b49\u5206\u5e03\u5f0f\u62f7\u8d1d\u547d\u4ee4\u3002 \u5220\u9664\u76ee\u7684\u5730\u76ee\u5f55\u3002\u5bf9\u4e8e\u975e\u5206\u533a\u8868\uff0c\u5220\u9664\u8868\u4e00\u7ea7\u76ee\u5f55\u3002\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u5220\u9664\u5b58\u5728\u51b2\u7a81\u7684\u5206\u533a\u7ea7\u76ee\u5f55\u3002\u5207\u52ff\u5220\u9664\u6e90\u76ee\u5f55\u3002","title":"\u5904\u7406\u65b9\u6cd5"},{"location":"backup/oss/jindotable/table_shell/","text":"\u4f7f\u7528 JindoTable \u547d\u4ee4\u5b9e\u73b0 Hive \u8868\u548c\u5206\u533a\u7684\u6570\u636e\u6cbb\u7406 \u00a7 JindoTable \u63d0\u4f9b\u591a\u4e2a\u547d\u4ee4\u884c\u547d\u4ee4\uff0c\u53ef\u4ee5\u4fbf\u6377\u67e5\u8be2 Hive \u8868\u548c\u5206\u533a\u7684\u57fa\u672c\u4fe1\u606f\u6216\u7a7a\u95f4\u5360\u7528\uff0c \u6216\u8005\u5b9e\u73b0\u6570\u636e\u7684\u8fc1\u79fb\u3001\u5f52\u6863\u3002\u672c\u6587\u4ecb\u7ecd\u8fd9\u4e9b\u547d\u4ee4\u884c\u547d\u4ee4\u7684\u7528\u6cd5\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u90e8\u7f72 JindoSDK \u00a7 \u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8 \u3002 \u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883 \u00a7 \u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e \u4f7f\u7528\u8bf4\u660e \u00a7 \u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f \u00a7 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u67e5\u770b JindoTable \u652f\u6301\u7684\u6240\u6709\u547d\u4ee4\uff1a jindotable -help \u5982\u679c\u60f3\u67e5\u770b\u547d\u4ee4\u7684\u5177\u4f53\u7528\u6cd5\uff0c\u6267\u884c\uff1a jindotable -help <cmd> \u5176\u4e2d\uff0c <cmd> \u53ef\u4ee5\u662f showTable \u3001 moveTo \u3001 archiveTable \u7b49\u3002 showTable \u547d\u4ee4 \u00a7 jindotable -showTable -t <dbName.tableName> \u5176\u4e2d <dbName.tableName> \u4e3a\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a \u5e93\u540d.\u8868\u540d \u529f\u80fd\uff1a\u5982\u679c\u662f\u5206\u533a\u8868\uff0c\u5219\u5c55\u793a\u6240\u6709\u5206\u533a\uff1b\u5982\u679c\u662f\u975e\u5206\u533a\u8868\uff0c\u5219\u8fd4\u56de\u8868\u7684\u5b58\u50a8\u60c5\u51b5\u3002 \u793a\u4f8b\uff1a\u5c55\u793a db1.t1 \u5206\u533a\u8868\u7684\u6240\u6709\u5206\u533a\u3002 jindo table -showTable -t db1.t1 showPartition \u547d\u4ee4 \u00a7 jindotable -showPartition -t <dbName.tableName> -p <partitionSpec> \u5176\u4e2d <dbName.tableName> \u4e3a\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a \u5e93\u540d.\u8868\u540d \uff1b <partitionSpec> \u4e3a\u5206\u533a\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a col1=v1,col2=v2 \u529f\u80fd\uff1a\u8fd4\u56de\u5206\u533a\u7684\u5b58\u50a8\u60c5\u51b5\u3002 \u793a\u4f8b\uff1a\u8fd4\u56de\u5206\u533a\u8868 db1.t1 \u5728 2020-10-12 \u65e5\u7684\u5b58\u50a8\u60c5\u51b5\u3002 jindo table -showPartition -t db1.t1 -p date=2020-10-12 listTables \u547d\u4ee4 \u00a7 jindotable -listTables [-db] <dbName> \u5176\u4e2d <dbName> \u4e3a\u6570\u636e\u5e93\u540d\u79f0\uff0c\u53ef\u7701\u7565\uff0c\u7701\u7565\u5219\u4e3a\u9ed8\u8ba4\u6570\u636e\u5e93\u3002 \u529f\u80fd\uff1a\u5c55\u793a\u6307\u5b9a\u6570\u636e\u5e93\u4e2d\u7684\u6240\u6709\u8868\u3002\u4e0d\u6307\u5b9a -db \u65f6\u9ed8\u8ba4\u5c55\u793a default \u5e93\u4e2d\u7684\u8868\u3002 \u793a\u4f8b\uff1a\u5217\u51fa\u6570\u636e\u5e93 db1 \u4e2d\u7684\u8868\u3002 jindo table -listTables -db db1 moveTo \u547d\u4ee4 \u00a7 \u8bf7\u53c2\u8003 \u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1 archiveTable/unarchiveTable \u547d\u4ee4 \u00a7 \u8bf7\u53c2\u8003 JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a cacheTable/uncacheTable \u547d\u4ee4 \u00a7 \u4ec5\u652f\u6301\u914d\u5907 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7684\u573a\u666f\uff0c\u8bf7\u53c2\u8003 \u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e","title":"\u4f7f\u7528 JindoTable \u547d\u4ee4\u5b9e\u73b0 Hive \u8868\u548c\u5206\u533a\u7684\u6570\u636e\u6cbb\u7406"},{"location":"backup/oss/jindotable/table_shell/#jindotable-hive","text":"JindoTable \u63d0\u4f9b\u591a\u4e2a\u547d\u4ee4\u884c\u547d\u4ee4\uff0c\u53ef\u4ee5\u4fbf\u6377\u67e5\u8be2 Hive \u8868\u548c\u5206\u533a\u7684\u57fa\u672c\u4fe1\u606f\u6216\u7a7a\u95f4\u5360\u7528\uff0c \u6216\u8005\u5b9e\u73b0\u6570\u636e\u7684\u8fc1\u79fb\u3001\u5f52\u6863\u3002\u672c\u6587\u4ecb\u7ecd\u8fd9\u4e9b\u547d\u4ee4\u884c\u547d\u4ee4\u7684\u7528\u6cd5\u3002","title":"\u4f7f\u7528 JindoTable \u547d\u4ee4\u5b9e\u73b0 Hive \u8868\u548c\u5206\u533a\u7684\u6570\u636e\u6cbb\u7406"},{"location":"backup/oss/jindotable/table_shell/#_1","text":"","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"backup/oss/jindotable/table_shell/#jindosdk","text":"\u5173\u4e8e\u5982\u4f55\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u901f\u5165\u95e8 \u3002","title":"\u5df2\u90e8\u7f72 JindoSDK"},{"location":"backup/oss/jindotable/table_shell/#hadoop-hive","text":"\u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e","title":"\u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883"},{"location":"backup/oss/jindotable/table_shell/#_2","text":"","title":"\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/oss/jindotable/table_shell/#_3","text":"\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u67e5\u770b JindoTable \u652f\u6301\u7684\u6240\u6709\u547d\u4ee4\uff1a jindotable -help \u5982\u679c\u60f3\u67e5\u770b\u547d\u4ee4\u7684\u5177\u4f53\u7528\u6cd5\uff0c\u6267\u884c\uff1a jindotable -help <cmd> \u5176\u4e2d\uff0c <cmd> \u53ef\u4ee5\u662f showTable \u3001 moveTo \u3001 archiveTable \u7b49\u3002","title":"\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f"},{"location":"backup/oss/jindotable/table_shell/#showtable","text":"jindotable -showTable -t <dbName.tableName> \u5176\u4e2d <dbName.tableName> \u4e3a\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a \u5e93\u540d.\u8868\u540d \u529f\u80fd\uff1a\u5982\u679c\u662f\u5206\u533a\u8868\uff0c\u5219\u5c55\u793a\u6240\u6709\u5206\u533a\uff1b\u5982\u679c\u662f\u975e\u5206\u533a\u8868\uff0c\u5219\u8fd4\u56de\u8868\u7684\u5b58\u50a8\u60c5\u51b5\u3002 \u793a\u4f8b\uff1a\u5c55\u793a db1.t1 \u5206\u533a\u8868\u7684\u6240\u6709\u5206\u533a\u3002 jindo table -showTable -t db1.t1","title":"showTable \u547d\u4ee4"},{"location":"backup/oss/jindotable/table_shell/#showpartition","text":"jindotable -showPartition -t <dbName.tableName> -p <partitionSpec> \u5176\u4e2d <dbName.tableName> \u4e3a\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a \u5e93\u540d.\u8868\u540d \uff1b <partitionSpec> \u4e3a\u5206\u533a\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a col1=v1,col2=v2 \u529f\u80fd\uff1a\u8fd4\u56de\u5206\u533a\u7684\u5b58\u50a8\u60c5\u51b5\u3002 \u793a\u4f8b\uff1a\u8fd4\u56de\u5206\u533a\u8868 db1.t1 \u5728 2020-10-12 \u65e5\u7684\u5b58\u50a8\u60c5\u51b5\u3002 jindo table -showPartition -t db1.t1 -p date=2020-10-12","title":"showPartition \u547d\u4ee4"},{"location":"backup/oss/jindotable/table_shell/#listtables","text":"jindotable -listTables [-db] <dbName> \u5176\u4e2d <dbName> \u4e3a\u6570\u636e\u5e93\u540d\u79f0\uff0c\u53ef\u7701\u7565\uff0c\u7701\u7565\u5219\u4e3a\u9ed8\u8ba4\u6570\u636e\u5e93\u3002 \u529f\u80fd\uff1a\u5c55\u793a\u6307\u5b9a\u6570\u636e\u5e93\u4e2d\u7684\u6240\u6709\u8868\u3002\u4e0d\u6307\u5b9a -db \u65f6\u9ed8\u8ba4\u5c55\u793a default \u5e93\u4e2d\u7684\u8868\u3002 \u793a\u4f8b\uff1a\u5217\u51fa\u6570\u636e\u5e93 db1 \u4e2d\u7684\u8868\u3002 jindo table -listTables -db db1","title":"listTables \u547d\u4ee4"},{"location":"backup/oss/jindotable/table_shell/#moveto","text":"\u8bf7\u53c2\u8003 \u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1","title":"moveTo \u547d\u4ee4"},{"location":"backup/oss/jindotable/table_shell/#archivetableunarchivetable","text":"\u8bf7\u53c2\u8003 JindoTable \u5f52\u6863\u548c\u89e3\u51bb OSS \u4e0a\u7684\u8868\u6216\u5206\u533a","title":"archiveTable/unarchiveTable \u547d\u4ee4"},{"location":"backup/oss/jindotable/table_shell/#cachetableuncachetable","text":"\u4ec5\u652f\u6301\u914d\u5907 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u7684\u573a\u666f\uff0c\u8bf7\u53c2\u8003 \u4f7f\u7528 JindoTable \u7f13\u5b58 Hive \u8868\u6216\u5206\u533a\u7684\u6570\u636e","title":"cacheTable/uncacheTable \u547d\u4ee4"},{"location":"backup/oss/presto/jindosdk_on_presto/","text":"Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS \u4e0a\u7684\u6570\u636e \u00a7 Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002 \u6b65\u9aa4 \u00a7 1. \u5b89\u88c5 jar \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/ 2. \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> 3. \u914d\u7f6e OSS Access Key \u00a7 \u5c06 OSS \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 4. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 \u4f7f\u7528\u793a\u4f8b \u00a7 \u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='oss://<bucket>/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS \u4e0a\u7684\u6570\u636e"},{"location":"backup/oss/presto/jindosdk_on_presto/#presto-jindosdk-oss","text":"Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002","title":"Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS \u4e0a\u7684\u6570\u636e"},{"location":"backup/oss/presto/jindosdk_on_presto/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/oss/presto/jindosdk_on_presto/#1-jar","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/","title":"1. \u5b89\u88c5 jar \u5305"},{"location":"backup/oss/presto/jindosdk_on_presto/#2-jindosdk-oss","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration>","title":"2. \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b"},{"location":"backup/oss/presto/jindosdk_on_presto/#3-oss-access-key","text":"\u5c06 OSS \u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002","title":"3. \u914d\u7f6e OSS Access Key"},{"location":"backup/oss/presto/jindosdk_on_presto/#4-presto","text":"","title":"4. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"backup/oss/presto/jindosdk_on_presto/#_2","text":"\u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='oss://<bucket>/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"\u4f7f\u7528\u793a\u4f8b"},{"location":"backup/oss/security/jindosdk_credential_provider_bucket_oss/","text":"JindoSDK OSS Credential Provider \u6309 OSS bucket \u914d\u7f6e\u4f7f\u7528\u8bf4\u660e \u00a7 \u6309\u7167 bucket \u914d\u7f6e Credential Provider \u00a7 <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleAliyunCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.AliyunCredentialsProvider\u7684\u5b9e\u73b0\u7c7b\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u8bf4\u660e XXX \u4e3a OSS bucket \u540d\u79f0\u3002 Credential Provider \u7c7b\u578b \u00a7 1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS bucket\u7684AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS bucket\u7684AccessKey Secret</value> </property> <property> <name>fs.oss.bucket.XXX.securityToken</name> <value>OSS bucket\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration> 2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS bucket\u7684AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS bucket\u7684AccessKey Secret</value> </property> </configuration> 3. EnvironmentVariableCredentialsProvider \u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6 AccessKey\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS bucket\u7684AccessKey Id OSS_ACCESS_KEY_SECRET OSS bucket\u7684AccessKey Secret OSS_SECURITY_TOKEN OSS bucket\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548Token\u65f6\u9700\u8981\u3002 4. CommonCredentialsProvider\u4e3a\u901a\u7528\u914d\u7f6e\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS bucket \u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS bucket \u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS bucket \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration>","title":"JindoSDK OSS Credential Provider \u6309 OSS bucket \u914d\u7f6e\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/oss/security/jindosdk_credential_provider_bucket_oss/#jindosdk-oss-credential-provider-oss-bucket","text":"","title":"JindoSDK OSS Credential Provider \u6309 OSS bucket \u914d\u7f6e\u4f7f\u7528\u8bf4\u660e"},{"location":"backup/oss/security/jindosdk_credential_provider_bucket_oss/#bucket-credential-provider","text":"<configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleAliyunCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.AliyunCredentialsProvider\u7684\u5b9e\u73b0\u7c7b\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u8bf4\u660e XXX \u4e3a OSS bucket \u540d\u79f0\u3002","title":"\u6309\u7167 bucket \u914d\u7f6e Credential Provider"},{"location":"backup/oss/security/jindosdk_credential_provider_bucket_oss/#credential-provider","text":"","title":"Credential Provider \u7c7b\u578b"},{"location":"backup/oss/security/jindosdk_credential_provider_bucket_oss/#1-temporarycredentialsprovider-accesskey-securitytoken-oss","text":"\u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS bucket\u7684AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS bucket\u7684AccessKey Secret</value> </property> <property> <name>fs.oss.bucket.XXX.securityToken</name> <value>OSS bucket\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration>","title":"1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002"},{"location":"backup/oss/security/jindosdk_credential_provider_bucket_oss/#2-simplecredentialsprovider-accesskey-oss","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS bucket\u7684AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS bucket\u7684AccessKey Secret</value> </property> </configuration>","title":"2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002"},{"location":"backup/oss/security/jindosdk_credential_provider_bucket_oss/#3-environmentvariablecredentialsprovider-accesskey","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS bucket\u7684AccessKey Id OSS_ACCESS_KEY_SECRET OSS bucket\u7684AccessKey Secret OSS_SECURITY_TOKEN OSS bucket\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548Token\u65f6\u9700\u8981\u3002","title":"3. EnvironmentVariableCredentialsProvider \u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6 AccessKey\u3002"},{"location":"backup/oss/security/jindosdk_credential_provider_bucket_oss/#4-commoncredentialsprovider","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS bucket \u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS bucket \u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS bucket \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration>","title":"4. CommonCredentialsProvider\u4e3a\u901a\u7528\u914d\u7f6e\u3002"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/","text":"JindoSDK OSS Credential Provider \u914d\u7f6e \u00a7 \u57fa\u672c\u914d\u7f6e\u65b9\u5f0f \u00a7 \u60a8\u53ef\u4ee5\u5c06 OSS \u7684 Access Key ID \u3001 Access Secret \u3001 Endpoint \u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \uff0c\u914d\u7f6e\u9879\u5982\u4e0b\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> \u4f7f\u7528 Hadoop Credential Providers \u5b58\u50a8 AccessKey \u4fe1\u606f \u00a7 \u4e0a\u9762\u7684 fs.oss.accessKeyId \u548c fs.oss.accessKeySecret \u5c06\u4f1a\u660e\u6587\u914d\u7f6e\u5728 core-site.xml \u4e2d\uff0c\u53ef\u4ee5\u5c06\u5176\u4ee5\u52a0\u5bc6\u5bf9\u65b9\u5f0f\u5b58\u50a8\u81f3 Hadoop Credential Providers\u6587\u4ef6\u4e2d\u3002 \u4f7f\u7528Hadoop\u63d0\u4f9b\u7684\u547d\u4ee4\uff0c\u5b58\u50a8AccessKey\u548cSecurityToken\u4fe1\u606f\u81f3Credential\u6587\u4ef6\u4e2d\u3002\u547d\u4ee4\u683c\u5f0f\u5982\u4e0b\u3002 hadoop credential <subcommand> [options] \u4f8b\u5982\uff0c\u5b58\u50a8AccessKey\u548cToken\u4fe1\u606f\u81f3JECKS\u6587\u4ef6\u4e2d\uff0c\u9664\u4e86\u4f7f\u7528\u6587\u4ef6\u6743\u9650\u4fdd\u62a4\u8be5\u6587\u4ef6\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u6307\u5b9a\u5bc6\u7801\u52a0\u5bc6\u5b58\u50a8\u4fe1\u606f\uff0c\u5982\u679c\u4e0d\u6307\u5b9a\u5bc6\u7801\u5219\u4f7f\u7528\u9ed8\u8ba4\u5b57\u7b26\u4e32\u52a0\u5bc6\u3002 hadoop credential create fs.oss.accessKeyId -value AAA -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.accessKeySecret -value BBB -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.securityToken -value CCC -provider jceks://file/root/oss.jceks \u751f\u6210Credential\u6587\u4ef6\u540e\uff0c\u60a8\u9700\u8981\u914d\u7f6e\u4e0b\u9762\u7684\u53c2\u6570\u6765\u6307\u5b9aProvider\u7684\u7c7b\u578b\u548c\u4f4d\u7f6e\u3002 <configuration> <property> <name>fs.oss.security.credential.provider.path</name> <value>jceks://file/root/oss.jceks</value> <description>\u914d\u7f6e\u5b58\u50a8AK\u7684Credential\u6587\u4ef6\u3002\u4f8b\u5982\uff0cjceks://file/${user.home}/oss.jceks\u4e3aHOME\u4e0b\u7684oss.jceks\u6587\u4ef6</description> </property> </configuration> \u4f7f\u7528 JindoSDK OSS Credential Provider \u00a7 \u9ed8\u8ba4\u4f1a\u914d\u7f6e SimpleCredentialsProvider\uff0cEnvironmentVariableCredentialsProvider\uff0cCommonCredentialsProvider \u8fd9\u4e09\u4e2aCredential Provider, \u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002 <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.JindoCredentialsProvider\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u60a8\u53ef\u4ee5\u6839\u636e\u60c5\u51b5\uff0c\u9009\u62e9\u4e0d\u540c\u7684Credential Provider\uff0c\u652f\u6301\u5982\u4e0bProvider\uff1a 1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS\u7684AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS\u7684AccessKey Secret</value> </property> <property> <name>fs.oss.securityToken</name> <value>OSS\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration> 2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS \u7684AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS \u7684AccessKey Secret</value> </property> </configuration> 3. EnvironmentVariableCredentialsProvider\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6AK\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS \u7684 AccessKey Id OSS_ACCESS_KEY_SECRET OSS \u7684 AccessKey Secret OSS_SECURITY_TOKEN OSS \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002 4. CommonCredentialsProvider \u4e3a\u901a\u7528\u914d\u7f6e\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS \u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS \u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS \u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration> 5. CustomCredentialsProvider \u5bf9\u63a5\u5b9a\u5236\u7684\u514d\u5bc6\u670d\u52a1\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CustomCredentialsProvider</value> </property> </configuration> \u914d\u7f6e\u514d\u5bc6\u670d\u52a1\u5730\u5740 <configuration> <property> <name>aliyun.oss.provider.url</name> <value>\u514d\u5bc6\u670d\u52a1\u5730\u5740</value> </property> </configuration> aliyun.oss.provider.url \u652f\u6301 http(s) \u534f\u8bae \u548c secrets \u534f\u8bae\uff1a a. http(s) \u534f\u8bae http(s) \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a http://localhost:1234/sts , http \u514d\u5bc6\u534f\u8bae\u8981\u6c42\u8fd4\u56de\u7ed3\u679c\u4e3a Json \u683c\u5f0f\uff0c\u5982\u679c\u60a8\u9700\u8981\u5bf9\u63a5\u60a8\u7684 http \u514d\u5bc6\u670d\u52a1\uff0c\u514d\u5bc6\u670d\u52a1\u7684\u8bbe\u8ba1\u53c2\u8003 \u300a\u901a\u8fc7API\u4f7f\u7528\u5b9e\u4f8bRAM\u89d2\u8272#\u4e34\u65f6\u6388\u6743Token\u300b { \"AccessKeyId\" : \"XXXXXXXXX\", \"AccessKeySecret\" : \"XXXXXXXXX\", \"Expiration\" : \"2020-11-01T05:20:01Z\", \"SecurityToken\" : \"XXXXXXXXX\", \"LastUpdated\" : \"2020-10-31T23:20:01Z\", \"Code\" : \"Success\" } b. Secrets \u534f\u8bae Secrets \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a secrets:///local_path_prefix \uff0c\u5e38\u89c1\u4f7f\u7528\u4e8e k8s \u573a\u666f \uff0c \u5176\u4e2d local_path_prefix \u4e3a\u8def\u5f84\u524d\u7f00\uff0c\u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOssAccessKeyId /secret/JindoOssAccessKeySecret /secret/JindoOssSecurityToken \u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss/ \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOss/AccessKeyId /secret/JindoOss/AccessKeySecret /secret/JindoOss/SecurityToken JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 OSS bucket \u914d\u7f6e\u4e0d\u540c\u7684 Credential Provider \u00a7 \u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u6309 OSS bucket \u914d\u7f6e\u4f7f\u7528\u8bf4\u660e \u3002","title":"JindoSDK OSS Credential Provider \u914d\u7f6e"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/#jindosdk-oss-credential-provider","text":"","title":"JindoSDK OSS Credential Provider \u914d\u7f6e"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/#_1","text":"\u60a8\u53ef\u4ee5\u5c06 OSS \u7684 Access Key ID \u3001 Access Secret \u3001 Endpoint \u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \uff0c\u914d\u7f6e\u9879\u5982\u4e0b\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration>","title":"\u57fa\u672c\u914d\u7f6e\u65b9\u5f0f"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/#hadoop-credential-providers-accesskey","text":"\u4e0a\u9762\u7684 fs.oss.accessKeyId \u548c fs.oss.accessKeySecret \u5c06\u4f1a\u660e\u6587\u914d\u7f6e\u5728 core-site.xml \u4e2d\uff0c\u53ef\u4ee5\u5c06\u5176\u4ee5\u52a0\u5bc6\u5bf9\u65b9\u5f0f\u5b58\u50a8\u81f3 Hadoop Credential Providers\u6587\u4ef6\u4e2d\u3002 \u4f7f\u7528Hadoop\u63d0\u4f9b\u7684\u547d\u4ee4\uff0c\u5b58\u50a8AccessKey\u548cSecurityToken\u4fe1\u606f\u81f3Credential\u6587\u4ef6\u4e2d\u3002\u547d\u4ee4\u683c\u5f0f\u5982\u4e0b\u3002 hadoop credential <subcommand> [options] \u4f8b\u5982\uff0c\u5b58\u50a8AccessKey\u548cToken\u4fe1\u606f\u81f3JECKS\u6587\u4ef6\u4e2d\uff0c\u9664\u4e86\u4f7f\u7528\u6587\u4ef6\u6743\u9650\u4fdd\u62a4\u8be5\u6587\u4ef6\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u6307\u5b9a\u5bc6\u7801\u52a0\u5bc6\u5b58\u50a8\u4fe1\u606f\uff0c\u5982\u679c\u4e0d\u6307\u5b9a\u5bc6\u7801\u5219\u4f7f\u7528\u9ed8\u8ba4\u5b57\u7b26\u4e32\u52a0\u5bc6\u3002 hadoop credential create fs.oss.accessKeyId -value AAA -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.accessKeySecret -value BBB -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.securityToken -value CCC -provider jceks://file/root/oss.jceks \u751f\u6210Credential\u6587\u4ef6\u540e\uff0c\u60a8\u9700\u8981\u914d\u7f6e\u4e0b\u9762\u7684\u53c2\u6570\u6765\u6307\u5b9aProvider\u7684\u7c7b\u578b\u548c\u4f4d\u7f6e\u3002 <configuration> <property> <name>fs.oss.security.credential.provider.path</name> <value>jceks://file/root/oss.jceks</value> <description>\u914d\u7f6e\u5b58\u50a8AK\u7684Credential\u6587\u4ef6\u3002\u4f8b\u5982\uff0cjceks://file/${user.home}/oss.jceks\u4e3aHOME\u4e0b\u7684oss.jceks\u6587\u4ef6</description> </property> </configuration>","title":"\u4f7f\u7528 Hadoop Credential Providers \u5b58\u50a8 AccessKey \u4fe1\u606f"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/#jindosdk-oss-credential-provider_1","text":"\u9ed8\u8ba4\u4f1a\u914d\u7f6e SimpleCredentialsProvider\uff0cEnvironmentVariableCredentialsProvider\uff0cCommonCredentialsProvider \u8fd9\u4e09\u4e2aCredential Provider, \u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002 <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.JindoCredentialsProvider\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u60a8\u53ef\u4ee5\u6839\u636e\u60c5\u51b5\uff0c\u9009\u62e9\u4e0d\u540c\u7684Credential Provider\uff0c\u652f\u6301\u5982\u4e0bProvider\uff1a","title":"\u4f7f\u7528 JindoSDK OSS Credential Provider"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/#1-temporarycredentialsprovider-accesskey-securitytoken-oss","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS\u7684AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS\u7684AccessKey Secret</value> </property> <property> <name>fs.oss.securityToken</name> <value>OSS\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration>","title":"1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/#2-simplecredentialsprovider-accesskey-oss","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS \u7684AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS \u7684AccessKey Secret</value> </property> </configuration>","title":"2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS \u7684\u60c5\u51b5\u3002"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/#3-environmentvariablecredentialsproviderak","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS \u7684 AccessKey Id OSS_ACCESS_KEY_SECRET OSS \u7684 AccessKey Secret OSS_SECURITY_TOKEN OSS \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002","title":"3. EnvironmentVariableCredentialsProvider\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6AK\u3002"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/#4-commoncredentialsprovider","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6eOSS AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS \u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS \u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS \u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration>","title":"4. CommonCredentialsProvider \u4e3a\u901a\u7528\u914d\u7f6e\u3002"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/#5-customcredentialsprovider","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CustomCredentialsProvider</value> </property> </configuration> \u914d\u7f6e\u514d\u5bc6\u670d\u52a1\u5730\u5740 <configuration> <property> <name>aliyun.oss.provider.url</name> <value>\u514d\u5bc6\u670d\u52a1\u5730\u5740</value> </property> </configuration> aliyun.oss.provider.url \u652f\u6301 http(s) \u534f\u8bae \u548c secrets \u534f\u8bae\uff1a a. http(s) \u534f\u8bae http(s) \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a http://localhost:1234/sts , http \u514d\u5bc6\u534f\u8bae\u8981\u6c42\u8fd4\u56de\u7ed3\u679c\u4e3a Json \u683c\u5f0f\uff0c\u5982\u679c\u60a8\u9700\u8981\u5bf9\u63a5\u60a8\u7684 http \u514d\u5bc6\u670d\u52a1\uff0c\u514d\u5bc6\u670d\u52a1\u7684\u8bbe\u8ba1\u53c2\u8003 \u300a\u901a\u8fc7API\u4f7f\u7528\u5b9e\u4f8bRAM\u89d2\u8272#\u4e34\u65f6\u6388\u6743Token\u300b { \"AccessKeyId\" : \"XXXXXXXXX\", \"AccessKeySecret\" : \"XXXXXXXXX\", \"Expiration\" : \"2020-11-01T05:20:01Z\", \"SecurityToken\" : \"XXXXXXXXX\", \"LastUpdated\" : \"2020-10-31T23:20:01Z\", \"Code\" : \"Success\" } b. Secrets \u534f\u8bae Secrets \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a secrets:///local_path_prefix \uff0c\u5e38\u89c1\u4f7f\u7528\u4e8e k8s \u573a\u666f \uff0c \u5176\u4e2d local_path_prefix \u4e3a\u8def\u5f84\u524d\u7f00\uff0c\u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOssAccessKeyId /secret/JindoOssAccessKeySecret /secret/JindoOssSecurityToken \u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss/ \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOss/AccessKeyId /secret/JindoOss/AccessKeySecret /secret/JindoOss/SecurityToken","title":"5. CustomCredentialsProvider \u5bf9\u63a5\u5b9a\u5236\u7684\u514d\u5bc6\u670d\u52a1\u3002"},{"location":"backup/oss/security/jindosdk_credential_provider_oss/#jindosdk-oss-bucket-credential-provider","text":"\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u6309 OSS bucket \u914d\u7f6e\u4f7f\u7528\u8bf4\u660e \u3002","title":"JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 OSS bucket \u914d\u7f6e\u4e0d\u540c\u7684 Credential Provider"},{"location":"backup/oss/spark/jindosdk_ide_spark/","text":"Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 \u00a7 \u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindosdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop with provided scope, your runtime should install hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.spark.sql.SparkSession; public class TestJindoSDK { public static void main(String[] args) throws Exception { SparkSession spark = SparkSession .builder() .config(\"spark.hadoop.fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\") .config(\"spark.hadoop.fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\") .config(\"spark.hadoop.fs.oss.accessKeyId\", \"xxx\") .config(\"spark.hadoop.fs.oss.accessKeySecret\", \"xxx\") // set accessKey, secret, endpoint and so on. .appName(\"TestJindoSDK\") .getOrCreate(); spark.read().parquet(\"oss://xxx\").count(); spark.stop(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency>","title":"Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5"},{"location":"backup/oss/spark/jindosdk_ide_spark/#spark-jindosdk-ide","text":"\u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>4.6.12</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindosdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop with provided scope, your runtime should install hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.spark.sql.SparkSession; public class TestJindoSDK { public static void main(String[] args) throws Exception { SparkSession spark = SparkSession .builder() .config(\"spark.hadoop.fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\") .config(\"spark.hadoop.fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\") .config(\"spark.hadoop.fs.oss.accessKeyId\", \"xxx\") .config(\"spark.hadoop.fs.oss.accessKeySecret\", \"xxx\") // set accessKey, secret, endpoint and so on. .appName(\"TestJindoSDK\") .getOrCreate(); spark.read().parquet(\"oss://xxx\").count(); spark.stop(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency>","title":"Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5"},{"location":"backup/oss/spark/jindosdk_on_spark/","text":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e \u00a7 JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411Hadoop/Spark\u751f\u6001\u7684OSS\u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684Hadoop FileSystem\u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fddSpark\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/ 3. \u914d\u7f6e JindoSDK \u00a7 \u5168\u5c40\u914d\u7f6e,\u4fee\u6539 Spark \u4f7f\u7528\u7684 core-site.xml \u00a7 \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002 \u4efb\u52a1\u7ea7\u522b\u914d\u7f6e \u00a7 \u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx --conf spark.hadoop.fs.oss.endpoint=oss-cn-xxx.aliyuncs.com 4. \u4f7f\u7528 Spark \u8bbf\u95ee OSS \u00a7 \u521b\u5efa\u8868 create table test_oss (c1 string) location \"oss://bucket/dir\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss; 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e"},{"location":"backup/oss/spark/jindosdk_on_spark/#spark-jindosdk-oss","text":"JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411Hadoop/Spark\u751f\u6001\u7684OSS\u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684Hadoop FileSystem\u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e"},{"location":"backup/oss/spark/jindosdk_on_spark/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"backup/oss/spark/jindosdk_on_spark/#1-spark-hadoop","text":"\u786e\u4fddSpark\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"backup/oss/spark/jindosdk_on_spark/#2-spark-classpath-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/","title":"2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK"},{"location":"backup/oss/spark/jindosdk_on_spark/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"backup/oss/spark/jindosdk_on_spark/#spark-core-sitexml","text":"\u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> <property> <name>fs.oss.endpoint</name> <!-- \u963f\u91cc\u4e91 ECS \u73af\u5883\u4e0b\u63a8\u8350\u4f7f\u7528\u5185\u7f51 OSS Endpoint\uff0c\u5373 oss-cn-xxx-internal.aliyuncs.com --> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS Credential Provider \u914d\u7f6e \u3002","title":"\u5168\u5c40\u914d\u7f6e,\u4fee\u6539 Spark \u4f7f\u7528\u7684 core-site.xml"},{"location":"backup/oss/spark/jindosdk_on_spark/#_2","text":"\u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx --conf spark.hadoop.fs.oss.endpoint=oss-cn-xxx.aliyuncs.com","title":"\u4efb\u52a1\u7ea7\u522b\u914d\u7f6e"},{"location":"backup/oss/spark/jindosdk_on_spark/#4-spark-oss","text":"\u521b\u5efa\u8868 create table test_oss (c1 string) location \"oss://bucket/dir\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss;","title":"4. \u4f7f\u7528 Spark \u8bbf\u95ee OSS"},{"location":"backup/oss/spark/jindosdk_on_spark/#5","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"backup/oss/sqoop/kitesdk_on_sqoop/","text":"Sqoop \u4f7f\u7528 Kite SDK \u8bfb\u5199 OSS \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 \u5728\u96c6\u7fa4\u4e0a\u6709\u5f00\u6e90\u7248\u672c Sqoop \u8f6f\u4ef6\uff0c\u7248\u672c\u4e0d\u4f4e\u4e8e 1.4.7\u3002 Sqoop \u4f9d\u8d56\u7684 Hadoop \u73af\u5883\u53ef\u8bbf\u95ee OSS\uff0c\u53ef\u53c2\u8003 Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u4f7f\u7528 Kite SDK \u8bfb\u5199 OSS \u00a7 Sqoop \u672c\u8eab\u5e76\u4e0d\u652f\u6301\u5bf9 OSS \u7684\u8bfb\u5199\uff0c\u9700\u8981\u4f7f\u7528\u7b2c\u4e09\u65b9 Kite SDK \u8fdb\u884c URI \u7684\u8f6c\u6362\u4ece\u800c\u8fdb\u884c\u6570\u636e\u4ea4\u4e92\u3002 \u6b65\u9aa4 \u00a7 1. \u5b89\u88c5 jar \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 jar \u5305 kite-data-oss-3.4.0.jar ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u5c06 sdk \u5305\u5b89\u88c5\u5230 Sqoop \u7684 classpath \u76ee\u5f55\u4e0b\u3002 cp ./kite-data-oss-3.4.0.jar <SQOOP_HOME>/lib/kite-data-oss-3.4.0.jar 2.\u6388\u4e88\u5bf9 JAR \u7684\u6743\u9650 \u00a7 sudo chmod 755 kite-data-oss-3.4.0.jar 3.\u4f7f\u7528 oss \u8fde\u63a5\u5668\u5bfc\u5165 mysql \u4e0a\u7684\u6570\u636e \u00a7 sqoop import --connect jdbc:mysql://<host>:<port>/database --username username --password password --table yourtable --target-dir \"oss://yourbucket/dir/\" --as-parquetfile -m 5","title":"Sqoop \u4f7f\u7528 Kite SDK \u8bfb\u5199 OSS"},{"location":"backup/oss/sqoop/kitesdk_on_sqoop/#sqoop-kite-sdk-oss","text":"","title":"Sqoop \u4f7f\u7528 Kite SDK \u8bfb\u5199 OSS"},{"location":"backup/oss/sqoop/kitesdk_on_sqoop/#_1","text":"\u5728\u96c6\u7fa4\u4e0a\u6709\u5f00\u6e90\u7248\u672c Sqoop \u8f6f\u4ef6\uff0c\u7248\u672c\u4e0d\u4f4e\u4e8e 1.4.7\u3002 Sqoop \u4f9d\u8d56\u7684 Hadoop \u73af\u5883\u53ef\u8bbf\u95ee OSS\uff0c\u53ef\u53c2\u8003 Hadoop \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS","title":"\u73af\u5883\u8981\u6c42"},{"location":"backup/oss/sqoop/kitesdk_on_sqoop/#kite-sdk-oss","text":"Sqoop \u672c\u8eab\u5e76\u4e0d\u652f\u6301\u5bf9 OSS \u7684\u8bfb\u5199\uff0c\u9700\u8981\u4f7f\u7528\u7b2c\u4e09\u65b9 Kite SDK \u8fdb\u884c URI \u7684\u8f6c\u6362\u4ece\u800c\u8fdb\u884c\u6570\u636e\u4ea4\u4e92\u3002","title":"\u4f7f\u7528 Kite SDK \u8bfb\u5199 OSS"},{"location":"backup/oss/sqoop/kitesdk_on_sqoop/#_2","text":"","title":"\u6b65\u9aa4"},{"location":"backup/oss/sqoop/kitesdk_on_sqoop/#1-jar","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 jar \u5305 kite-data-oss-3.4.0.jar ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u5c06 sdk \u5305\u5b89\u88c5\u5230 Sqoop \u7684 classpath \u76ee\u5f55\u4e0b\u3002 cp ./kite-data-oss-3.4.0.jar <SQOOP_HOME>/lib/kite-data-oss-3.4.0.jar","title":"1. \u5b89\u88c5 jar \u5305"},{"location":"backup/oss/sqoop/kitesdk_on_sqoop/#2-jar","text":"sudo chmod 755 kite-data-oss-3.4.0.jar","title":"2.\u6388\u4e88\u5bf9 JAR \u7684\u6743\u9650"},{"location":"backup/oss/sqoop/kitesdk_on_sqoop/#3-oss-mysql","text":"sqoop import --connect jdbc:mysql://<host>:<port>/database --username username --password password --table yourtable --target-dir \"oss://yourbucket/dir/\" --as-parquetfile -m 5","title":"3.\u4f7f\u7528 oss \u8fde\u63a5\u5668\u5bfc\u5165 mysql \u4e0a\u7684\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_aliyun_sls_example/","text":"\u4f7f\u7528\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u6536\u96c6\u670d\u52a1 \u00a7 JindoRuntime \u9ed8\u8ba4\u65e5\u5fd7\u6253\u5728\u5bb9\u5668 std \u8f93\u51fa\u91cc\u9762\uff0c\u53ef\u4ee5\u4f7f\u7528\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u6536\u96c6\u670d\u52a1\u5c06\u65e5\u5fd7\u6536\u96c6\u8d77\u6765\u3002 1\u3001ack \u96c6\u7fa4\u7ed1\u5b9a sls \u670d\u52a1 \u00a7 \u60a8\u53ef\u4ee5\u5728\u521b\u5efa\u96c6\u7fa4\u65f6\u7ed1\u5b9a sls \u670d\u52a1 \u4e5f\u53ef\u4ee5\u5728\u521b\u5efa\u96c6\u7fa4\u540e\u7ed1\u5b9a\u963f\u91cc\u4e91 sls \u670d\u52a1 \u9009\u62e9\u96c6\u7fa4\u4fe1\u606f > \u96c6\u7fa4\u8d44\u6e90 > \u65e5\u5fd7\u670d\u52a1 Project 2\u3001\u521b\u5efa JindoRuntime \u96c6\u7fa4 \u00a7 \u53ef\u53c2\u8003\u6587\u7ae0 \u52a0\u901fOSS\u4e0a\u6570\u636e \u521b\u5efa JindoRuntime \u96c6\u7fa4\u3002 3\u3001\u914d\u7f6e sls \u6536\u96c6 \u00a7 \u60a8\u53ef\u4ee5\u9009\u62e9\u5728 ack \u63a7\u5236\u53f0\u64cd\u4f5c\uff0c\u6216\u8005\u76f4\u63a5\u64cd\u4f5c\u96c6\u7fa4 K8S \u8d44\u6e90 3.1 ack \u63a7\u5236\u53f0\u64cd\u4f5c \u00a7 \u672c\u6587\u4ee5 JindoRuntime \u540d\u5b57\u4e3a test \u8fdb\u884c\u4e3e\u4f8b \u60a8\u53ef\u4ee5\u767b\u9646 ack \u63a7\u5236\u53f0 > \u5de5\u4f5c\u8d1f\u8f7d * \u6709\u72b6\u6001\uff08master/worker\uff09 \u5b88\u62a4\u8fdb\u7a0b\u96c6\uff08fuse\uff09 \u70b9\u51fb \u7f16\u8f91 \u6dfb\u52a0\u65e5\u5fd7\u6536\u96c6 \u5176\u4e2d\u65e5\u5fd7\u5e93\u540d\u79f0\u53ef\u81ea\u5b9a\u4e49\u586b\u5199\uff0c\u8be5\u540d\u79f0\u5373\u4e3a sls \u4e0a\u65e5\u5fd7\u5e93\u7684\u540d\u5b57\uff0c\u5bb9\u5668\u5185\u65e5\u5fd7\u8def\u5f84\u4e3a stdout \u70b9\u51fb\u53f3\u4fa7\u66f4\u65b0 3.2\u3001\u76f4\u63a5\u64cd\u4f5c K8S \u8d44\u6e90 \u00a7 \u4ee5\u6536\u96c6 fuse \u65e5\u5fd7\u4e3a\u4f8b\uff0c\u6267\u884c kubectl edit daemonset test-jindofs-fuse \u4fee\u6539 spec.env \u589e\u52a0 - name: aliyun_logs_jindo-fuse value: stdout name \u683c\u5f0f\u4e3a\uff1a aliyun_logs_ \uff0c\u524d\u7f00\u4e3a aliyun_logs_ value \u4e3a\u56fa\u5b9a\u503c stdout \u5982\u679c\u60f3\u4fee\u6539 master/worker\uff0c\u53ef\u6267\u884c kubectl edit statefulset test-jindofs-master kubectl edit statefulset test-jindofs-worker \u8fdb\u884c\u5bf9\u5e94\u7684\u7f16\u8f91 4\u3001\u767b\u9646\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u670d\u52a1\u67e5\u770b\u76f8\u5173\u65e5\u5fd7\u60c5\u51b5 \u00a7 \u4ee5 jindofuse \u65e5\u5fd7\u6536\u96c6\u4e3a\u4f8b\uff0c\u65e5\u5fd7\u5e93\u540d\u79f0\u4e3a jindo-fuse ,\u5728sls\u4e0a\u627e\u5230\u5bf9\u5e94\u7684\u65e5\u5fd7\u5e93","title":"\u4f7f\u7528\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u6536\u96c6\u670d\u52a1"},{"location":"fluid-jindoruntime/jindo_fluid_aliyun_sls_example/#sls","text":"JindoRuntime \u9ed8\u8ba4\u65e5\u5fd7\u6253\u5728\u5bb9\u5668 std \u8f93\u51fa\u91cc\u9762\uff0c\u53ef\u4ee5\u4f7f\u7528\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u6536\u96c6\u670d\u52a1\u5c06\u65e5\u5fd7\u6536\u96c6\u8d77\u6765\u3002","title":"\u4f7f\u7528\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u6536\u96c6\u670d\u52a1"},{"location":"fluid-jindoruntime/jindo_fluid_aliyun_sls_example/#1ack-sls","text":"\u60a8\u53ef\u4ee5\u5728\u521b\u5efa\u96c6\u7fa4\u65f6\u7ed1\u5b9a sls \u670d\u52a1 \u4e5f\u53ef\u4ee5\u5728\u521b\u5efa\u96c6\u7fa4\u540e\u7ed1\u5b9a\u963f\u91cc\u4e91 sls \u670d\u52a1 \u9009\u62e9\u96c6\u7fa4\u4fe1\u606f > \u96c6\u7fa4\u8d44\u6e90 > \u65e5\u5fd7\u670d\u52a1 Project","title":"1\u3001ack \u96c6\u7fa4\u7ed1\u5b9a sls \u670d\u52a1"},{"location":"fluid-jindoruntime/jindo_fluid_aliyun_sls_example/#2-jindoruntime","text":"\u53ef\u53c2\u8003\u6587\u7ae0 \u52a0\u901fOSS\u4e0a\u6570\u636e \u521b\u5efa JindoRuntime \u96c6\u7fa4\u3002","title":"2\u3001\u521b\u5efa JindoRuntime \u96c6\u7fa4"},{"location":"fluid-jindoruntime/jindo_fluid_aliyun_sls_example/#3-sls","text":"\u60a8\u53ef\u4ee5\u9009\u62e9\u5728 ack \u63a7\u5236\u53f0\u64cd\u4f5c\uff0c\u6216\u8005\u76f4\u63a5\u64cd\u4f5c\u96c6\u7fa4 K8S \u8d44\u6e90","title":"3\u3001\u914d\u7f6e sls \u6536\u96c6"},{"location":"fluid-jindoruntime/jindo_fluid_aliyun_sls_example/#31-ack","text":"\u672c\u6587\u4ee5 JindoRuntime \u540d\u5b57\u4e3a test \u8fdb\u884c\u4e3e\u4f8b \u60a8\u53ef\u4ee5\u767b\u9646 ack \u63a7\u5236\u53f0 > \u5de5\u4f5c\u8d1f\u8f7d * \u6709\u72b6\u6001\uff08master/worker\uff09 \u5b88\u62a4\u8fdb\u7a0b\u96c6\uff08fuse\uff09 \u70b9\u51fb \u7f16\u8f91 \u6dfb\u52a0\u65e5\u5fd7\u6536\u96c6 \u5176\u4e2d\u65e5\u5fd7\u5e93\u540d\u79f0\u53ef\u81ea\u5b9a\u4e49\u586b\u5199\uff0c\u8be5\u540d\u79f0\u5373\u4e3a sls \u4e0a\u65e5\u5fd7\u5e93\u7684\u540d\u5b57\uff0c\u5bb9\u5668\u5185\u65e5\u5fd7\u8def\u5f84\u4e3a stdout \u70b9\u51fb\u53f3\u4fa7\u66f4\u65b0","title":"3.1 ack \u63a7\u5236\u53f0\u64cd\u4f5c"},{"location":"fluid-jindoruntime/jindo_fluid_aliyun_sls_example/#32-k8s","text":"\u4ee5\u6536\u96c6 fuse \u65e5\u5fd7\u4e3a\u4f8b\uff0c\u6267\u884c kubectl edit daemonset test-jindofs-fuse \u4fee\u6539 spec.env \u589e\u52a0 - name: aliyun_logs_jindo-fuse value: stdout name \u683c\u5f0f\u4e3a\uff1a aliyun_logs_ \uff0c\u524d\u7f00\u4e3a aliyun_logs_ value \u4e3a\u56fa\u5b9a\u503c stdout \u5982\u679c\u60f3\u4fee\u6539 master/worker\uff0c\u53ef\u6267\u884c kubectl edit statefulset test-jindofs-master kubectl edit statefulset test-jindofs-worker \u8fdb\u884c\u5bf9\u5e94\u7684\u7f16\u8f91","title":"3.2\u3001\u76f4\u63a5\u64cd\u4f5c K8S \u8d44\u6e90"},{"location":"fluid-jindoruntime/jindo_fluid_aliyun_sls_example/#4-sls","text":"\u4ee5 jindofuse \u65e5\u5fd7\u6536\u96c6\u4e3a\u4f8b\uff0c\u65e5\u5fd7\u5e93\u540d\u79f0\u4e3a jindo-fuse ,\u5728sls\u4e0a\u627e\u5230\u5bf9\u5e94\u7684\u65e5\u5fd7\u5e93","title":"4\u3001\u767b\u9646\u963f\u91cc\u4e91 sls \u65e5\u5fd7\u670d\u52a1\u67e5\u770b\u76f8\u5173\u65e5\u5fd7\u60c5\u51b5"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/","text":"\u6570\u636e\u9884\u52a0\u8f7d \u00a7 \u914d\u7f6e\u5f85\u521b\u5efa\u7684DataLoad\u5bf9\u8c61 \u521b\u5efaDataLoad\u5bf9\u8c61 \u67e5\u770b\u521b\u5efa\u7684DataLoad\u5bf9\u8c61\u72b6\u6001 \u7b49\u5f85\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5b8c\u6210 DataLoad\u8fdb\u9636\u914d\u7f6e \u6307\u5b9a\u4e00\u4e2a\u6216\u591a\u4e2a\u6570\u636e\u96c6\u5b50\u76ee\u5f55\u8fdb\u884c\u52a0\u8f7d \u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf \u539f\u5b50\u6027\u7f13\u5b58 \u5355\u72ec\u8fdb\u884c\u5143\u6570\u636e\u7f13\u5b58\uff0c\u4e0d\u505a\u6570\u636e\u7f13\u5b58 \u7f13\u5b58\u8fdb\u5ea6\u53caLOG\u67e5\u770b \u73af\u5883\u6e05\u7406 \u4e3a\u4e86\u4fdd\u8bc1\u5e94\u7528\u5728\u8bbf\u95ee\u6570\u636e\u65f6\u7684\u6027\u80fd\uff0c\u53ef\u4ee5\u901a\u8fc7 \u6570\u636e\u9884\u52a0\u8f7d \u63d0\u524d\u5c06\u8fdc\u7a0b\u5b58\u50a8\u7cfb\u7edf\u4e2d\u7684\u6570\u636e\u62c9\u53d6\u5230\u9760\u8fd1\u8ba1\u7b97\u7ed3\u70b9\u7684\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d\uff0c\u4f7f\u5f97\u6d88\u8d39\u8be5\u6570\u636e\u96c6\u7684\u5e94\u7528\u80fd\u591f\u5728\u9996\u6b21\u8fd0\u884c\u65f6\u5373\u53ef\u4eab\u53d7\u5230\u7f13\u5b58\u5e26\u6765\u7684\u52a0\u901f\u6548\u679c\u3002 \u4e3a\u6b64\uff0c\u6211\u4eec\u63d0\u4f9b\u4e86DataLoad CRD, \u8be5CRD\u8ba9\u4f60\u53ef\u901a\u8fc7\u7b80\u5355\u7684\u914d\u7f6e\u5c31\u80fd\u5b8c\u6210\u6574\u4e2a\u6570\u636e\u9884\u52a0\u8f7d\u8fc7\u7a0b\uff0c\u5e76\u5bf9\u6570\u636e\u9884\u52a0\u8f7d\u7684\u8bb8\u591a\u884c\u4e3a\u8fdb\u884c\u81ea\u5b9a\u4e49\u63a7\u5236\u3002 \u5047\u8bbe\u6211\u4eec\u5df2\u7ecf\u6709\u5982\u4e0b\u4e00\u4e2aname\u4e3a spark \u7684dataset\uff1a apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: spark spec: mounts: - mountPoint: oss://test-bucket/spark name: sparkV1 accessModes: - ReadWriteMany \u63a5\u4e0b\u6765\u5bf9\u8fd9\u4e2adataset\u8fdb\u884c\u9884\u70ed\uff0c\u8bf7\u6ce8\u610f\u4e0b\u6587\u4e2d\u7684 /sparkV1 \u4e3a mounts[].name \u7684\u503c\uff0c\u5982\u679c\u60a8\u7684 mounts[].path \u4e3a / , \u90a3\u4e48\u4e0b\u6587\u4e2d\u7684 /sparkV1 \u53ef\u4ee5\u76f4\u63a5\u6539\u4e3a / \u914d\u7f6e\u5f85\u521b\u5efa\u7684DataLoad\u5bf9\u8c61 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default target: - path: /sparkV1/ spec.dataset \u6307\u660e\u4e86\u9700\u8981\u8fdb\u884c\u6570\u636e\u9884\u52a0\u8f7d\u7684\u76ee\u6807\u6570\u636e\u96c6\uff0c\u5728\u8be5\u4f8b\u5b50\u4e2d\uff0c\u6211\u4eec\u7684\u6570\u636e\u9884\u52a0\u8f7d\u76ee\u6807\u4e3a default \u547d\u540d\u7a7a\u95f4\u4e0b\u540d\u4e3a spark \u7684\u6570\u636e\u96c6\uff0c\u5982\u679c\u8be5\u914d\u7f6e\u4e0e\u4f60\u6240\u5728\u7684\u5b9e\u9645\u73af\u5883\u4e0d\u7b26\uff0c\u8bf7\u6839\u636e\u4f60\u7684\u5b9e\u9645\u73af\u5883\u5bf9\u5176\u8fdb\u884c\u8c03\u6574\u3002 \u521b\u5efaDataLoad\u5bf9\u8c61 \u00a7 kubectl create -f dataload.yaml \u67e5\u770b\u521b\u5efa\u7684DataLoad\u5bf9\u8c61\u72b6\u6001 \u00a7 kubectl get dataload spark-dataload \u4e0a\u8ff0\u547d\u4ee4\u4f1a\u5f97\u5230\u7c7b\u4f3c\u4ee5\u4e0b\u7ed3\u679c\uff1a NAME DATASET PHASE AGE spark-dataload spark Loading 2m13s \u4f60\u4e5f\u53ef\u4ee5\u901a\u8fc7 kubectl describe \u83b7\u53d6\u6709\u5173\u8be5DataLoad\u7684\u66f4\u591a\u8be6\u7ec6\u4fe1\u606f\uff1a kubectl describe dataload spark-dataload \u5f97\u5230\u4ee5\u4e0b\u7ed3\u679c\uff1a Name: spark-dataload Namespace: default Labels: <none> Annotations: <none> API Version: data.fluid.io/v1alpha1 Kind: DataLoad ... Spec: Dataset: Name: spark Namespace: default Status: Conditions: Phase: Loading Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal DataLoadJobStarted 80s DataLoad The DataLoad job spark-dataload-loader-job started \u4e0a\u8ff0\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u6839\u636e\u4f60\u6240\u5728\u7684\u7f51\u7edc\u73af\u5883\u4e0d\u540c\uff0c\u53ef\u80fd\u4f1a\u8017\u8d39\u6570\u5206\u949f \u7b49\u5f85\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5b8c\u6210 \u00a7 kubectl get dataload spark-dataload \u4f60\u4f1a\u770b\u5230\u8be5DataLoad\u7684 Phase \u72b6\u6001\u5df2\u7ecf\u4ece Loading \u53d8\u4e3a Complete \uff0c\u8fd9\u8868\u660e\u6574\u4e2a\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5df2\u7ecf\u5b8c\u6210 NAME DATASET PHASE AGE spark-dataload spark Complete 5m17s \u6b64\u65f6\u518d\u6b21\u67e5\u770bDataset\u5bf9\u8c61\u7684\u7f13\u5b58\u72b6\u6001\uff1a kubectl get dataset spark \u53ef\u53d1\u73b0\uff0c\u8fdc\u7a0b\u5b58\u50a8\u7cfb\u7edf\u4e2d\u7684\u5168\u90e8\u6570\u636e\u5747\u5df2\u6210\u529f\u7f13\u5b58\u5230\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE spark 1.92GiB 1.92GiB 4.00GiB 100.0% Bound 7m41s DataLoad\u8fdb\u9636\u914d\u7f6e \u00a7 \u6307\u5b9a\u4e00\u4e2a\u6216\u591a\u4e2a\u6570\u636e\u96c6\u5b50\u76ee\u5f55\u8fdb\u884c\u52a0\u8f7d \u00a7 \u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6\u53ef\u4ee5\u52a0\u8f7d\u6307\u5b9a\u7684\u5b50\u76ee\u5f55(\u6216\u6587\u4ef6)\uff0c\u800c\u4e0d\u662f\u6574\u4e2a\u6570\u636e\u96c6\uff0c\u4f8b\u5982\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 - path: /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \u4e0a\u8ff0DataLoad\u4ec5\u4f1a\u52a0\u8f7d /sparkV1/spark-2.4.7 \u76ee\u5f55\u4e0b\u7684\u5168\u90e8\u6587\u4ef6\uff0c\u4ee5\u53ca /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \u6587\u4ef6 \u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf \u00a7 \u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6,\u4f60\u4e5f\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u63a7\u5236\u52a0\u8f7d\u7684\u6570\u636e\u526f\u672c\u6570\u91cf\uff0c\u4f8b\u5982\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 replicas: 1 - path: /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz replicas: 2 \u4e0a\u8ff0DataLoad\u5728\u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6\uff0c\u5bf9\u4e8e /sparkV1/spark-2.4.7 \u76ee\u5f55\u4e0b\u7684\u6587\u4ef6\u4ec5\u4f1a\u5728\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d\u4fdd\u7559 1\u4efd \u6570\u636e\u7f13\u5b58\u526f\u672c\uff0c\u800c\u5bf9\u4e8e\u6587\u4ef6 /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \uff0c\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u5c06\u4f1a\u4fdd\u7559 2\u4efd \u7f13\u5b58\u526f\u672c\u3002 \u539f\u5b50\u6027\u7f13\u5b58 \u00a7 \u5982\u60a8\u60f3\u4fdd\u6301\u7f13\u5b58\u4e00\u81f4\u6027\u548c\u539f\u5b50\u6027\uff0c\u90a3\u4e48\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b yaml \u6765\u5b8c\u6210\u8fd9\u4e2a\u529f\u80fd apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default options: atomicCache: \"true\" \u5355\u72ec\u8fdb\u884c\u5143\u6570\u636e\u7f13\u5b58\uff0c\u4e0d\u505a\u6570\u636e\u7f13\u5b58 \u00a7 \u5728\u8fdb\u884c dataload \u9884\u70ed\u65f6\uff0c\u9ed8\u8ba4\u662f\u4f1a\u505a\u6570\u636e\u7f13\u5b58\uff0c\u539f\u6570\u636e\u7f13\u5b58\u662f\u53ef\u9009\uff0c\u4f46\u5982\u679c\u60a8\u53ea\u60f3\u5143\u6570\u636e\u7f13\u5b58\u4e0d\u60f3\u505a\u6570\u636e\u7f13\u5b58\u7684\u8bdd\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b yaml \u6253\u5f00\u53ea\u505a\u5143\u6570\u636e\u7f13\u5b58\u7684\u5f00\u5173 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default options: loadMetadataOnly: \"true\" \u901a\u8fc7\u901a\u914d\u7b26\u8fdb\u884c\u7f13\u5b58\u8fc7\u6ee4 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 replicas: 1 options: filter: \"*.tgz\" cacheThread: \"2\" dryrun: \"false\" \u4f7f\u7528\u5217\u8868\u6587\u4ef6\u8fdb\u884c\u7f13\u5b58 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true options: accessKeyId: \"xxx\" accessKeySecret: \"xxx\" endpoint: \"oss-cn-shanghai-internal.aliyuncs.com\" url: \"oss://test-bucket/filelist\" dryrun: \"false\" \u7f13\u5b58\u8fdb\u5ea6\u53caLOG\u67e5\u770b \u00a7 $ kubectl get dataload spark-dataload NAME DATASET PHASE AGE spark-dataload spark Loading 2m13s \u67e5\u770b\u5f53\u524d\u7f13\u5b58\u7684\u76f8\u5173\u4fe1\u606f kubectl logs spark-dataload \u73af\u5883\u6e05\u7406 \u00a7 $ kubectl delete dataload spark-dataload","title":"\u6570\u636e\u9884\u52a0\u8f7d"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#_1","text":"\u914d\u7f6e\u5f85\u521b\u5efa\u7684DataLoad\u5bf9\u8c61 \u521b\u5efaDataLoad\u5bf9\u8c61 \u67e5\u770b\u521b\u5efa\u7684DataLoad\u5bf9\u8c61\u72b6\u6001 \u7b49\u5f85\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5b8c\u6210 DataLoad\u8fdb\u9636\u914d\u7f6e \u6307\u5b9a\u4e00\u4e2a\u6216\u591a\u4e2a\u6570\u636e\u96c6\u5b50\u76ee\u5f55\u8fdb\u884c\u52a0\u8f7d \u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf \u539f\u5b50\u6027\u7f13\u5b58 \u5355\u72ec\u8fdb\u884c\u5143\u6570\u636e\u7f13\u5b58\uff0c\u4e0d\u505a\u6570\u636e\u7f13\u5b58 \u7f13\u5b58\u8fdb\u5ea6\u53caLOG\u67e5\u770b \u73af\u5883\u6e05\u7406 \u4e3a\u4e86\u4fdd\u8bc1\u5e94\u7528\u5728\u8bbf\u95ee\u6570\u636e\u65f6\u7684\u6027\u80fd\uff0c\u53ef\u4ee5\u901a\u8fc7 \u6570\u636e\u9884\u52a0\u8f7d \u63d0\u524d\u5c06\u8fdc\u7a0b\u5b58\u50a8\u7cfb\u7edf\u4e2d\u7684\u6570\u636e\u62c9\u53d6\u5230\u9760\u8fd1\u8ba1\u7b97\u7ed3\u70b9\u7684\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d\uff0c\u4f7f\u5f97\u6d88\u8d39\u8be5\u6570\u636e\u96c6\u7684\u5e94\u7528\u80fd\u591f\u5728\u9996\u6b21\u8fd0\u884c\u65f6\u5373\u53ef\u4eab\u53d7\u5230\u7f13\u5b58\u5e26\u6765\u7684\u52a0\u901f\u6548\u679c\u3002 \u4e3a\u6b64\uff0c\u6211\u4eec\u63d0\u4f9b\u4e86DataLoad CRD, \u8be5CRD\u8ba9\u4f60\u53ef\u901a\u8fc7\u7b80\u5355\u7684\u914d\u7f6e\u5c31\u80fd\u5b8c\u6210\u6574\u4e2a\u6570\u636e\u9884\u52a0\u8f7d\u8fc7\u7a0b\uff0c\u5e76\u5bf9\u6570\u636e\u9884\u52a0\u8f7d\u7684\u8bb8\u591a\u884c\u4e3a\u8fdb\u884c\u81ea\u5b9a\u4e49\u63a7\u5236\u3002 \u5047\u8bbe\u6211\u4eec\u5df2\u7ecf\u6709\u5982\u4e0b\u4e00\u4e2aname\u4e3a spark \u7684dataset\uff1a apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: spark spec: mounts: - mountPoint: oss://test-bucket/spark name: sparkV1 accessModes: - ReadWriteMany \u63a5\u4e0b\u6765\u5bf9\u8fd9\u4e2adataset\u8fdb\u884c\u9884\u70ed\uff0c\u8bf7\u6ce8\u610f\u4e0b\u6587\u4e2d\u7684 /sparkV1 \u4e3a mounts[].name \u7684\u503c\uff0c\u5982\u679c\u60a8\u7684 mounts[].path \u4e3a / , \u90a3\u4e48\u4e0b\u6587\u4e2d\u7684 /sparkV1 \u53ef\u4ee5\u76f4\u63a5\u6539\u4e3a /","title":"\u6570\u636e\u9884\u52a0\u8f7d"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#dataload","text":"apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default target: - path: /sparkV1/ spec.dataset \u6307\u660e\u4e86\u9700\u8981\u8fdb\u884c\u6570\u636e\u9884\u52a0\u8f7d\u7684\u76ee\u6807\u6570\u636e\u96c6\uff0c\u5728\u8be5\u4f8b\u5b50\u4e2d\uff0c\u6211\u4eec\u7684\u6570\u636e\u9884\u52a0\u8f7d\u76ee\u6807\u4e3a default \u547d\u540d\u7a7a\u95f4\u4e0b\u540d\u4e3a spark \u7684\u6570\u636e\u96c6\uff0c\u5982\u679c\u8be5\u914d\u7f6e\u4e0e\u4f60\u6240\u5728\u7684\u5b9e\u9645\u73af\u5883\u4e0d\u7b26\uff0c\u8bf7\u6839\u636e\u4f60\u7684\u5b9e\u9645\u73af\u5883\u5bf9\u5176\u8fdb\u884c\u8c03\u6574\u3002","title":"\u914d\u7f6e\u5f85\u521b\u5efa\u7684DataLoad\u5bf9\u8c61"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#dataload_1","text":"kubectl create -f dataload.yaml","title":"\u521b\u5efaDataLoad\u5bf9\u8c61"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#dataload_2","text":"kubectl get dataload spark-dataload \u4e0a\u8ff0\u547d\u4ee4\u4f1a\u5f97\u5230\u7c7b\u4f3c\u4ee5\u4e0b\u7ed3\u679c\uff1a NAME DATASET PHASE AGE spark-dataload spark Loading 2m13s \u4f60\u4e5f\u53ef\u4ee5\u901a\u8fc7 kubectl describe \u83b7\u53d6\u6709\u5173\u8be5DataLoad\u7684\u66f4\u591a\u8be6\u7ec6\u4fe1\u606f\uff1a kubectl describe dataload spark-dataload \u5f97\u5230\u4ee5\u4e0b\u7ed3\u679c\uff1a Name: spark-dataload Namespace: default Labels: <none> Annotations: <none> API Version: data.fluid.io/v1alpha1 Kind: DataLoad ... Spec: Dataset: Name: spark Namespace: default Status: Conditions: Phase: Loading Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal DataLoadJobStarted 80s DataLoad The DataLoad job spark-dataload-loader-job started \u4e0a\u8ff0\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u6839\u636e\u4f60\u6240\u5728\u7684\u7f51\u7edc\u73af\u5883\u4e0d\u540c\uff0c\u53ef\u80fd\u4f1a\u8017\u8d39\u6570\u5206\u949f","title":"\u67e5\u770b\u521b\u5efa\u7684DataLoad\u5bf9\u8c61\u72b6\u6001"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#_2","text":"kubectl get dataload spark-dataload \u4f60\u4f1a\u770b\u5230\u8be5DataLoad\u7684 Phase \u72b6\u6001\u5df2\u7ecf\u4ece Loading \u53d8\u4e3a Complete \uff0c\u8fd9\u8868\u660e\u6574\u4e2a\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5df2\u7ecf\u5b8c\u6210 NAME DATASET PHASE AGE spark-dataload spark Complete 5m17s \u6b64\u65f6\u518d\u6b21\u67e5\u770bDataset\u5bf9\u8c61\u7684\u7f13\u5b58\u72b6\u6001\uff1a kubectl get dataset spark \u53ef\u53d1\u73b0\uff0c\u8fdc\u7a0b\u5b58\u50a8\u7cfb\u7edf\u4e2d\u7684\u5168\u90e8\u6570\u636e\u5747\u5df2\u6210\u529f\u7f13\u5b58\u5230\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE spark 1.92GiB 1.92GiB 4.00GiB 100.0% Bound 7m41s","title":"\u7b49\u5f85\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u5b8c\u6210"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#dataload_3","text":"","title":"DataLoad\u8fdb\u9636\u914d\u7f6e"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#_3","text":"\u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6\u53ef\u4ee5\u52a0\u8f7d\u6307\u5b9a\u7684\u5b50\u76ee\u5f55(\u6216\u6587\u4ef6)\uff0c\u800c\u4e0d\u662f\u6574\u4e2a\u6570\u636e\u96c6\uff0c\u4f8b\u5982\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 - path: /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \u4e0a\u8ff0DataLoad\u4ec5\u4f1a\u52a0\u8f7d /sparkV1/spark-2.4.7 \u76ee\u5f55\u4e0b\u7684\u5168\u90e8\u6587\u4ef6\uff0c\u4ee5\u53ca /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \u6587\u4ef6","title":"\u6307\u5b9a\u4e00\u4e2a\u6216\u591a\u4e2a\u6570\u636e\u96c6\u5b50\u76ee\u5f55\u8fdb\u884c\u52a0\u8f7d"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#_4","text":"\u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6,\u4f60\u4e5f\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u63a7\u5236\u52a0\u8f7d\u7684\u6570\u636e\u526f\u672c\u6570\u91cf\uff0c\u4f8b\u5982\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 replicas: 1 - path: /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz replicas: 2 \u4e0a\u8ff0DataLoad\u5728\u8fdb\u884c\u6570\u636e\u52a0\u8f7d\u65f6\uff0c\u5bf9\u4e8e /sparkV1/spark-2.4.7 \u76ee\u5f55\u4e0b\u7684\u6587\u4ef6\u4ec5\u4f1a\u5728\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u4e2d\u4fdd\u7559 1\u4efd \u6570\u636e\u7f13\u5b58\u526f\u672c\uff0c\u800c\u5bf9\u4e8e\u6587\u4ef6 /sparkV1/spark-3.0.1/pyspark-3.0.1.tar.gz \uff0c\u5206\u5e03\u5f0f\u7f13\u5b58\u5f15\u64ce\u5c06\u4f1a\u4fdd\u7559 2\u4efd \u7f13\u5b58\u526f\u672c\u3002","title":"\u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#_5","text":"\u5982\u60a8\u60f3\u4fdd\u6301\u7f13\u5b58\u4e00\u81f4\u6027\u548c\u539f\u5b50\u6027\uff0c\u90a3\u4e48\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b yaml \u6765\u5b8c\u6210\u8fd9\u4e2a\u529f\u80fd apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default options: atomicCache: \"true\"","title":"\u539f\u5b50\u6027\u7f13\u5b58"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#_6","text":"\u5728\u8fdb\u884c dataload \u9884\u70ed\u65f6\uff0c\u9ed8\u8ba4\u662f\u4f1a\u505a\u6570\u636e\u7f13\u5b58\uff0c\u539f\u6570\u636e\u7f13\u5b58\u662f\u53ef\u9009\uff0c\u4f46\u5982\u679c\u60a8\u53ea\u60f3\u5143\u6570\u636e\u7f13\u5b58\u4e0d\u60f3\u505a\u6570\u636e\u7f13\u5b58\u7684\u8bdd\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b yaml \u6253\u5f00\u53ea\u505a\u5143\u6570\u636e\u7f13\u5b58\u7684\u5f00\u5173 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: loadMetadata: true dataset: name: spark namespace: default options: loadMetadataOnly: \"true\"","title":"\u5355\u72ec\u8fdb\u884c\u5143\u6570\u636e\u7f13\u5b58\uff0c\u4e0d\u505a\u6570\u636e\u7f13\u5b58"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#_7","text":"apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true target: - path: /sparkV1/spark-2.4.7 replicas: 1 options: filter: \"*.tgz\" cacheThread: \"2\" dryrun: \"false\"","title":"\u901a\u8fc7\u901a\u914d\u7b26\u8fdb\u884c\u7f13\u5b58\u8fc7\u6ee4"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#_8","text":"apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: spark-dataload spec: dataset: name: spark namespace: default loadMetadata: true options: accessKeyId: \"xxx\" accessKeySecret: \"xxx\" endpoint: \"oss-cn-shanghai-internal.aliyuncs.com\" url: \"oss://test-bucket/filelist\" dryrun: \"false\"","title":"\u4f7f\u7528\u5217\u8868\u6587\u4ef6\u8fdb\u884c\u7f13\u5b58"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#log","text":"$ kubectl get dataload spark-dataload NAME DATASET PHASE AGE spark-dataload spark Loading 2m13s \u67e5\u770b\u5f53\u524d\u7f13\u5b58\u7684\u76f8\u5173\u4fe1\u606f kubectl logs spark-dataload","title":"\u7f13\u5b58\u8fdb\u5ea6\u53caLOG\u67e5\u770b"},{"location":"fluid-jindoruntime/jindo_fluid_dataload/#_9","text":"$ kubectl delete dataload spark-dataload","title":"\u73af\u5883\u6e05\u7406"},{"location":"fluid-jindoruntime/jindo_fluid_download/","text":"\u4e0b\u8f7d 1.0.0-20240112 \u4e0b\u8f7d \u00a7 1.0.0-20240112 \u00a7 Fluid 1.0.0 JindoRuntime\u76f8\u5173\u529f\u80fd\u5347\u7ea7\uff1a * JindoCache \u5347\u7ea7\u5230 6.2.0","title":"Jindo fluid download"},{"location":"fluid-jindoruntime/jindo_fluid_download/#_1","text":"","title":"\u4e0b\u8f7d"},{"location":"fluid-jindoruntime/jindo_fluid_download/#100-20240112","text":"Fluid 1.0.0 JindoRuntime\u76f8\u5173\u529f\u80fd\u5347\u7ea7\uff1a * JindoCache \u5347\u7ea7\u5230 6.2.0","title":"1.0.0-20240112"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/","text":"\u52a0\u901fHDFS\u4e0a\u6570\u636e \u00a7 JindoFSx \u63d0\u4f9b\u4e86\u8bbf\u95ee HDFS \u7684\u80fd\u529b\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e HDFS \u4f5c\u4e3a JindoFSx \u7684\u540e\u7aef\u5b58\u50a8\uff0c\u4f7f Fluid \u901a\u8fc7 JindoFSx \u6765\u8bbf\u95ee HDFS \u4e0a\u7684\u6570\u636e\uff0c\u540c\u65f6 JindoFSx \u4e5f\u63d0\u4f9b\u4e86\u5bf9 HDFS \u4e0a\u7684\u6570\u636e\u4ee5\u53ca\u5143\u6570\u636e\u7684\u7f13\u5b58\u52a0\u901f\u529f\u80fd\u3002 \u672c\u6587\u6863\u5c55\u793a\u4e86\u5982\u4f55\u5728 Fluid \u4e2d\u4ee5\u58f0\u660e\u5f0f\u7684\u65b9\u5f0f\u5b8c\u6210 JindoFSx \u90e8\u7f72\uff0c\u5bf9\u63a5 HDFS \u6570\u636e\u6e90\u3002 \u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 \u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz \u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 1\u3001\u521b\u5efa dataset \u548c JindoRuntime \u00a7 1.1\u3001HA \u96c6\u7fa4\u4ece HDFS \u914d\u7f6e\u6587\u4ef6\u4e2d\u76f8\u5173\u4fe1\u606f\uff0c\u975e HA \u96c6\u7fa4\u8bf7\u7565\u8fc7\u6b64\u6b65\u9aa4 \u00a7 \u4ece HDFS \u96c6\u7fa4\u4e0a\u627e\u5230 hdfs-site.xml \u914d\u7f6e\u6587\u4ef6 * \u963f\u91cc\u4e91EMR\u96c6\u7fa4\u5728 /etc/ecm/hadoop-conf \u8def\u5f84\u4e0b * CDH \u96c6\u7fa4\u53ef\u53c2\u8003 What-is-the-Path-of-hdfs-site-xml-core-xml \u9700\u8981\u5c06\u5176\u4e2d rpc \u7684\u8bbf\u95ee\u5730\u5740\u4e2d\u57df\u540d\u7684\u90e8\u5206\u53c2\u6570\uff0c\u4fee\u6539\u6210\u53ef\u8bbf\u95ee\u7684\u5185\u7f51 IP \u5730\u5740\uff0c\u4f8b\u5982\uff1a <property> <name>dfs.namenode.rpc-address.emr-cluster.nn1</name> <value>emr-header-1.cluster-xxx:8020</value> </property> <property> <name>dfs.namenode.rpc-address.emr-cluster.nn2</name> <value>emr-header-2.cluster-xxx:8020</value> </property> \u4fee\u6539\u4e3a <property> <name>dfs.namenode.rpc-address.emr-cluster.nn1</name> <value>192.168.0.1:8020</value> </property> <property> <name>dfs.namenode.rpc-address.emr-cluster.nn2</name> <value>192.168.0.2:8020</value> </property> \u4ece\u914d\u7f6e\u6587\u4ef6\u91cc\u6211\u4eec\u53ef\u4ee5\u62ff\u5230\u5982\u4e0b\u51e0\u4e2a\u4fe1\u606f \u96c6\u7fa4namespaceService\u540d\u79f0\uff1aemr-cluster \u4e24\u4e2anamenode\u540d\u5b57\uff1ann1 \u548c nn2 \u4e24\u4e2anamenode\u7684\u8bbf\u95ee\u5730\u5740\uff1a192.168.0.1:8020 \u548c 192.168.0.2:8020 1.2\u3001\u521b\u5efa Dataset \u8d44\u6e90\u5bf9\u8c61 \u00a7 \u9996\u5148\u521b\u5efa\u4e00\u4e2a\u540d\u4e3a dataset.yaml \u7684\u6587\u4ef6\uff0c\u9488\u5bf9 HDFS \u96c6\u7fa4\u5206\u4e3a HA \u96c6\u7fa4\u548c\u975e HA \u96c6\u7fa4\u4e24\u79cd\u4e0d\u540c\u7684\u60c5\u51b5\u3002 HA \u96c6\u7fa4 \u00a7 \u5c061.1\u4e2d\u62ff\u5230\u7684\u51e0\u4e2a\u4fe1\u606f\u653e\u5230 mounts.options \u91cc\u9762\uff0cuser \u4ee3\u8868\u7528\u6237\u6743\u9650 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hdfs spec: mounts: - mountPoint: hdfs://emr-cluster/ name: hdfs options: fs.hdfs.emr-cluster.dfs.ha.namenodes = nn1,nn2 fs.hdfs.emr-cluster.dfs.namenode.rpc-address.nn1 = 192.168.0.1:8020 fs.hdfs.emr-cluster.dfs.namenode.rpc-address.nn2 = 192.168.0.2:8020 fs.hdfs.user = hadoop path: / accessModes: - ReadOnlyMany \u5176\u4e2d mountPoint \u4e3a\u9700\u8981\u8bbf\u95ee\u7684 HDFS \u8def\u5f84\uff08\u963f\u91cc\u4e91EMR\u96c6\u7fa4\u9ed8\u8ba4\u540d\u5b57\u4e3a emr-cluster\uff0c\u5982\u4e0b\u6240\u793a\uff09 - accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 <property> <name>fs.defaultFS</name> <value>hdfs://emr-cluster</value> </property> \u975e HA \u96c6\u7fa4 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hdfs spec: mounts: - mountPoint: hdfs://<namenodeIP>:<port>/ name: hdfs path: / accessModes: - ReadOnlyMany \u5176\u4e2d mountPoint \u4e3a HDFS \u96c6\u7fa4\u7684 namenode \u7684 IP \u5730\u5740\u548c\u7aef\u53e3\u7684\u7ec4\u5408 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 \u00a7 1.3\u3001\u521b\u5efa Dataset \u00a7 $ kubectl create -f dataset.yaml 1.4 \u521b\u5efa JindoRuntime \u8d44\u6e90\u5bf9\u8c61 \u00a7 \u521b\u5efa\u4e00\u4e2a JindoRuntime \u5bf9\u5e94\u7684 JindoFSx \u96c6\u7fa4 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hdfs spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /mnt/disk1/ quota: 290Gi high: \"0.9\" low: \"0.2\" \u591a\u5757\u76d8\u591a\u4e2aquota\u8bf7\u53c2\u8003 path: /mnt/disk1/,/mnt/disk2/,/mnt/disk3/ quotaList: 290G,290G,290G \u5176\u4e2d path \u548c quota \u7684\u6570\u91cf\u5e94\u8be5\u76f8\u540c\u3002 1.5\u3001\u521b\u5efa JindoRuntime \u00a7 \u6267\u884c create \u547d\u4ee4 $ kubectl create -f runtime.yaml \u67e5\u770b dataset \u72b6\u6001 \u00a7 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hdfs 180.01GiB 0.00B 261.00GiB 0.0% Bound 62m \u68c0\u67e5 PV\uff0cPVC \u521b\u5efa\u60c5\u51b5 \u00a7 $ kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hdfs 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hdfs Bound hadoop 100Gi RWX 58m \u68c0\u67e5 jindoruntime \u90e8\u7f72\u60c5\u51b5 \u00a7 $ kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hdfs Ready Ready 62m","title":"\u52a0\u901fHDFS\u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#hdfs","text":"JindoFSx \u63d0\u4f9b\u4e86\u8bbf\u95ee HDFS \u7684\u80fd\u529b\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e HDFS \u4f5c\u4e3a JindoFSx \u7684\u540e\u7aef\u5b58\u50a8\uff0c\u4f7f Fluid \u901a\u8fc7 JindoFSx \u6765\u8bbf\u95ee HDFS \u4e0a\u7684\u6570\u636e\uff0c\u540c\u65f6 JindoFSx \u4e5f\u63d0\u4f9b\u4e86\u5bf9 HDFS \u4e0a\u7684\u6570\u636e\u4ee5\u53ca\u5143\u6570\u636e\u7684\u7f13\u5b58\u52a0\u901f\u529f\u80fd\u3002 \u672c\u6587\u6863\u5c55\u793a\u4e86\u5982\u4f55\u5728 Fluid \u4e2d\u4ee5\u58f0\u660e\u5f0f\u7684\u65b9\u5f0f\u5b8c\u6210 JindoFSx \u90e8\u7f72\uff0c\u5bf9\u63a5 HDFS \u6570\u636e\u6e90\u3002","title":"\u52a0\u901fHDFS\u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#_1","text":"kubectl create ns fluid-system","title":"\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#fluid","text":"","title":"\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#fluid_1","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#1-dataset-jindoruntime","text":"","title":"1\u3001\u521b\u5efa dataset \u548c JindoRuntime"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#11ha-hdfs-ha","text":"\u4ece HDFS \u96c6\u7fa4\u4e0a\u627e\u5230 hdfs-site.xml \u914d\u7f6e\u6587\u4ef6 * \u963f\u91cc\u4e91EMR\u96c6\u7fa4\u5728 /etc/ecm/hadoop-conf \u8def\u5f84\u4e0b * CDH \u96c6\u7fa4\u53ef\u53c2\u8003 What-is-the-Path-of-hdfs-site-xml-core-xml \u9700\u8981\u5c06\u5176\u4e2d rpc \u7684\u8bbf\u95ee\u5730\u5740\u4e2d\u57df\u540d\u7684\u90e8\u5206\u53c2\u6570\uff0c\u4fee\u6539\u6210\u53ef\u8bbf\u95ee\u7684\u5185\u7f51 IP \u5730\u5740\uff0c\u4f8b\u5982\uff1a <property> <name>dfs.namenode.rpc-address.emr-cluster.nn1</name> <value>emr-header-1.cluster-xxx:8020</value> </property> <property> <name>dfs.namenode.rpc-address.emr-cluster.nn2</name> <value>emr-header-2.cluster-xxx:8020</value> </property> \u4fee\u6539\u4e3a <property> <name>dfs.namenode.rpc-address.emr-cluster.nn1</name> <value>192.168.0.1:8020</value> </property> <property> <name>dfs.namenode.rpc-address.emr-cluster.nn2</name> <value>192.168.0.2:8020</value> </property> \u4ece\u914d\u7f6e\u6587\u4ef6\u91cc\u6211\u4eec\u53ef\u4ee5\u62ff\u5230\u5982\u4e0b\u51e0\u4e2a\u4fe1\u606f \u96c6\u7fa4namespaceService\u540d\u79f0\uff1aemr-cluster \u4e24\u4e2anamenode\u540d\u5b57\uff1ann1 \u548c nn2 \u4e24\u4e2anamenode\u7684\u8bbf\u95ee\u5730\u5740\uff1a192.168.0.1:8020 \u548c 192.168.0.2:8020","title":"1.1\u3001HA \u96c6\u7fa4\u4ece HDFS \u914d\u7f6e\u6587\u4ef6\u4e2d\u76f8\u5173\u4fe1\u606f\uff0c\u975e HA \u96c6\u7fa4\u8bf7\u7565\u8fc7\u6b64\u6b65\u9aa4"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#12-dataset","text":"\u9996\u5148\u521b\u5efa\u4e00\u4e2a\u540d\u4e3a dataset.yaml \u7684\u6587\u4ef6\uff0c\u9488\u5bf9 HDFS \u96c6\u7fa4\u5206\u4e3a HA \u96c6\u7fa4\u548c\u975e HA \u96c6\u7fa4\u4e24\u79cd\u4e0d\u540c\u7684\u60c5\u51b5\u3002","title":"1.2\u3001\u521b\u5efa Dataset \u8d44\u6e90\u5bf9\u8c61"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#ha","text":"\u5c061.1\u4e2d\u62ff\u5230\u7684\u51e0\u4e2a\u4fe1\u606f\u653e\u5230 mounts.options \u91cc\u9762\uff0cuser \u4ee3\u8868\u7528\u6237\u6743\u9650 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hdfs spec: mounts: - mountPoint: hdfs://emr-cluster/ name: hdfs options: fs.hdfs.emr-cluster.dfs.ha.namenodes = nn1,nn2 fs.hdfs.emr-cluster.dfs.namenode.rpc-address.nn1 = 192.168.0.1:8020 fs.hdfs.emr-cluster.dfs.namenode.rpc-address.nn2 = 192.168.0.2:8020 fs.hdfs.user = hadoop path: / accessModes: - ReadOnlyMany \u5176\u4e2d mountPoint \u4e3a\u9700\u8981\u8bbf\u95ee\u7684 HDFS \u8def\u5f84\uff08\u963f\u91cc\u4e91EMR\u96c6\u7fa4\u9ed8\u8ba4\u540d\u5b57\u4e3a emr-cluster\uff0c\u5982\u4e0b\u6240\u793a\uff09 - accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 <property> <name>fs.defaultFS</name> <value>hdfs://emr-cluster</value> </property>","title":"HA \u96c6\u7fa4"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#ha_1","text":"apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hdfs spec: mounts: - mountPoint: hdfs://<namenodeIP>:<port>/ name: hdfs path: / accessModes: - ReadOnlyMany \u5176\u4e2d mountPoint \u4e3a HDFS \u96c6\u7fa4\u7684 namenode \u7684 IP \u5730\u5740\u548c\u7aef\u53e3\u7684\u7ec4\u5408 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199","title":"\u975e HA \u96c6\u7fa4"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#_2","text":"","title":""},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#13-dataset","text":"$ kubectl create -f dataset.yaml","title":"1.3\u3001\u521b\u5efa Dataset"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#14-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a JindoRuntime \u5bf9\u5e94\u7684 JindoFSx \u96c6\u7fa4 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hdfs spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /mnt/disk1/ quota: 290Gi high: \"0.9\" low: \"0.2\" \u591a\u5757\u76d8\u591a\u4e2aquota\u8bf7\u53c2\u8003 path: /mnt/disk1/,/mnt/disk2/,/mnt/disk3/ quotaList: 290G,290G,290G \u5176\u4e2d path \u548c quota \u7684\u6570\u91cf\u5e94\u8be5\u76f8\u540c\u3002","title":"1.4 \u521b\u5efa JindoRuntime \u8d44\u6e90\u5bf9\u8c61"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#15-jindoruntime","text":"\u6267\u884c create \u547d\u4ee4 $ kubectl create -f runtime.yaml","title":"1.5\u3001\u521b\u5efa JindoRuntime"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#dataset","text":"$ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hdfs 180.01GiB 0.00B 261.00GiB 0.0% Bound 62m","title":"\u67e5\u770b dataset \u72b6\u6001"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#pvpvc","text":"$ kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hdfs 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hdfs Bound hadoop 100Gi RWX 58m","title":"\u68c0\u67e5 PV\uff0cPVC \u521b\u5efa\u60c5\u51b5"},{"location":"fluid-jindoruntime/jindo_fluid_hdfs_ufs_example/#jindoruntime","text":"$ kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hdfs Ready Ready 62m","title":"\u68c0\u67e5 jindoruntime \u90e8\u7f72\u60c5\u51b5"},{"location":"fluid-jindoruntime/jindo_fluid_monitor_metric/","text":"\u76d1\u63a7\u548c\u53ef\u89c2\u6d4b\u6027\u529f\u80fd \u00a7 \u672c\u6587\u4ecb\u7ecd\u5728\u542f\u52a8 Fluid JindoRuntime \u7684\u524d\u63d0\u4e0b\uff0c\u901a\u8fc7 Prometheus + Grafana \u6536\u96c6\u7f13\u5b58\u7cfb\u7edf\u76f8\u5173\u7684 metrics \u6307\u6807\uff0c\u8fdb\u884c\u53ef\u89c6\u5316\u5c55\u793a\u7684\u4f7f\u7528\u65b9\u6cd5\u3002 \u4f7f\u7528 In-Cluster \u6a21\u5f0f\u90e8\u7f72 prometheus \u548c grafana \u76d1\u63a7 Fluid \u5e94\u7528 \u00a7 1. \u90e8\u7f72\u6216\u914d\u7f6e Prometheus \u00a7 \u5982\u679c\u96c6\u7fa4\u5185\u65e0 prometheus\uff0c\u8bf7\u70b9\u51fb\u4e0b\u8f7d prometheus-jindofs.yaml \u6587\u4ef6 $ kubectl apply -f prometheus-jindofs.yaml \u5982\u96c6\u7fa4\u5185\u6709 prometheus,\u53ef\u5c06\u4ee5\u4e0b\u914d\u7f6e\u5199\u5230 prometheus \u914d\u7f6e\u6587\u4ef6\u4e2d: scrape_configs: - job_name: 'jindo runtime' metrics_path: /jindometrics/prometheus kubernetes_sd_configs: - role: endpoints relabel_configs: - source_labels: [__meta_kubernetes_service_label_role] regex: jindofs-master action: keep - source_labels: [__meta_kubernetes_endpoint_port_name] regex: rpc action: keep - source_labels: [__meta_kubernetes_namespace] target_label: namespace replacement: $1 action: replace - source_labels: [__meta_kubernetes_service_label_release] target_label: fluid_runtime replacement: $1 action: replace - source_labels: [__meta_kubernetes_endpoint_address_target_name] target_label: pod replacement: $1 action: replace 2. \u90e8\u7f72 grafana \u00a7 \u5982\u679c\u96c6\u7fa4\u5185\u65e0 grafana\uff0c\u8bf7\u70b9\u51fb\u4e0b\u8f7d grafana.yaml \u6587\u4ef6 $ kubectl apply -f grafana.yaml 3. \u914d\u7f6e grafana \u00a7 \u767b\u5f55 grafana \u627e\u5230 grafana \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\u548c\u6539\u8282\u70b9\u5bf9\u5916\u53ef\u8bbf\u95ee grafana-node-ip \uff0c\u5982\u679c\u60f3\u8981\u5728\u516c\u7f51\u8bbf\u95ee\uff0c\u53ef\u4ee5\u7ed1\u5b9a\u516c\u7f51IP\uff0c $ kubectl get pod -n kube-system -o wide | grep grafana monitoring-grafana-7dfcf6d5cd-5k8qw 1/1 Running 0 108m 10.54.1.11 cn-shanghai.192.168.1.1 <none> <none> \u627e\u5230 grafana \u670d\u52a1\u5bf9\u5e94\u7684 NodePort $ kubectl describe svc monitoring-grafana -n kube-system Name: monitoring-grafana Namespace: kube-system Labels: <none> Annotations: <none> Selector: app=grafana Type: NodePort IP Families: <none> IP: 172.16.83.188 IPs: 172.16.83.188 Port: <unset> 80/TCP TargetPort: 3000/TCP NodePort: <unset> 30396/TCP Endpoints: 10.54.1.75:3000 Session Affinity: None External Traffic Policy: Cluster Events: <none> \u53ef\u4ee5\u770b\u5230\u5f53\u524d\u66b4\u9732\u7684 NodePort \u4e3a 30396 \u8bbf\u95ee http://$grafana-node-ip:NodePort \uff0c\u9ed8\u8ba4\u8d26\u53f7\u5bc6\u7801 admin:admin \u67e5\u770b prometheus svc \u7aef\u53e3 \u540c\u6837\u5148\u627e\u5230 prometheus \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\uff0c\u627e\u5230\u8be5\u8282\u70b9\u7684 IP $ kubectl get pod -n kube-system -o wide | grep grafana prometheus-deployment-d6d8554b4-wfl98 1/1 Running 0 17d 10.54.1.204 cn-shanghai.192.168.1.1 <none> <none> kubectl get no cn-shanghai.192.168.31.107 -o wide NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP cn-shanghai.192.168.1.1 Ready <none> 5h51m v1.20.4-aliyun.1 192.168.1.1 101.132.66.11 \u901a\u8fc7\u67e5\u770b\u8282\u70b9\u7684 INTERNAL-IP \uff0c\u53ef\u4e86\u89e3\u8be5\u8282\u70b9\u7684\u5185\u90e8\u8bbf\u95ee IP \u4e3a 192.168.1.1 $ kubectl get svc -n kube-system | grep prometheus-svc prometheus-svc NodePort 10.100.0.144 <none> 9090:31225/TCP 22h $ kubectl describe svc prometheus-svc -n kube-system Name: prometheus-svc Namespace: kube-system Labels: kubernetes.io/name=Prometheus name=prometheus-svc Annotations: kubectl.kubernetes.io/last-applied-configuration: {\"apiVersion\":\"v1\",\"kind\":\"Service\",\"metadata\":{\"annotations\":{},\"labels\":{\"kubernetes.io/name\":\"Prometheus\",\"name\":\"prometheus-svc\"},\"nam... Selector: app=prometheus Type: NodePort IP: 10.100.0.144 Port: prometheus 9090/TCP TargetPort: 9090/TCP NodePort: prometheus 31225/TCP Endpoints: 10.99.224.138:9090 Session Affinity: None External Traffic Policy: Cluster Events: <none> \u53ef\u4ee5\u770b\u5230\u5f53\u524d prometheus svc \u66b4\u9732\u7684\u7aef\u53e3\u662f 31225 \uff0c\u540e\u7eed\u4f7f\u7528 http://IP:NodePort \u7684\u5f62\u5f0f\u6765\u53d1\u73b0 prometheus \u670d\u52a1 \u914d\u7f6e prometheus data source \u4f7f\u7528 http://IP:NodePort \u7684\u5f62\u5f0f\u6765\u53d1\u73b0 prometheus \u670d\u52a1 \u5bfc\u5165\u5b8c\u6210\u540e\u70b9\u51fbSave & Test \u663e\u793a Data source is working \u5373\u53ef \u5bfc\u5165\u6a21\u677f\u6587\u4ef6 grafana \u9009\u62e9\u5bfc\u5165\u6a21\u677f Json \u6587\u4ef6\uff0c\u70b9\u51fb\u6b64\u5904\u4e0b\u8f7d fluid-prometheus-grafana-monitor-jindofs.json \u67e5\u770b\u76d1\u63a7 \u5728 DashBoards \u4e2d\u627e\u5230\u5bfc\u5165\u7684\u6a21\u7248\u89c6\u56fe \u6ce8\uff1afluid_runtime \u5bf9\u5e94Fluid runtime name; namespace \u5bf9\u5e94 Fluid runtime namespace JindoRuntime Prometheus \u6307\u6807\u9884\u89c8(\u66f4\u65b0\u4e2d) \u00a7 \u4e0b\u9762\u5bf9 JindoRuntime \u5f53\u524d\u6536\u96c6\u7684\u6307\u6807\u8fdb\u884c\u9884\u89c8\u548c\u63cf\u8ff0\uff0c\u60a8\u53ef\u4ee5\u57fa\u4e8e\u5df2\u6536\u96c6\u6307\u6807\u5728 Grafana \u4e2d\u8fdb\u884c\u7ec4\u5408\u548c\u56fe\u8868\u5c55\u793a\uff0c\u5982\u60a8\u9700\u8981\u989d\u5916\u7684\u57fa\u672c metrics \u5c55\u793a\uff0c\u8bf7\u5f00 issue \u8054\u7cfb\u6211\u4eec # HELP jindofsx_ns_backend_read_bytes_time_total_window # TYPE jindofsx_ns_backend_read_bytes_time_total_window gauge jindofsx_ns_backend_read_bytes_time_total_window 20367553 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_read_bytes_total # TYPE jindofsx_ns_backend_read_bytes_total gauge jindofsx_ns_backend_read_bytes_total 166703347021 #\u603b\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_bytes_total_window # TYPE jindofsx_ns_backend_read_bytes_total_window gauge jindofsx_ns_backend_read_bytes_total_window 1174671374 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_time_total # TYPE jindofsx_ns_backend_read_time_total gauge jindofsx_ns_backend_read_time_total 8717632094 #\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_readop_num_total # TYPE jindofsx_ns_backend_readop_num_total gauge jindofsx_ns_backend_readop_num_total 181951 #\u5e95\u5c42\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_local_read_bytes_time_total_window # TYPE jindofsx_ns_local_read_bytes_time_total_window gauge jindofsx_ns_local_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_bytes_total # TYPE jindofsx_ns_local_read_bytes_total gauge jindofsx_ns_local_read_bytes_total 0 #\u603b\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_local_read_bytes_total_window # TYPE jindofsx_ns_local_read_bytes_total_window gauge jindofsx_ns_local_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_time_total # TYPE jindofsx_ns_local_read_time_total gauge jindofsx_ns_local_read_time_total 0 #\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_readop_num_total # TYPE jindofsx_ns_local_readop_num_total gauge jindofsx_ns_local_readop_num_total 0 #\u77ed\u8def\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_remote_read_bytes_time_total_window # TYPE jindofsx_ns_remote_read_bytes_time_total_window gauge jindofsx_ns_remote_read_bytes_time_total_window 73714663 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_read_bytes_total # TYPE jindofsx_ns_remote_read_bytes_total gauge jindofsx_ns_remote_read_bytes_total 112108045498 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_bytes_total_window # TYPE jindofsx_ns_remote_read_bytes_total_window gauge jindofsx_ns_remote_read_bytes_total_window 13380884043 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_time_total # TYPE jindofsx_ns_remote_read_time_total gauge jindofsx_ns_remote_read_time_total 765272359 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u65f6\u95f4\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_readop_num_total # TYPE jindofsx_ns_remote_readop_num_total gauge jindofsx_ns_remote_readop_num_total 107314 #\u8fdc\u7aef\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_slicelet_read_bytes_time_total_window # TYPE jindofsx_ns_slicelet_read_bytes_time_total_window gauge jindofsx_ns_slicelet_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_read_bytes_total # TYPE jindofsx_ns_slicelet_read_bytes_total gauge jindofsx_ns_slicelet_read_bytes_total 0 #\u603b\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_bytes_total_window # TYPE jindofsx_ns_slicelet_read_bytes_total_window gauge jindofsx_ns_slicelet_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_time_total # TYPE jindofsx_ns_slicelet_read_time_total gauge jindofsx_ns_slicelet_read_time_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_readop_num_total # TYPE jindofsx_ns_slicelet_readop_num_total gauge jindofsx_ns_slicelet_readop_num_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_total_disk_cap # TYPE jindofsx_ns_total_disk_cap gauge jindofsx_ns_total_disk_cap 840739848192 #\u78c1\u76d8\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_mem_cap # TYPE jindofsx_ns_total_mem_cap gauge jindofsx_ns_total_mem_cap 0 #\u5185\u5b58\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_stsnodes_num # TYPE jindofsx_ns_total_stsnodes_num gauge jindofsx_ns_total_stsnodes_num 3 #alive worker \u8282\u70b9\u7684\u6570\u91cf # HELP jindofsx_ns_total_used_disk_cap # TYPE jindofsx_ns_total_used_disk_cap gauge jindofsx_ns_total_used_disk_cap 153423446016 #\u5df2\u7528\u78c1\u76d8\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_used_mem_cap # TYPE jindofsx_ns_total_used_mem_cap gauge jindofsx_ns_total_used_mem_cap 0 #\u5df2\u7528\u5185\u5b58\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte","title":"\u76d1\u63a7\u548c\u53ef\u89c2\u6d4b\u6027\u529f\u80fd"},{"location":"fluid-jindoruntime/jindo_fluid_monitor_metric/#_1","text":"\u672c\u6587\u4ecb\u7ecd\u5728\u542f\u52a8 Fluid JindoRuntime \u7684\u524d\u63d0\u4e0b\uff0c\u901a\u8fc7 Prometheus + Grafana \u6536\u96c6\u7f13\u5b58\u7cfb\u7edf\u76f8\u5173\u7684 metrics \u6307\u6807\uff0c\u8fdb\u884c\u53ef\u89c6\u5316\u5c55\u793a\u7684\u4f7f\u7528\u65b9\u6cd5\u3002","title":"\u76d1\u63a7\u548c\u53ef\u89c2\u6d4b\u6027\u529f\u80fd"},{"location":"fluid-jindoruntime/jindo_fluid_monitor_metric/#in-cluster-prometheus-grafana-fluid","text":"","title":"\u4f7f\u7528 In-Cluster \u6a21\u5f0f\u90e8\u7f72 prometheus \u548c grafana \u76d1\u63a7 Fluid \u5e94\u7528"},{"location":"fluid-jindoruntime/jindo_fluid_monitor_metric/#1-prometheus","text":"\u5982\u679c\u96c6\u7fa4\u5185\u65e0 prometheus\uff0c\u8bf7\u70b9\u51fb\u4e0b\u8f7d prometheus-jindofs.yaml \u6587\u4ef6 $ kubectl apply -f prometheus-jindofs.yaml \u5982\u96c6\u7fa4\u5185\u6709 prometheus,\u53ef\u5c06\u4ee5\u4e0b\u914d\u7f6e\u5199\u5230 prometheus \u914d\u7f6e\u6587\u4ef6\u4e2d: scrape_configs: - job_name: 'jindo runtime' metrics_path: /jindometrics/prometheus kubernetes_sd_configs: - role: endpoints relabel_configs: - source_labels: [__meta_kubernetes_service_label_role] regex: jindofs-master action: keep - source_labels: [__meta_kubernetes_endpoint_port_name] regex: rpc action: keep - source_labels: [__meta_kubernetes_namespace] target_label: namespace replacement: $1 action: replace - source_labels: [__meta_kubernetes_service_label_release] target_label: fluid_runtime replacement: $1 action: replace - source_labels: [__meta_kubernetes_endpoint_address_target_name] target_label: pod replacement: $1 action: replace","title":"1. \u90e8\u7f72\u6216\u914d\u7f6e Prometheus"},{"location":"fluid-jindoruntime/jindo_fluid_monitor_metric/#2-grafana","text":"\u5982\u679c\u96c6\u7fa4\u5185\u65e0 grafana\uff0c\u8bf7\u70b9\u51fb\u4e0b\u8f7d grafana.yaml \u6587\u4ef6 $ kubectl apply -f grafana.yaml","title":"2. \u90e8\u7f72 grafana"},{"location":"fluid-jindoruntime/jindo_fluid_monitor_metric/#3-grafana","text":"\u767b\u5f55 grafana \u627e\u5230 grafana \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\u548c\u6539\u8282\u70b9\u5bf9\u5916\u53ef\u8bbf\u95ee grafana-node-ip \uff0c\u5982\u679c\u60f3\u8981\u5728\u516c\u7f51\u8bbf\u95ee\uff0c\u53ef\u4ee5\u7ed1\u5b9a\u516c\u7f51IP\uff0c $ kubectl get pod -n kube-system -o wide | grep grafana monitoring-grafana-7dfcf6d5cd-5k8qw 1/1 Running 0 108m 10.54.1.11 cn-shanghai.192.168.1.1 <none> <none> \u627e\u5230 grafana \u670d\u52a1\u5bf9\u5e94\u7684 NodePort $ kubectl describe svc monitoring-grafana -n kube-system Name: monitoring-grafana Namespace: kube-system Labels: <none> Annotations: <none> Selector: app=grafana Type: NodePort IP Families: <none> IP: 172.16.83.188 IPs: 172.16.83.188 Port: <unset> 80/TCP TargetPort: 3000/TCP NodePort: <unset> 30396/TCP Endpoints: 10.54.1.75:3000 Session Affinity: None External Traffic Policy: Cluster Events: <none> \u53ef\u4ee5\u770b\u5230\u5f53\u524d\u66b4\u9732\u7684 NodePort \u4e3a 30396 \u8bbf\u95ee http://$grafana-node-ip:NodePort \uff0c\u9ed8\u8ba4\u8d26\u53f7\u5bc6\u7801 admin:admin \u67e5\u770b prometheus svc \u7aef\u53e3 \u540c\u6837\u5148\u627e\u5230 prometheus \u670d\u52a1\u6240\u5728\u7684\u8282\u70b9\uff0c\u627e\u5230\u8be5\u8282\u70b9\u7684 IP $ kubectl get pod -n kube-system -o wide | grep grafana prometheus-deployment-d6d8554b4-wfl98 1/1 Running 0 17d 10.54.1.204 cn-shanghai.192.168.1.1 <none> <none> kubectl get no cn-shanghai.192.168.31.107 -o wide NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP cn-shanghai.192.168.1.1 Ready <none> 5h51m v1.20.4-aliyun.1 192.168.1.1 101.132.66.11 \u901a\u8fc7\u67e5\u770b\u8282\u70b9\u7684 INTERNAL-IP \uff0c\u53ef\u4e86\u89e3\u8be5\u8282\u70b9\u7684\u5185\u90e8\u8bbf\u95ee IP \u4e3a 192.168.1.1 $ kubectl get svc -n kube-system | grep prometheus-svc prometheus-svc NodePort 10.100.0.144 <none> 9090:31225/TCP 22h $ kubectl describe svc prometheus-svc -n kube-system Name: prometheus-svc Namespace: kube-system Labels: kubernetes.io/name=Prometheus name=prometheus-svc Annotations: kubectl.kubernetes.io/last-applied-configuration: {\"apiVersion\":\"v1\",\"kind\":\"Service\",\"metadata\":{\"annotations\":{},\"labels\":{\"kubernetes.io/name\":\"Prometheus\",\"name\":\"prometheus-svc\"},\"nam... Selector: app=prometheus Type: NodePort IP: 10.100.0.144 Port: prometheus 9090/TCP TargetPort: 9090/TCP NodePort: prometheus 31225/TCP Endpoints: 10.99.224.138:9090 Session Affinity: None External Traffic Policy: Cluster Events: <none> \u53ef\u4ee5\u770b\u5230\u5f53\u524d prometheus svc \u66b4\u9732\u7684\u7aef\u53e3\u662f 31225 \uff0c\u540e\u7eed\u4f7f\u7528 http://IP:NodePort \u7684\u5f62\u5f0f\u6765\u53d1\u73b0 prometheus \u670d\u52a1 \u914d\u7f6e prometheus data source \u4f7f\u7528 http://IP:NodePort \u7684\u5f62\u5f0f\u6765\u53d1\u73b0 prometheus \u670d\u52a1 \u5bfc\u5165\u5b8c\u6210\u540e\u70b9\u51fbSave & Test \u663e\u793a Data source is working \u5373\u53ef \u5bfc\u5165\u6a21\u677f\u6587\u4ef6 grafana \u9009\u62e9\u5bfc\u5165\u6a21\u677f Json \u6587\u4ef6\uff0c\u70b9\u51fb\u6b64\u5904\u4e0b\u8f7d fluid-prometheus-grafana-monitor-jindofs.json \u67e5\u770b\u76d1\u63a7 \u5728 DashBoards \u4e2d\u627e\u5230\u5bfc\u5165\u7684\u6a21\u7248\u89c6\u56fe \u6ce8\uff1afluid_runtime \u5bf9\u5e94Fluid runtime name; namespace \u5bf9\u5e94 Fluid runtime namespace","title":"3. \u914d\u7f6e grafana"},{"location":"fluid-jindoruntime/jindo_fluid_monitor_metric/#jindoruntime-prometheus","text":"\u4e0b\u9762\u5bf9 JindoRuntime \u5f53\u524d\u6536\u96c6\u7684\u6307\u6807\u8fdb\u884c\u9884\u89c8\u548c\u63cf\u8ff0\uff0c\u60a8\u53ef\u4ee5\u57fa\u4e8e\u5df2\u6536\u96c6\u6307\u6807\u5728 Grafana \u4e2d\u8fdb\u884c\u7ec4\u5408\u548c\u56fe\u8868\u5c55\u793a\uff0c\u5982\u60a8\u9700\u8981\u989d\u5916\u7684\u57fa\u672c metrics \u5c55\u793a\uff0c\u8bf7\u5f00 issue \u8054\u7cfb\u6211\u4eec # HELP jindofsx_ns_backend_read_bytes_time_total_window # TYPE jindofsx_ns_backend_read_bytes_time_total_window gauge jindofsx_ns_backend_read_bytes_time_total_window 20367553 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_read_bytes_total # TYPE jindofsx_ns_backend_read_bytes_total gauge jindofsx_ns_backend_read_bytes_total 166703347021 #\u603b\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_bytes_total_window # TYPE jindofsx_ns_backend_read_bytes_total_window gauge jindofsx_ns_backend_read_bytes_total_window 1174671374 #\u4e00\u5206\u949f\u5185\u5e95\u5c42\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_backend_read_time_total # TYPE jindofsx_ns_backend_read_time_total gauge jindofsx_ns_backend_read_time_total 8717632094 #\u5e95\u5c42\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_backend_readop_num_total # TYPE jindofsx_ns_backend_readop_num_total gauge jindofsx_ns_backend_readop_num_total 181951 #\u5e95\u5c42\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_local_read_bytes_time_total_window # TYPE jindofsx_ns_local_read_bytes_time_total_window gauge jindofsx_ns_local_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_bytes_total # TYPE jindofsx_ns_local_read_bytes_total gauge jindofsx_ns_local_read_bytes_total 0 #\u603b\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_local_read_bytes_total_window # TYPE jindofsx_ns_local_read_bytes_total_window gauge jindofsx_ns_local_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u77ed\u8def\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_read_time_total # TYPE jindofsx_ns_local_read_time_total gauge jindofsx_ns_local_read_time_total 0 #\u77ed\u8def\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_local_readop_num_total # TYPE jindofsx_ns_local_readop_num_total gauge jindofsx_ns_local_readop_num_total 0 #\u77ed\u8def\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_remote_read_bytes_time_total_window # TYPE jindofsx_ns_remote_read_bytes_time_total_window gauge jindofsx_ns_remote_read_bytes_time_total_window 73714663 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_read_bytes_total # TYPE jindofsx_ns_remote_read_bytes_total gauge jindofsx_ns_remote_read_bytes_total 112108045498 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_bytes_total_window # TYPE jindofsx_ns_remote_read_bytes_total_window gauge jindofsx_ns_remote_read_bytes_total_window 13380884043 #\u4e00\u5206\u949f\u5185\u8fdc\u7aef\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_remote_read_time_total # TYPE jindofsx_ns_remote_read_time_total gauge jindofsx_ns_remote_read_time_total 765272359 #\u603b\u8fdc\u7aef\u8bfb\u53d6\u65f6\u95f4\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_remote_readop_num_total # TYPE jindofsx_ns_remote_readop_num_total gauge jindofsx_ns_remote_readop_num_total 107314 #\u8fdc\u7aef\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_slicelet_read_bytes_time_total_window # TYPE jindofsx_ns_slicelet_read_bytes_time_total_window gauge jindofsx_ns_slicelet_read_bytes_time_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u6240\u7528\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_read_bytes_total # TYPE jindofsx_ns_slicelet_read_bytes_total gauge jindofsx_ns_slicelet_read_bytes_total 0 #\u603b\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_bytes_total_window # TYPE jindofsx_ns_slicelet_read_bytes_total_window gauge jindofsx_ns_slicelet_read_bytes_total_window 0 #\u4e00\u5206\u949f\u5185\u5c0f\u6587\u4ef6\u8bfb\u53d6\u603b\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_slicelet_read_time_total # TYPE jindofsx_ns_slicelet_read_time_total gauge jindofsx_ns_slicelet_read_time_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u53d6\u6570\u636e\u6240\u7528\u603b\u65f6\u95f4\u5927\u5c0f\uff0c\u5355\u4f4d\u5fae\u79d2 # HELP jindofsx_ns_slicelet_readop_num_total # TYPE jindofsx_ns_slicelet_readop_num_total gauge jindofsx_ns_slicelet_readop_num_total 0 #\u5c0f\u6587\u4ef6\u8bfb\u603b\u8bfb\u53d6\u6570\u636e\u6b21\u6570\uff0c\u5bf9\u5e94 JindoFSx \u4e2d Block \u4e2a\u6570 # HELP jindofsx_ns_total_disk_cap # TYPE jindofsx_ns_total_disk_cap gauge jindofsx_ns_total_disk_cap 840739848192 #\u78c1\u76d8\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_mem_cap # TYPE jindofsx_ns_total_mem_cap gauge jindofsx_ns_total_mem_cap 0 #\u5185\u5b58\u7f13\u5b58\u603b\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_stsnodes_num # TYPE jindofsx_ns_total_stsnodes_num gauge jindofsx_ns_total_stsnodes_num 3 #alive worker \u8282\u70b9\u7684\u6570\u91cf # HELP jindofsx_ns_total_used_disk_cap # TYPE jindofsx_ns_total_used_disk_cap gauge jindofsx_ns_total_used_disk_cap 153423446016 #\u5df2\u7528\u78c1\u76d8\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte # HELP jindofsx_ns_total_used_mem_cap # TYPE jindofsx_ns_total_used_mem_cap gauge jindofsx_ns_total_used_mem_cap 0 #\u5df2\u7528\u5185\u5b58\u7f13\u5b58\u5bb9\u91cf\u5927\u5c0f\uff0c\u5355\u4f4dByte","title":"JindoRuntime Prometheus \u6307\u6807\u9884\u89c8(\u66f4\u65b0\u4e2d)"},{"location":"fluid-jindoruntime/jindo_fluid_nas_ufs_example/","text":"\u52a0\u901fOSS\u4e0a\u6570\u636e \u00a7 \u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002 1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001 \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907 \u00a7 5.1 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: nas spec: mounts: - mountPoint: local:///mnt/nas name: data path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: nas spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d NAS \u7684\u8def\u5f84\uff0c local:// \u4e3a\u672c\u5730\u8def\u5f84\u524d\u7f00\uff0c /mnt/nas \u8868\u793a\u672c\u5730\u771f\u5b9e\u8def\u5f84\uff0c\u8be5\u8def\u5f84\u9700\u8981\u5728\u5404\u4e2a\u8282\u70b9\u4e0a\u90fd\u5b58\u5728\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JinRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime nas NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE nas Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset nas NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE nas 511MiB 0.00B 180.00GiB 0.0% Bound 1h \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/nas 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/nas Bound hadoop 100Gi RWX 58m 6\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime nas \u5220\u9664 dataset kubectl delete dataset nas","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_nas_ufs_example/#oss","text":"\u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_nas_ufs_example/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"fluid-jindoruntime/jindo_fluid_nas_ufs_example/#2-fluid","text":"","title":"2\u3001\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"fluid-jindoruntime/jindo_fluid_nas_ufs_example/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"fluid-jindoruntime/jindo_fluid_nas_ufs_example/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"fluid-jindoruntime/jindo_fluid_nas_ufs_example/#5","text":"","title":"5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907"},{"location":"fluid-jindoruntime/jindo_fluid_nas_ufs_example/#51-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: nas spec: mounts: - mountPoint: local:///mnt/nas name: data path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: nas spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d NAS \u7684\u8def\u5f84\uff0c local:// \u4e3a\u672c\u5730\u8def\u5f84\u524d\u7f00\uff0c /mnt/nas \u8868\u793a\u672c\u5730\u771f\u5b9e\u8def\u5f84\uff0c\u8be5\u8def\u5f84\u9700\u8981\u5728\u5404\u4e2a\u8282\u70b9\u4e0a\u90fd\u5b58\u5728\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JinRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime nas NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE nas Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset nas NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE nas 511MiB 0.00B 180.00GiB 0.0% Bound 1h \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/nas 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/nas Bound hadoop 100Gi RWX 58m","title":"5.1 \u521b\u5efa dataset \u548c JindoRuntime"},{"location":"fluid-jindoruntime/jindo_fluid_nas_ufs_example/#6","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime nas \u5220\u9664 dataset kubectl delete dataset nas","title":"6\u3001\u73af\u5883\u6e05\u7406"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/","text":"\u52a0\u901f OSS-HDFS \u4e0a\u6570\u636e \u00a7 \u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS-HDFS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002 1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001 \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907 \u00a7 5.1 \u51c6\u5907 OSS-HDFS \u670d\u52a1 \u00a7 \u60a8\u9700\u8981\u5f00\u901a\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1(Object Storage Service\uff0c\u7b80\u79f0OSS)\uff0c\u6309\u7167\u4f7f\u7528\u6587\u6863\u8fdb\u884cbucket\u521b\u5efa\uff0c\u5e76\u5728\u521b\u5efa\u65f6\u6307\u5b9abucket\u4e3a\u652f\u6301 OSS-HDFS \u6a21\u5f0f\uff0c\u8bbf\u95ee\u94fe\u63a5 https://github.com../../tree/master/docsdocs/user/4.x/4.6.x/4.6.12/jindofs/jindo_dls_quickstart.md 5.2 \u51c6\u5907\u6d4b\u8bd5\u6837\u4f8b\u6570\u636e \u00a7 \u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 Apache \u955c\u50cf\u7ad9\u70b9\u4e0a\u7684 Spark \u76f8\u5173\u8d44\u6e90\u4f5c\u4e3a\u6f14\u793a\u4e2d\u4f7f\u7528\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002\u8fd9\u4e2a\u9009\u62e9\u5e76\u6ca1\u6709\u4efb\u4f55\u7279\u6b8a\u4e4b\u5904\uff0c\u4f60\u53ef\u4ee5\u5c06\u8fd9\u4e2a\u8fdc\u7a0b\u6587\u4ef6\u4fee\u6539\u4e3a\u4efb\u610f\u4f60\u559c\u6b22\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002 \u73b0\u5728\u8fdc\u7a0b\u8d44\u6e90\u6587\u4ef6\u5230\u672c\u5730 mkdir tmp cd tmp wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz \u4e0a\u4f20\u672c\u5730\u6587\u4ef6\u5230 OSS-HDFS \u4e0a \u60a8\u53ef\u4ee5\u4f7f\u7528 JindoSDK \uff0c\u6309\u7167\u4f7f\u7528\u8bf4\u660e\u5c06\u672c\u5730\u4e0b\u8f7d\u7684\u6587\u4ef6\u4e0a\u4f20\u5230\u8fdc\u7a0b OSS-HDFS \u7684bucket\u4e0a\u3002\u5982\u60a8\u7684bucket\u540d\u5b57\u4e3a test-bucket \u4e14\u4f7f\u7528 JindoSDK \u4f5c\u4e3a\u5ba2\u6237\u7aef\u5de5\u5177\uff0c\u53ef\u6267\u884c\u5982\u4e0b\u547d\u4ee4\u4e0a\u4f20\u6587\u4ef6\u5230 OSS-HDFS \u4e0a\u9762\u3002 hadoop fs -put spark-2.4.7-bin-hadoop2.7.tgz oss://test-bucket/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz hadoop fs -put spark-3.1.1-bin-hadoop3.2.tgz oss://test-bucket/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz 5.3 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: v1 kind: Secret metadata: name: mysecret stringData: fs.oss.accessKeyId: <OSS_HDFS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_HDFS_ACCESS_KEY_SECRET> --- apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/dir/ options: fs.oss.endpoint: <OSS_HDFS_ENDPOINT> name: hadoop path: / encryptOptions: - name: fs.oss.accessKeyId valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeyId - name: fs.oss.accessKeySecret valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeySecret accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aOSS-HDFS bucket\u7684endpoint\u4fe1\u606f\uff0c\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08cn-xxx.oss-dls.aliyuncs.com\uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08oss-cn-xxx.aliyuncs.com\uff09\u4e0d\u540c\uff0c\u5982\u60a8\u7684\u670d\u52a1\u5728 hangzhou\uff0c\u90a3\u4e48\u5bf9\u5e94\u7684 endpoint \u4e3a\uff1acn-hangzhou.oss-dls.aliyuncs.com replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JindoRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hadoop Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE HCFS URL TOTAL FILES CACHE HIT RATIO AGE hadoop [Calculating] 0.00B 200.00GiB Bound hadoop-jindofs-master-0.default:18844 [Calculating] 6m28s \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m 6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: hadoop \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u67e5\u770b\u6587\u4ef6\u5927\u5c0f $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 \u8fdb\u884c\u6587\u4ef6\u7684cp\u89c2\u5bdf\u65f6\u95f4\u6d88\u8017\u4e8618s $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m18.386s user 0m0.002s sys 0m0.105s \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 oss \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: hadoop-dataload spec: loadMetadata: true dataset: name: hadoop namespace: default \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload spark-dataload NAME DATASET PHASE AGE hadoop-dataload hadoop Completed 2m13s \u67e5\u770b\u6b64\u65f6 dataset \u7684\u7f13\u5b58\u60c5\u51b5\uff0c\u53d1\u73b0210MB\u7684\u6570\u636e\u5df2\u7ecf\u90fd\u7f13\u5b58\u5230\u4e86\u672c\u5730\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hadoop 210.00MiB 210.00MiB 180.00GiB 100.0% Bound 1h \u4e3a\u4e86\u907f\u514d\u5176\u4ed6\u56e0\u7d20(\u6bd4\u5982page cache)\u5bf9\u7ed3\u679c\u9020\u6210\u5f71\u54cd\uff0c\u6211\u4eec\u5c06\u5220\u9664\u4e4b\u524d\u7684\u5bb9\u5668\uff0c\u65b0\u5efa\u76f8\u540c\u7684\u5e94\u7528\uff0c\u5c1d\u8bd5\u8bbf\u95ee\u540c\u6837\u7684\u6587\u4ef6\u3002\u7531\u4e8e\u6b64\u65f6\u6587\u4ef6\u5df2\u7ecf\u88ab JindoFSx \u7f13\u5b58\uff0c\u53ef\u4ee5\u770b\u5230\u7b2c\u4e8c\u6b21\u8bbf\u95ee\u6240\u9700\u65f6\u95f4\u8fdc\u5c0f\u4e8e\u7b2c\u4e00\u6b21\u3002 kubectl delete -f app.yaml && kubectl create -f app.yaml \u8fdb\u884c\u6587\u4ef6\u7684\u62f7\u8d1d\u89c2\u5bdf\u65f6\u95f4\uff0c\u53d1\u73b0\u6d88\u801748ms\uff0c\u6574\u4e2a\u62f7\u8d1d\u7684\u65f6\u95f4\u7f29\u77ed\u4e86300\u500d $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m0.048s user 0m0.001s sys 0m0.046s 7\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"\u52a0\u901f OSS-HDFS \u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#oss-hdfs","text":"\u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS-HDFS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002","title":"\u52a0\u901f OSS-HDFS \u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#2-fluid","text":"","title":"2\u3001\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#5","text":"","title":"5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#51-oss-hdfs","text":"\u60a8\u9700\u8981\u5f00\u901a\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1(Object Storage Service\uff0c\u7b80\u79f0OSS)\uff0c\u6309\u7167\u4f7f\u7528\u6587\u6863\u8fdb\u884cbucket\u521b\u5efa\uff0c\u5e76\u5728\u521b\u5efa\u65f6\u6307\u5b9abucket\u4e3a\u652f\u6301 OSS-HDFS \u6a21\u5f0f\uff0c\u8bbf\u95ee\u94fe\u63a5 https://github.com../../tree/master/docsdocs/user/4.x/4.6.x/4.6.12/jindofs/jindo_dls_quickstart.md","title":"5.1 \u51c6\u5907 OSS-HDFS \u670d\u52a1"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#52","text":"\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 Apache \u955c\u50cf\u7ad9\u70b9\u4e0a\u7684 Spark \u76f8\u5173\u8d44\u6e90\u4f5c\u4e3a\u6f14\u793a\u4e2d\u4f7f\u7528\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002\u8fd9\u4e2a\u9009\u62e9\u5e76\u6ca1\u6709\u4efb\u4f55\u7279\u6b8a\u4e4b\u5904\uff0c\u4f60\u53ef\u4ee5\u5c06\u8fd9\u4e2a\u8fdc\u7a0b\u6587\u4ef6\u4fee\u6539\u4e3a\u4efb\u610f\u4f60\u559c\u6b22\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002 \u73b0\u5728\u8fdc\u7a0b\u8d44\u6e90\u6587\u4ef6\u5230\u672c\u5730 mkdir tmp cd tmp wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz \u4e0a\u4f20\u672c\u5730\u6587\u4ef6\u5230 OSS-HDFS \u4e0a \u60a8\u53ef\u4ee5\u4f7f\u7528 JindoSDK \uff0c\u6309\u7167\u4f7f\u7528\u8bf4\u660e\u5c06\u672c\u5730\u4e0b\u8f7d\u7684\u6587\u4ef6\u4e0a\u4f20\u5230\u8fdc\u7a0b OSS-HDFS \u7684bucket\u4e0a\u3002\u5982\u60a8\u7684bucket\u540d\u5b57\u4e3a test-bucket \u4e14\u4f7f\u7528 JindoSDK \u4f5c\u4e3a\u5ba2\u6237\u7aef\u5de5\u5177\uff0c\u53ef\u6267\u884c\u5982\u4e0b\u547d\u4ee4\u4e0a\u4f20\u6587\u4ef6\u5230 OSS-HDFS \u4e0a\u9762\u3002 hadoop fs -put spark-2.4.7-bin-hadoop2.7.tgz oss://test-bucket/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz hadoop fs -put spark-3.1.1-bin-hadoop3.2.tgz oss://test-bucket/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz","title":"5.2 \u51c6\u5907\u6d4b\u8bd5\u6837\u4f8b\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#53-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: v1 kind: Secret metadata: name: mysecret stringData: fs.oss.accessKeyId: <OSS_HDFS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_HDFS_ACCESS_KEY_SECRET> --- apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/dir/ options: fs.oss.endpoint: <OSS_HDFS_ENDPOINT> name: hadoop path: / encryptOptions: - name: fs.oss.accessKeyId valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeyId - name: fs.oss.accessKeySecret valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeySecret accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aOSS-HDFS bucket\u7684endpoint\u4fe1\u606f\uff0c\u8bbf\u95ee OSS Bucket \u4e0a OSS-HDFS \u670d\u52a1\u9700\u8981\u914d\u7f6e Endpoint\uff08cn-xxx.oss-dls.aliyuncs.com\uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08oss-cn-xxx.aliyuncs.com\uff09\u4e0d\u540c\uff0c\u5982\u60a8\u7684\u670d\u52a1\u5728 hangzhou\uff0c\u90a3\u4e48\u5bf9\u5e94\u7684 endpoint \u4e3a\uff1acn-hangzhou.oss-dls.aliyuncs.com replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JindoRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hadoop Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE HCFS URL TOTAL FILES CACHE HIT RATIO AGE hadoop [Calculating] 0.00B 200.00GiB Bound hadoop-jindofs-master-0.default:18844 [Calculating] 6m28s \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m","title":"5.3 \u521b\u5efa dataset \u548c JindoRuntime"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#6","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: hadoop \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u67e5\u770b\u6587\u4ef6\u5927\u5c0f $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 \u8fdb\u884c\u6587\u4ef6\u7684cp\u89c2\u5bdf\u65f6\u95f4\u6d88\u8017\u4e8618s $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m18.386s user 0m0.002s sys 0m0.105s \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 oss \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: hadoop-dataload spec: loadMetadata: true dataset: name: hadoop namespace: default \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload spark-dataload NAME DATASET PHASE AGE hadoop-dataload hadoop Completed 2m13s \u67e5\u770b\u6b64\u65f6 dataset \u7684\u7f13\u5b58\u60c5\u51b5\uff0c\u53d1\u73b0210MB\u7684\u6570\u636e\u5df2\u7ecf\u90fd\u7f13\u5b58\u5230\u4e86\u672c\u5730\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hadoop 210.00MiB 210.00MiB 180.00GiB 100.0% Bound 1h \u4e3a\u4e86\u907f\u514d\u5176\u4ed6\u56e0\u7d20(\u6bd4\u5982page cache)\u5bf9\u7ed3\u679c\u9020\u6210\u5f71\u54cd\uff0c\u6211\u4eec\u5c06\u5220\u9664\u4e4b\u524d\u7684\u5bb9\u5668\uff0c\u65b0\u5efa\u76f8\u540c\u7684\u5e94\u7528\uff0c\u5c1d\u8bd5\u8bbf\u95ee\u540c\u6837\u7684\u6587\u4ef6\u3002\u7531\u4e8e\u6b64\u65f6\u6587\u4ef6\u5df2\u7ecf\u88ab JindoFSx \u7f13\u5b58\uff0c\u53ef\u4ee5\u770b\u5230\u7b2c\u4e8c\u6b21\u8bbf\u95ee\u6240\u9700\u65f6\u95f4\u8fdc\u5c0f\u4e8e\u7b2c\u4e00\u6b21\u3002 kubectl delete -f app.yaml && kubectl create -f app.yaml \u8fdb\u884c\u6587\u4ef6\u7684\u62f7\u8d1d\u89c2\u5bdf\u65f6\u95f4\uff0c\u53d1\u73b0\u6d88\u801748ms\uff0c\u6574\u4e2a\u62f7\u8d1d\u7684\u65f6\u95f4\u7f29\u77ed\u4e86300\u500d $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m0.048s user 0m0.001s sys 0m0.046s","title":"6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c"},{"location":"fluid-jindoruntime/jindo_fluid_oss_hdfs_ufs_example/#7","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"7\u3001\u73af\u5883\u6e05\u7406"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/","text":"\u52a0\u901fOSS\u4e0a\u6570\u636e \u00a7 \u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002 1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001 \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907 \u00a7 5.1 \u51c6\u5907 OSS \u670d\u52a1 \u00a7 \u60a8\u9700\u8981\u5f00\u901a\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1(Object Storage Service\uff0c\u7b80\u79f0OSS)\uff0c\u6309\u7167\u4f7f\u7528\u6587\u6863\u8fdb\u884cbucket\u521b\u5efa( https://www.aliyun.com/product/oss ) 5.2 \u51c6\u5907\u6d4b\u8bd5\u6837\u4f8b\u6570\u636e \u00a7 \u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 Apache \u955c\u50cf\u7ad9\u70b9\u4e0a\u7684 Spark \u76f8\u5173\u8d44\u6e90\u4f5c\u4e3a\u6f14\u793a\u4e2d\u4f7f\u7528\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002\u8fd9\u4e2a\u9009\u62e9\u5e76\u6ca1\u6709\u4efb\u4f55\u7279\u6b8a\u4e4b\u5904\uff0c\u4f60\u53ef\u4ee5\u5c06\u8fd9\u4e2a\u8fdc\u7a0b\u6587\u4ef6\u4fee\u6539\u4e3a\u4efb\u610f\u4f60\u559c\u6b22\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002 \u73b0\u5728\u8fdc\u7a0b\u8d44\u6e90\u6587\u4ef6\u5230\u672c\u5730 mkdir tmp cd tmp wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz \u4e0a\u4f20\u672c\u5730\u6587\u4ef6\u5230 OSS \u4e0a \u60a8\u53ef\u4ee5\u4f7f\u7528 OSS \u63d0\u4f9b\u7684\u5ba2\u6237\u7aef ossutil \u6216\u8005\u963f\u91cc\u4e91EMR\u56e2\u961f\u63d0\u4f9b\u7684 JindoSDK \uff0c\u6309\u7167\u4f7f\u7528\u8bf4\u660e\u5c06\u672c\u5730\u4e0b\u8f7d\u7684\u6587\u4ef6\u4e0a\u4f20\u5230\u8fdc\u7a0b OSS \u7684bucket\u4e0a\u3002\u5982\u60a8\u7684bucket\u540d\u5b57\u4e3a test-bucket \u4e14\u4f7f\u7528 JindoSDK \u4f5c\u4e3a\u5ba2\u6237\u7aef\u5de5\u5177\uff0c\u53ef\u6267\u884c\u5982\u4e0b\u547d\u4ee4\u4e0a\u4f20\u6587\u4ef6\u5230 OSS \u4e0a\u9762\u3002 hadoop fs -put spark-2.4.7-bin-hadoop2.7.tgz oss://test-bucket/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz hadoop fs -put spark-3.1.1-bin-hadoop3.2.tgz oss://test-bucket/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz 5.3 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: v1 kind: Secret metadata: name: mysecret stringData: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> --- apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/dir/ options: fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop path: / encryptOptions: - name: fs.oss.accessKeyId valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeyId - name: fs.oss.accessKeySecret valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeySecret accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aoss bucket\u7684endpoint\u4fe1\u606f\uff0c\u516c\u7f51\u6216\u5185\u7f51\u5730\u5740\u7686\u53ef\uff0c\u5982\u60a8\u7684 bucket \u5728\u676d\u5dde Region\uff0c\u90a3\u4e48\u516c\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou.aliyuncs.com\uff0c\u5185\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou-internal.aliyuncs.com(\u5185\u7f51\u5730\u5740\u4f7f\u7528\u6761\u4ef6\u4e3a\u60a8\u7684 K8S \u96c6\u7fa4\u6240\u5728\u533a\u57df\u548coss\u533a\u57df\u76f8\u540c)\u3002 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JindoRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hadoop Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE HCFS URL TOTAL FILES CACHE HIT RATIO AGE hadoop [Calculating] 0.00B 200.00GiB Bound hadoop-jindofs-master-0.default:18844 [Calculating] 6m28s \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m 6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: hadoop \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u67e5\u770b\u6587\u4ef6\u5927\u5c0f $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 \u8fdb\u884c\u6587\u4ef6\u7684cp\u89c2\u5bdf\u65f6\u95f4\u6d88\u8017\u4e8618s $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m18.386s user 0m0.002s sys 0m0.105s \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 oss \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: hadoop-dataload spec: loadMetadata: true dataset: name: hadoop namespace: default \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload spark-dataload NAME DATASET PHASE AGE hadoop-dataload hadoop Completed 2m13s \u67e5\u770b\u6b64\u65f6 dataset \u7684\u7f13\u5b58\u60c5\u51b5\uff0c\u53d1\u73b0210MB\u7684\u6570\u636e\u5df2\u7ecf\u90fd\u7f13\u5b58\u5230\u4e86\u672c\u5730\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hadoop 210.00MiB 210.00MiB 180.00GiB 100.0% Bound 1h \u4e3a\u4e86\u907f\u514d\u5176\u4ed6\u56e0\u7d20(\u6bd4\u5982page cache)\u5bf9\u7ed3\u679c\u9020\u6210\u5f71\u54cd\uff0c\u6211\u4eec\u5c06\u5220\u9664\u4e4b\u524d\u7684\u5bb9\u5668\uff0c\u65b0\u5efa\u76f8\u540c\u7684\u5e94\u7528\uff0c\u5c1d\u8bd5\u8bbf\u95ee\u540c\u6837\u7684\u6587\u4ef6\u3002\u7531\u4e8e\u6b64\u65f6\u6587\u4ef6\u5df2\u7ecf\u88ab JindoFSx \u7f13\u5b58\uff0c\u53ef\u4ee5\u770b\u5230\u7b2c\u4e8c\u6b21\u8bbf\u95ee\u6240\u9700\u65f6\u95f4\u8fdc\u5c0f\u4e8e\u7b2c\u4e00\u6b21\u3002 kubectl delete -f app.yaml && kubectl create -f app.yaml \u8fdb\u884c\u6587\u4ef6\u7684\u62f7\u8d1d\u89c2\u5bdf\u65f6\u95f4\uff0c\u53d1\u73b0\u6d88\u801748ms\uff0c\u6574\u4e2a\u62f7\u8d1d\u7684\u65f6\u95f4\u7f29\u77ed\u4e86300\u500d $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m0.048s user 0m0.001s sys 0m0.046s 7\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#oss","text":"\u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#2-fluid","text":"","title":"2\u3001\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#5","text":"","title":"5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#51-oss","text":"\u60a8\u9700\u8981\u5f00\u901a\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1(Object Storage Service\uff0c\u7b80\u79f0OSS)\uff0c\u6309\u7167\u4f7f\u7528\u6587\u6863\u8fdb\u884cbucket\u521b\u5efa( https://www.aliyun.com/product/oss )","title":"5.1 \u51c6\u5907 OSS \u670d\u52a1"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#52","text":"\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 Apache \u955c\u50cf\u7ad9\u70b9\u4e0a\u7684 Spark \u76f8\u5173\u8d44\u6e90\u4f5c\u4e3a\u6f14\u793a\u4e2d\u4f7f\u7528\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002\u8fd9\u4e2a\u9009\u62e9\u5e76\u6ca1\u6709\u4efb\u4f55\u7279\u6b8a\u4e4b\u5904\uff0c\u4f60\u53ef\u4ee5\u5c06\u8fd9\u4e2a\u8fdc\u7a0b\u6587\u4ef6\u4fee\u6539\u4e3a\u4efb\u610f\u4f60\u559c\u6b22\u7684\u8fdc\u7a0b\u6587\u4ef6\u3002 \u73b0\u5728\u8fdc\u7a0b\u8d44\u6e90\u6587\u4ef6\u5230\u672c\u5730 mkdir tmp cd tmp wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz wget https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz \u4e0a\u4f20\u672c\u5730\u6587\u4ef6\u5230 OSS \u4e0a \u60a8\u53ef\u4ee5\u4f7f\u7528 OSS \u63d0\u4f9b\u7684\u5ba2\u6237\u7aef ossutil \u6216\u8005\u963f\u91cc\u4e91EMR\u56e2\u961f\u63d0\u4f9b\u7684 JindoSDK \uff0c\u6309\u7167\u4f7f\u7528\u8bf4\u660e\u5c06\u672c\u5730\u4e0b\u8f7d\u7684\u6587\u4ef6\u4e0a\u4f20\u5230\u8fdc\u7a0b OSS \u7684bucket\u4e0a\u3002\u5982\u60a8\u7684bucket\u540d\u5b57\u4e3a test-bucket \u4e14\u4f7f\u7528 JindoSDK \u4f5c\u4e3a\u5ba2\u6237\u7aef\u5de5\u5177\uff0c\u53ef\u6267\u884c\u5982\u4e0b\u547d\u4ee4\u4e0a\u4f20\u6587\u4ef6\u5230 OSS \u4e0a\u9762\u3002 hadoop fs -put spark-2.4.7-bin-hadoop2.7.tgz oss://test-bucket/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz hadoop fs -put spark-3.1.1-bin-hadoop3.2.tgz oss://test-bucket/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz","title":"5.2 \u51c6\u5907\u6d4b\u8bd5\u6837\u4f8b\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#53-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: v1 kind: Secret metadata: name: mysecret stringData: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> --- apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/dir/ options: fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop path: / encryptOptions: - name: fs.oss.accessKeyId valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeyId - name: fs.oss.accessKeySecret valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeySecret accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aoss bucket\u7684endpoint\u4fe1\u606f\uff0c\u516c\u7f51\u6216\u5185\u7f51\u5730\u5740\u7686\u53ef\uff0c\u5982\u60a8\u7684 bucket \u5728\u676d\u5dde Region\uff0c\u90a3\u4e48\u516c\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou.aliyuncs.com\uff0c\u5185\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou-internal.aliyuncs.com(\u5185\u7f51\u5730\u5740\u4f7f\u7528\u6761\u4ef6\u4e3a\u60a8\u7684 K8S \u96c6\u7fa4\u6240\u5728\u533a\u57df\u548coss\u533a\u57df\u76f8\u540c)\u3002 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JindoRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime hadoop NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE hadoop Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE HCFS URL TOTAL FILES CACHE HIT RATIO AGE hadoop [Calculating] 0.00B 200.00GiB Bound hadoop-jindofs-master-0.default:18844 [Calculating] 6m28s \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m","title":"5.3 \u521b\u5efa dataset \u548c JindoRuntime"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#6","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: hadoop \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u67e5\u770b\u6587\u4ef6\u5927\u5c0f $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 \u8fdb\u884c\u6587\u4ef6\u7684cp\u89c2\u5bdf\u65f6\u95f4\u6d88\u8017\u4e8618s $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m18.386s user 0m0.002s sys 0m0.105s \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 oss \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: hadoop-dataload spec: loadMetadata: true dataset: name: hadoop namespace: default \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload spark-dataload NAME DATASET PHASE AGE hadoop-dataload hadoop Completed 2m13s \u67e5\u770b\u6b64\u65f6 dataset \u7684\u7f13\u5b58\u60c5\u51b5\uff0c\u53d1\u73b0210MB\u7684\u6570\u636e\u5df2\u7ecf\u90fd\u7f13\u5b58\u5230\u4e86\u672c\u5730\u3002 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE hadoop 210.00MiB 210.00MiB 180.00GiB 100.0% Bound 1h \u4e3a\u4e86\u907f\u514d\u5176\u4ed6\u56e0\u7d20(\u6bd4\u5982page cache)\u5bf9\u7ed3\u679c\u9020\u6210\u5f71\u54cd\uff0c\u6211\u4eec\u5c06\u5220\u9664\u4e4b\u524d\u7684\u5bb9\u5668\uff0c\u65b0\u5efa\u76f8\u540c\u7684\u5e94\u7528\uff0c\u5c1d\u8bd5\u8bbf\u95ee\u540c\u6837\u7684\u6587\u4ef6\u3002\u7531\u4e8e\u6b64\u65f6\u6587\u4ef6\u5df2\u7ecf\u88ab JindoFSx \u7f13\u5b58\uff0c\u53ef\u4ee5\u770b\u5230\u7b2c\u4e8c\u6b21\u8bbf\u95ee\u6240\u9700\u65f6\u95f4\u8fdc\u5c0f\u4e8e\u7b2c\u4e00\u6b21\u3002 kubectl delete -f app.yaml && kubectl create -f app.yaml \u8fdb\u884c\u6587\u4ef6\u7684\u62f7\u8d1d\u89c2\u5bdf\u65f6\u95f4\uff0c\u53d1\u73b0\u6d88\u801748ms\uff0c\u6574\u4e2a\u62f7\u8d1d\u7684\u65f6\u95f4\u7f29\u77ed\u4e86300\u500d $ time cp /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 /dev/null real 0m0.048s user 0m0.001s sys 0m0.046s","title":"6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c"},{"location":"fluid-jindoruntime/jindo_fluid_oss_ufs_example/#7","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"7\u3001\u73af\u5883\u6e05\u7406"},{"location":"fluid-jindoruntime/jindo_fluid_overview/","text":"Fluid \u7ed3\u5408 JindoCache \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4ecb\u7ecd\u548c\u4f7f\u7528 \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 Kubernetes version > 1.14, \u652f\u6301CSI Golang 1.12+ Helm 3 \u5b89\u88c5\u5305\u4e0b\u8f7d \u00a7 \u4e0b\u8f7d\u9875\u9762 \u4f7f\u7528\u6587\u6863 \u00a7 \u52a0\u901fOSS\u4e0a\u6570\u636e \u52a0\u901fHDFS\u4e0a\u6570\u636e \u52a0\u901fOSS-HDFS\u4e0a\u6570\u636e \u52a0\u901fS3\u4e0a\u6570\u636e \u52a0\u901fNAS\u4e0a\u6570\u636e \u52a0\u901fPVC\u4e0a\u6570\u636e Fluid + JindoCache \u9ad8\u9636\u529f\u80fd\u4f7f\u7528 \u6570\u636e\u9884\u52a0\u8f7d \u76d1\u63a7\u53ca\u53ef\u89c2\u6d4b\u529f\u80fd \u95ee\u9898\u8bca\u65ad\u53ca\u5904\u7406 \u5bf9\u63a5\u963f\u91cc\u4e91SLS\u65e5\u5fd7\u670d\u52a1 \u6700\u4f73\u5b9e\u8df5\u7cfb\u5217 \u00a7 \u4e91\u539f\u751f AI \u5728\u81ea\u52a8\u9a7e\u9a76\u9886\u57df\u7684\u5e94\u7528\u63a2\u7d22\u4e0e\u6beb\u672b\u667a\u884c\u843d\u5730\u5b9e\u8df5 \u5fae\u535a\u6d77\u91cf\u6df1\u5ea6\u5b66\u4e60\u6a21\u578b\u8bad\u7ec3\u6548\u7387\u8dc3\u5347\u7684\u79d8\u5bc6 \u4f5c\u4e1a\u5e2e\u68c0\u7d22\u670d\u52a1\u57fa\u4e8e Fluid \u7684\u8ba1\u7b97\u5b58\u50a8\u5206\u79bb\u5b9e\u8df5","title":"Fluid JindoRuntime"},{"location":"fluid-jindoruntime/jindo_fluid_overview/#fluid-jindocache","text":"","title":"Fluid \u7ed3\u5408 JindoCache \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u4ecb\u7ecd\u548c\u4f7f\u7528"},{"location":"fluid-jindoruntime/jindo_fluid_overview/#_1","text":"Kubernetes version > 1.14, \u652f\u6301CSI Golang 1.12+ Helm 3","title":"\u73af\u5883\u8981\u6c42"},{"location":"fluid-jindoruntime/jindo_fluid_overview/#_2","text":"\u4e0b\u8f7d\u9875\u9762","title":"\u5b89\u88c5\u5305\u4e0b\u8f7d"},{"location":"fluid-jindoruntime/jindo_fluid_overview/#_3","text":"\u52a0\u901fOSS\u4e0a\u6570\u636e \u52a0\u901fHDFS\u4e0a\u6570\u636e \u52a0\u901fOSS-HDFS\u4e0a\u6570\u636e \u52a0\u901fS3\u4e0a\u6570\u636e \u52a0\u901fNAS\u4e0a\u6570\u636e \u52a0\u901fPVC\u4e0a\u6570\u636e Fluid + JindoCache \u9ad8\u9636\u529f\u80fd\u4f7f\u7528 \u6570\u636e\u9884\u52a0\u8f7d \u76d1\u63a7\u53ca\u53ef\u89c2\u6d4b\u529f\u80fd \u95ee\u9898\u8bca\u65ad\u53ca\u5904\u7406 \u5bf9\u63a5\u963f\u91cc\u4e91SLS\u65e5\u5fd7\u670d\u52a1","title":"\u4f7f\u7528\u6587\u6863"},{"location":"fluid-jindoruntime/jindo_fluid_overview/#_4","text":"\u4e91\u539f\u751f AI \u5728\u81ea\u52a8\u9a7e\u9a76\u9886\u57df\u7684\u5e94\u7528\u63a2\u7d22\u4e0e\u6beb\u672b\u667a\u884c\u843d\u5730\u5b9e\u8df5 \u5fae\u535a\u6d77\u91cf\u6df1\u5ea6\u5b66\u4e60\u6a21\u578b\u8bad\u7ec3\u6548\u7387\u8dc3\u5347\u7684\u79d8\u5bc6 \u4f5c\u4e1a\u5e2e\u68c0\u7d22\u670d\u52a1\u57fa\u4e8e Fluid \u7684\u8ba1\u7b97\u5b58\u50a8\u5206\u79bb\u5b9e\u8df5","title":"\u6700\u4f73\u5b9e\u8df5\u7cfb\u5217"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/","text":"\u52a0\u901fPVC\u4e0a\u6570\u636e \u00a7 \u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c PVC \u6570\u636e\u5377\u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002 1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001 \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907 \u00a7 5.1 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u4e3a\u5df2\u5b58\u5728\u7684PVC\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 testPVC\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: pvc spec: mounts: - mountPoint: pvc://testPVC name: data path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: pvc spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d pvc \u7684\u8def\u5f84\uff0c pvc:// \u4e3a\u6302\u8f7d\u672c\u5730volume\u524d\u7f00\uff0c testPVC \u8868\u793a\u9700\u8981\u52a0\u901f\u7684PVC\u7684\u540d\u5b57\uff0c\u9700\u8981\u6709\u8bbf\u95ee\u6743\u9650\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JinRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime pvc NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE pvc Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset pvc NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE pvc 511MiB 0.00B 180.00GiB 0.0% Bound 1h \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/pvc 100Gi ROX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/pvc Bound hadoop 100Gi RWX 58m 6\u3001\u6570\u636e\u9884\u70ed \u00a7 \u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf \u00a7 \u53ef\u4ee5\u8fdb\u884c\u6570\u636e\u9884\u70ed\u63d0\u524d\u5c06\u6587\u4ef6\u7f13\u5b58\u5230\u672c\u5730\uff0c\u5047\u8bbe\u9700\u8981\u52a0\u901f\u8be5 pvc \u4e0b\u9762\u7684\u4e24\u4e2a\u5b50\u76ee\u5f55 /data1 \u548c /data1 \u540c\u65f6\u5206\u522b\u6307\u5b9a\u7f13\u5b58\u526f\u672c\u6570\u91cf\uff0c\u53ef\u4ee5\u53c2\u8003\u5982\u4e0bdataload.yaml\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: test spec: dataset: name: pvc namespace: default loadMetadata: true target: - path: /data1/ replicas: 1 - path: /data2/ replicas: 2 kubectl create dataload.yaml \u89c2\u5bdf dataload \u72b6\u6001 kubectl get dataload test \u5177\u4f53 dataload \u7684\u529f\u80fd\u53ef\u53c2\u8003 \u6570\u636e\u9884\u52a0\u8f7d 7\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime \u548c dataset kubectl delete dataset pvc","title":"\u52a0\u901fPVC\u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/#pvc","text":"\u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c PVC \u6570\u636e\u5377\u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002","title":"\u52a0\u901fPVC\u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/#2-fluid","text":"","title":"2\u3001\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/#5","text":"","title":"5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/#51-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u4e3a\u5df2\u5b58\u5728\u7684PVC\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 testPVC\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: pvc spec: mounts: - mountPoint: pvc://testPVC name: data path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: pvc spec: replicas: 2 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d pvc \u7684\u8def\u5f84\uff0c pvc:// \u4e3a\u6302\u8f7d\u672c\u5730volume\u524d\u7f00\uff0c testPVC \u8868\u793a\u9700\u8981\u52a0\u901f\u7684PVC\u7684\u540d\u5b57\uff0c\u9700\u8981\u6709\u8bbf\u95ee\u6743\u9650\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JinRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime pvc NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE pvc Ready Ready 62m \u67e5\u770b dataset \u7684\u60c5\u51b5\uff0c\u73b0\u5b9e Bound \u72b6\u6001\u8868\u793a dataset \u7ed1\u5b9a\u6210\u529f\u3002 $ kubectl get dataset pvc NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE pvc 511MiB 0.00B 180.00GiB 0.0% Bound 1h \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/pvc 100Gi ROX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/pvc Bound hadoop 100Gi RWX 58m","title":"5.1 \u521b\u5efa dataset \u548c JindoRuntime"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/#6","text":"","title":"6\u3001\u6570\u636e\u9884\u70ed"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/#_1","text":"\u53ef\u4ee5\u8fdb\u884c\u6570\u636e\u9884\u70ed\u63d0\u524d\u5c06\u6587\u4ef6\u7f13\u5b58\u5230\u672c\u5730\uff0c\u5047\u8bbe\u9700\u8981\u52a0\u901f\u8be5 pvc \u4e0b\u9762\u7684\u4e24\u4e2a\u5b50\u76ee\u5f55 /data1 \u548c /data1 \u540c\u65f6\u5206\u522b\u6307\u5b9a\u7f13\u5b58\u526f\u672c\u6570\u91cf\uff0c\u53ef\u4ee5\u53c2\u8003\u5982\u4e0bdataload.yaml\uff1a apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: test spec: dataset: name: pvc namespace: default loadMetadata: true target: - path: /data1/ replicas: 1 - path: /data2/ replicas: 2 kubectl create dataload.yaml \u89c2\u5bdf dataload \u72b6\u6001 kubectl get dataload test \u5177\u4f53 dataload \u7684\u529f\u80fd\u53ef\u53c2\u8003 \u6570\u636e\u9884\u52a0\u8f7d","title":"\u8bbe\u7f6e\u6570\u636e\u52a0\u8f7d\u65f6\u7684\u7f13\u5b58\u526f\u672c\u6570\u91cf"},{"location":"fluid-jindoruntime/jindo_fluid_pvc_ufs_example/#7","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime \u548c dataset kubectl delete dataset pvc","title":"7\u3001\u73af\u5883\u6e05\u7406"},{"location":"fluid-jindoruntime/jindo_fluid_question/","text":"\u95ee\u9898\u8bca\u65ad\u53ca\u5904\u7406 \u00a7 \u5982\u60a8\u7684\u73af\u5883\u51fa\u73b0\u95ee\u9898\uff0c\u4e5f\u53ef\u5f00 ISSUE \u7ed9\u6211\u4eec\uff0c\u4f1a\u7b2c\u4e00\u65f6\u95f4\u5904\u7406 Fluid \u5b89\u88c5\u4f7f\u7528\u76f8\u5173\u95ee\u9898 \u00a7 1. \u4e3a\u4ec0\u4e48\u6211\u4f7f\u7528Helm\u5b89\u88c5fluid\u5931\u8d25\u4e86\uff1f \u00a7 \u56de\u7b54 :\u63a8\u8350\u6309\u7167\u5b89\u88c5\u6587\u6863\u4f9d\u6b21\u786e\u8ba4Fluid\u7ec4\u4ef6\u662f\u5426\u6b63\u5e38\u8fd0\u884c\u3002 Fluid\u5b89\u88c5\u6587\u6863\u662f\u4ee5 Helm 3 \u4e3a\u4f8b\u8fdb\u884c\u90e8\u7f72\u7684\u3002\u5982\u679c\u60a8\u4f7f\u7528 Helm 3 \u4ee5\u4e0b\u7684\u7248\u672c\u90e8\u7f72Fluid\uff0c \u5e76\u4e14\u9047\u5230\u4e86 CRD\u6ca1\u6709\u6b63\u5e38\u542f\u52a8 \u7684\u60c5\u51b5\uff0c\u8fd9\u53ef\u80fd\u662f\u56e0\u4e3a Helm 3 \u53ca\u5176\u4ee5\u4e0a\u7248\u672c\u4f1a\u5728 helm install \u7684\u65f6\u5019\u81ea\u52a8\u5b89\u88c5CRD\uff0c \u800c\u4f4e\u7248\u672c\u7684Helm\u5219\u4e0d\u4f1a\u3002 \u53c2\u89c1 Helm\u5b98\u65b9\u6587\u6863\u8bf4\u660e \u3002 \u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u60a8\u9700\u8981\u624b\u52a8\u5b89\u88c5CRD\uff1a $ kubectl create -f fluid/crds 2. \u4e3a\u4ec0\u4e48\u6211\u65e0\u6cd5\u5220\u9664Runtime\uff1f \u00a7 \u56de\u7b54 :\u8bf7\u68c0\u67e5\u76f8\u5173Pod\u8fd0\u884c\u72b6\u6001\u548cRuntime\u7684Events\u3002 \u53ea\u8981\u6709\u4efb\u4f55\u6d3b\u8dc3Pod\u8fd8\u5728\u4f7f\u7528Fluid\u521b\u5efa\u7684Volume\uff0cFluid\u5c31\u4e0d\u4f1a\u5b8c\u6210\u5220\u9664\u64cd\u4f5c\u3002 \u5982\u4e0b\u7684\u547d\u4ee4\u53ef\u4ee5\u5feb\u901f\u5730\u627e\u51fa\u8fd9\u4e9b\u6d3b\u8dc3Pod\uff0c\u4f7f\u7528\u65f6\u628a <dataset_name> \u548c <dataset_namespace> \u6362\u6210\u81ea\u5df1\u7684\u5373\u53ef\uff1a kubectl describe pvc <dataset_name> -n <dataset_namespace> | \\ awk '/^Mounted/ {flag=1}; /^Events/ {flag=0}; flag' | \\ awk 'NR==1 {print $3}; NR!=1 {print $1}' | \\ xargs -I {} kubectl get po {} | \\ grep -E \"Running|Terminating|Pending\" | \\ cut -d \" \" -f 1 3. \u4e3a\u4ec0\u4e48\u6211\u5728\u521b\u5efa\u4efb\u52a1\u6302\u8f7d Runtime \u521b\u5efa\u7684 PVC \u7684\u65f6\u5019\u51fa\u73b0 driver name fuse.csi.fluid.io not found in the list of registered CSI drivers \u9519\u8bef\uff1f \u00a7 \u56de\u7b54 :\u8bf7\u67e5\u770b\u4efb\u52a1\u88ab\u8c03\u5ea6\u8282\u70b9\u6240\u5728\u7684 kubelet \u914d\u7f6e\u662f\u5426\u4e3a\u9ed8\u8ba4 /var/lib/kubelet \u3002 \u9996\u5148\u901a\u8fc7\u547d\u4ee4\u67e5\u770bFluid\u7684CSI\u7ec4\u4ef6\u662f\u5426\u6b63\u5e38 \u5982\u4e0b\u7684\u547d\u4ee4\u53ef\u4ee5\u5feb\u901f\u5730\u627e\u51faPod\uff0c\u4f7f\u7528\u65f6\u628a <node_name> \u548c <fluid_namespace> \u6362\u6210\u81ea\u5df1\u7684\u5373\u53ef\uff1a kubectl get pod -n <fluid_namespace> | grep <node_name> # <pod_name> \u4e3a\u4e0a\u4e00\u6b65pod\u540d kubectl logs <pod_name> node-driver-registrar -n <fluid_namespace> kubectl logs <pod_name> plugins -n <fluid_namespace> \u5982\u679c\u4e0a\u8ff0\u6b65\u9aa4\u7684Log\u65e0\u9519\u8bef\uff0c\u8bf7\u67e5\u770bcsidriver\u5bf9\u8c61\u662f\u5426\u5b58\u5728\uff1a kubectl get csidriver \u5982\u679ccsidriver\u5bf9\u8c61\u5b58\u5728\uff0c\u8bf7\u67e5\u770b\u67e5\u770bcsi\u6ce8\u518c\u8282\u70b9\u662f\u5426\u5305\u542b <node_name> \uff1a kubectl get csinode | grep <node_name> \u5982\u679c\u4e0a\u8ff0\u547d\u4ee4\u65e0\u8f93\u51fa\uff0c\u67e5\u770b\u4efb\u52a1\u88ab\u8c03\u5ea6\u8282\u70b9\u6240\u5728\u7684 kubelet \u914d\u7f6e\u662f\u5426\u4e3a\u9ed8\u8ba4 /var/lib/kubelet \u3002\u56e0\u4e3aFluid\u7684CSI\u7ec4\u4ef6\u901a\u8fc7\u56fa\u5b9a\u5730\u5740\u7684socket\u6ce8\u518c\u5230kubelet\uff0c\u9ed8\u8ba4\u4e3a --csi-address=/var/lib/kubelet/csi-plugins/fuse.csi.fluid.io/csi.sock --kubelet-registration-path=/var/lib/kubelet/csi-plugins/fuse.csi.fluid.io/csi.sock \u3002 4. \u4e3a\u4ec0\u4e48\u66f4\u65b0\u4e86fluid\u540e\uff0c\u4f7f\u7528 kubectl get \u67e5\u8be2\u66f4\u65b0\u524d\u521b\u5efa\u7684dataset\uff0c\u53d1\u73b0\u76f8\u6bd4\u65b0\u5efa\u7684dataset\u7f3a\u5c11\u4e86\u67d0\u4e9b\u5b57\u6bb5\uff1f \u00a7 \u56de\u7b54 :\u7531\u4e8e\u6211\u4eec\u5728fluid\u7684\u5347\u7ea7\u8fc7\u7a0b\u4e2d\u53ef\u80fd\u66f4\u65b0\u4e86CRD\uff0c\u4f60\u5728\u65e7\u7248\u672c\u521b\u5efa\u7684dataset\uff0c\u4f1a\u5c06CRD\u4e2d\u65b0\u589e\u7684\u5b57\u6bb5\u8bbe\u7f6e\u4e3a\u7a7a \u4f8b\u5982\uff0c\u5982\u679c\u4f60\u4ecev0.4\u6216\u66f4\u65e9\u7248\u672c\u5347\u7ea7\uff0c\u90a3\u65f6\u5019\u7684dataset\u6ca1\u6709FileNum\u5b57\u6bb5 \u66f4\u65b0fluid\u540e\uff0c\u5982\u679c\u4f60\u4f7f\u7528 kubectl get \u547d\u4ee4\uff0c\u65e0\u6cd5\u67e5\u8be2\u5230\u8be5dataset\u7684FileNum \u4f60\u53ef\u4ee5\u91cd\u5efadataset\uff0c\u65b0\u5efa\u7684dataset\u4f1a\u6b63\u5e38\u663e\u793a\u8fd9\u4e9b\u5b57\u6bb5 5. \u4e3a\u4ec0\u4e48\u5728\u5e94\u7528\u7a0b\u5e8f\u4e2d\u4f7f\u7528 PVC \u65f6\u4f1a\u4ea7\u751f\u4e86 Volume Attachment \u8d85\u65f6\u95ee\u9898\uff1f \u00a7 \u56de\u7b54 :Volume Attachment \u8d85\u65f6\u95ee\u9898\u662f Kubelet \u8fdb\u884c\u8bf7\u6c42 CSI Driver \u65f6\u672a\u6536\u5230 CSI Driver \u7684\u54cd\u5e94\u800c\u9020\u6210\u7684\u8d85\u65f6\u3002 \u8be5\u95ee\u9898\u662f\u7531\u4e8e Fluid \u7684 CSI Driver \u6ca1\u6709\u5b89\u88c5\uff0c\u6216\u8005kubelet\u6ca1\u6709\u8bbf\u95ee CSI Driver \u7684\u6743\u9650\u5bfc\u81f4\u7684\u3002 \u7531\u4e8e CSI Driver \u662f\u7531 Kubelet \u8fdb\u884c\u56de\u8c03\uff0c\u4f46\u662f\u5982\u679c Fluid \u6ca1\u6709\u5b89\u88c5 CSI Driver \u6216\u8005 Kubelet \u6ca1\u6709\u6743\u9650\u67e5\u770b CSI Driver\uff0c\u5c31\u4f1a\u5bfc\u81f4 CSI Plugin \u6ca1\u6709\u88ab\u6b63\u786e\u89e6\u53d1\u3002 \u9996\u5148\u9700\u8981\u4f7f\u7528\u547d\u4ee4 kubectl get csidriver \u67e5\u770b\u662f\u5426\u5b89\u88c5\u4e86 CSI Driver\u3002 \u5982\u679c\u6ca1\u6709\u5b89\u88c5\uff0c\u4f7f\u7528\u547d\u4ee4 kubectl apply -f charts/fluid/fluid/templates/csi/driver.yaml \u8fdb\u884c\u5b89\u88c5\uff0c\u7136\u540e\u89c2\u5bdf PVC \u662f\u5426\u6210\u529f\u6302\u8f7d\u5230\u5e94\u7528\u7a0b\u5e8f\u4e2d\u3002 \u5982\u679c\u4ecd\u672a\u80fd\u89e3\u51b3\uff0c\u4f7f\u7528 export KUBECONFIG=/etc/kubernetes/kubelet.conf && kubectl get csidriver \u6765\u67e5\u770b Kubelet \u80fd\u591f\u5177\u6709\u6743\u9650\u770b\u5230 CSI Driver","title":"\u95ee\u9898\u8bca\u65ad\u53ca\u5904\u7406"},{"location":"fluid-jindoruntime/jindo_fluid_question/#_1","text":"\u5982\u60a8\u7684\u73af\u5883\u51fa\u73b0\u95ee\u9898\uff0c\u4e5f\u53ef\u5f00 ISSUE \u7ed9\u6211\u4eec\uff0c\u4f1a\u7b2c\u4e00\u65f6\u95f4\u5904\u7406","title":"\u95ee\u9898\u8bca\u65ad\u53ca\u5904\u7406"},{"location":"fluid-jindoruntime/jindo_fluid_question/#fluid","text":"","title":"Fluid \u5b89\u88c5\u4f7f\u7528\u76f8\u5173\u95ee\u9898"},{"location":"fluid-jindoruntime/jindo_fluid_question/#1-helmfluid","text":"\u56de\u7b54 :\u63a8\u8350\u6309\u7167\u5b89\u88c5\u6587\u6863\u4f9d\u6b21\u786e\u8ba4Fluid\u7ec4\u4ef6\u662f\u5426\u6b63\u5e38\u8fd0\u884c\u3002 Fluid\u5b89\u88c5\u6587\u6863\u662f\u4ee5 Helm 3 \u4e3a\u4f8b\u8fdb\u884c\u90e8\u7f72\u7684\u3002\u5982\u679c\u60a8\u4f7f\u7528 Helm 3 \u4ee5\u4e0b\u7684\u7248\u672c\u90e8\u7f72Fluid\uff0c \u5e76\u4e14\u9047\u5230\u4e86 CRD\u6ca1\u6709\u6b63\u5e38\u542f\u52a8 \u7684\u60c5\u51b5\uff0c\u8fd9\u53ef\u80fd\u662f\u56e0\u4e3a Helm 3 \u53ca\u5176\u4ee5\u4e0a\u7248\u672c\u4f1a\u5728 helm install \u7684\u65f6\u5019\u81ea\u52a8\u5b89\u88c5CRD\uff0c \u800c\u4f4e\u7248\u672c\u7684Helm\u5219\u4e0d\u4f1a\u3002 \u53c2\u89c1 Helm\u5b98\u65b9\u6587\u6863\u8bf4\u660e \u3002 \u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u60a8\u9700\u8981\u624b\u52a8\u5b89\u88c5CRD\uff1a $ kubectl create -f fluid/crds","title":"1. \u4e3a\u4ec0\u4e48\u6211\u4f7f\u7528Helm\u5b89\u88c5fluid\u5931\u8d25\u4e86\uff1f"},{"location":"fluid-jindoruntime/jindo_fluid_question/#2-runtime","text":"\u56de\u7b54 :\u8bf7\u68c0\u67e5\u76f8\u5173Pod\u8fd0\u884c\u72b6\u6001\u548cRuntime\u7684Events\u3002 \u53ea\u8981\u6709\u4efb\u4f55\u6d3b\u8dc3Pod\u8fd8\u5728\u4f7f\u7528Fluid\u521b\u5efa\u7684Volume\uff0cFluid\u5c31\u4e0d\u4f1a\u5b8c\u6210\u5220\u9664\u64cd\u4f5c\u3002 \u5982\u4e0b\u7684\u547d\u4ee4\u53ef\u4ee5\u5feb\u901f\u5730\u627e\u51fa\u8fd9\u4e9b\u6d3b\u8dc3Pod\uff0c\u4f7f\u7528\u65f6\u628a <dataset_name> \u548c <dataset_namespace> \u6362\u6210\u81ea\u5df1\u7684\u5373\u53ef\uff1a kubectl describe pvc <dataset_name> -n <dataset_namespace> | \\ awk '/^Mounted/ {flag=1}; /^Events/ {flag=0}; flag' | \\ awk 'NR==1 {print $3}; NR!=1 {print $1}' | \\ xargs -I {} kubectl get po {} | \\ grep -E \"Running|Terminating|Pending\" | \\ cut -d \" \" -f 1","title":"2. \u4e3a\u4ec0\u4e48\u6211\u65e0\u6cd5\u5220\u9664Runtime\uff1f"},{"location":"fluid-jindoruntime/jindo_fluid_question/#3-runtime-pvc-driver-name-fusecsifluidio-not-found-in-the-list-of-registered-csi-drivers","text":"\u56de\u7b54 :\u8bf7\u67e5\u770b\u4efb\u52a1\u88ab\u8c03\u5ea6\u8282\u70b9\u6240\u5728\u7684 kubelet \u914d\u7f6e\u662f\u5426\u4e3a\u9ed8\u8ba4 /var/lib/kubelet \u3002 \u9996\u5148\u901a\u8fc7\u547d\u4ee4\u67e5\u770bFluid\u7684CSI\u7ec4\u4ef6\u662f\u5426\u6b63\u5e38 \u5982\u4e0b\u7684\u547d\u4ee4\u53ef\u4ee5\u5feb\u901f\u5730\u627e\u51faPod\uff0c\u4f7f\u7528\u65f6\u628a <node_name> \u548c <fluid_namespace> \u6362\u6210\u81ea\u5df1\u7684\u5373\u53ef\uff1a kubectl get pod -n <fluid_namespace> | grep <node_name> # <pod_name> \u4e3a\u4e0a\u4e00\u6b65pod\u540d kubectl logs <pod_name> node-driver-registrar -n <fluid_namespace> kubectl logs <pod_name> plugins -n <fluid_namespace> \u5982\u679c\u4e0a\u8ff0\u6b65\u9aa4\u7684Log\u65e0\u9519\u8bef\uff0c\u8bf7\u67e5\u770bcsidriver\u5bf9\u8c61\u662f\u5426\u5b58\u5728\uff1a kubectl get csidriver \u5982\u679ccsidriver\u5bf9\u8c61\u5b58\u5728\uff0c\u8bf7\u67e5\u770b\u67e5\u770bcsi\u6ce8\u518c\u8282\u70b9\u662f\u5426\u5305\u542b <node_name> \uff1a kubectl get csinode | grep <node_name> \u5982\u679c\u4e0a\u8ff0\u547d\u4ee4\u65e0\u8f93\u51fa\uff0c\u67e5\u770b\u4efb\u52a1\u88ab\u8c03\u5ea6\u8282\u70b9\u6240\u5728\u7684 kubelet \u914d\u7f6e\u662f\u5426\u4e3a\u9ed8\u8ba4 /var/lib/kubelet \u3002\u56e0\u4e3aFluid\u7684CSI\u7ec4\u4ef6\u901a\u8fc7\u56fa\u5b9a\u5730\u5740\u7684socket\u6ce8\u518c\u5230kubelet\uff0c\u9ed8\u8ba4\u4e3a --csi-address=/var/lib/kubelet/csi-plugins/fuse.csi.fluid.io/csi.sock --kubelet-registration-path=/var/lib/kubelet/csi-plugins/fuse.csi.fluid.io/csi.sock \u3002","title":"3. \u4e3a\u4ec0\u4e48\u6211\u5728\u521b\u5efa\u4efb\u52a1\u6302\u8f7d Runtime \u521b\u5efa\u7684 PVC \u7684\u65f6\u5019\u51fa\u73b0 driver name fuse.csi.fluid.io not found in the list of registered CSI drivers \u9519\u8bef\uff1f"},{"location":"fluid-jindoruntime/jindo_fluid_question/#4-fluid-kubectl-get-datasetdataset","text":"\u56de\u7b54 :\u7531\u4e8e\u6211\u4eec\u5728fluid\u7684\u5347\u7ea7\u8fc7\u7a0b\u4e2d\u53ef\u80fd\u66f4\u65b0\u4e86CRD\uff0c\u4f60\u5728\u65e7\u7248\u672c\u521b\u5efa\u7684dataset\uff0c\u4f1a\u5c06CRD\u4e2d\u65b0\u589e\u7684\u5b57\u6bb5\u8bbe\u7f6e\u4e3a\u7a7a \u4f8b\u5982\uff0c\u5982\u679c\u4f60\u4ecev0.4\u6216\u66f4\u65e9\u7248\u672c\u5347\u7ea7\uff0c\u90a3\u65f6\u5019\u7684dataset\u6ca1\u6709FileNum\u5b57\u6bb5 \u66f4\u65b0fluid\u540e\uff0c\u5982\u679c\u4f60\u4f7f\u7528 kubectl get \u547d\u4ee4\uff0c\u65e0\u6cd5\u67e5\u8be2\u5230\u8be5dataset\u7684FileNum \u4f60\u53ef\u4ee5\u91cd\u5efadataset\uff0c\u65b0\u5efa\u7684dataset\u4f1a\u6b63\u5e38\u663e\u793a\u8fd9\u4e9b\u5b57\u6bb5","title":"4. \u4e3a\u4ec0\u4e48\u66f4\u65b0\u4e86fluid\u540e\uff0c\u4f7f\u7528 kubectl get \u67e5\u8be2\u66f4\u65b0\u524d\u521b\u5efa\u7684dataset\uff0c\u53d1\u73b0\u76f8\u6bd4\u65b0\u5efa\u7684dataset\u7f3a\u5c11\u4e86\u67d0\u4e9b\u5b57\u6bb5\uff1f"},{"location":"fluid-jindoruntime/jindo_fluid_question/#5-pvc-volume-attachment","text":"\u56de\u7b54 :Volume Attachment \u8d85\u65f6\u95ee\u9898\u662f Kubelet \u8fdb\u884c\u8bf7\u6c42 CSI Driver \u65f6\u672a\u6536\u5230 CSI Driver \u7684\u54cd\u5e94\u800c\u9020\u6210\u7684\u8d85\u65f6\u3002 \u8be5\u95ee\u9898\u662f\u7531\u4e8e Fluid \u7684 CSI Driver \u6ca1\u6709\u5b89\u88c5\uff0c\u6216\u8005kubelet\u6ca1\u6709\u8bbf\u95ee CSI Driver \u7684\u6743\u9650\u5bfc\u81f4\u7684\u3002 \u7531\u4e8e CSI Driver \u662f\u7531 Kubelet \u8fdb\u884c\u56de\u8c03\uff0c\u4f46\u662f\u5982\u679c Fluid \u6ca1\u6709\u5b89\u88c5 CSI Driver \u6216\u8005 Kubelet \u6ca1\u6709\u6743\u9650\u67e5\u770b CSI Driver\uff0c\u5c31\u4f1a\u5bfc\u81f4 CSI Plugin \u6ca1\u6709\u88ab\u6b63\u786e\u89e6\u53d1\u3002 \u9996\u5148\u9700\u8981\u4f7f\u7528\u547d\u4ee4 kubectl get csidriver \u67e5\u770b\u662f\u5426\u5b89\u88c5\u4e86 CSI Driver\u3002 \u5982\u679c\u6ca1\u6709\u5b89\u88c5\uff0c\u4f7f\u7528\u547d\u4ee4 kubectl apply -f charts/fluid/fluid/templates/csi/driver.yaml \u8fdb\u884c\u5b89\u88c5\uff0c\u7136\u540e\u89c2\u5bdf PVC \u662f\u5426\u6210\u529f\u6302\u8f7d\u5230\u5e94\u7528\u7a0b\u5e8f\u4e2d\u3002 \u5982\u679c\u4ecd\u672a\u80fd\u89e3\u51b3\uff0c\u4f7f\u7528 export KUBECONFIG=/etc/kubernetes/kubelet.conf && kubectl get csidriver \u6765\u67e5\u770b Kubelet \u80fd\u591f\u5177\u6709\u6743\u9650\u770b\u5230 CSI Driver","title":"5. \u4e3a\u4ec0\u4e48\u5728\u5e94\u7528\u7a0b\u5e8f\u4e2d\u4f7f\u7528 PVC \u65f6\u4f1a\u4ea7\u751f\u4e86 Volume Attachment \u8d85\u65f6\u95ee\u9898\uff1f"},{"location":"fluid-jindoruntime/jindo_fluid_s3_ufs_example/","text":"\u62e5\u62b1\u4e91\u539f\u751f\uff0cFluid \u7ed3\u5408 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff1a\u52a0\u901f\u5bf9\u8c61\u5b58\u50a8\u4f7f\u7528 S3 \u534f\u8bae \u00a7 JindoFSx \u63d0\u4f9b\u4e86\u8bbf\u95ee S3 \u534f\u8bae\u7684\u80fd\u529b\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e S3 \u4f5c\u4e3a JindoFSx \u7684\u540e\u7aef\u5b58\u50a8\uff0c\u4f7f Fluid \u901a\u8fc7 JindoFSx \u6765\u8bbf\u95ee S3 \u4e0a\u7684\u6570\u636e\uff0c\u540c\u65f6 JindoFSx \u4e5f\u63d0\u4f9b\u4e86\u5bf9 S3 \u4e0a\u7684\u6570\u636e\u4ee5\u53ca\u5143\u6570\u636e\u7684\u7f13\u5b58\u52a0\u901f\u529f\u80fd\u3002 \u672c\u6587\u6863\u5c55\u793a\u4e86\u5982\u4f55\u5728 Fluid \u4e2d\u4ee5\u58f0\u660e\u5f0f\u7684\u65b9\u5f0f\u5b8c\u6210 JindoFSx \u90e8\u7f72\uff0c\u5bf9\u63a5 S3 \u6570\u636e\u6e90\uff08\u5305\u62ec\u63d0\u4f9b S3 \u517c\u5bb9\u7684\u6240\u6709\u5bf9\u8c61\u5b58\u50a8\uff09\u3002 \u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 \u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz \u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: s3 spec: mounts: - mountPoint: s3://<bucket>/<dir> options: fs.s3.accessKeyId: xxx fs.s3.accessKeySecret: xxx fs.s3.endpoint: s3.ap-east-1.amazonaws.com fs.s3.region: ap-east-1 name: s3 path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: s3 spec: replicas: 2 tieredstore: levels: - mediumtype: MEM path: /dev/shm quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d s3 \u7684\u8def\u5f84\uff0c\u652f\u6301\u6807\u51c6 s3 \u534f\u8bae\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u7684 worker \u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c\u5355\u4f4dGi\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b dataset \u7684\u60c5\u51b5 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE s3 210MiB 0.00B 180.00GiB 0.0% Bound 1h \u6267\u884c create \u547d\u4ee4 $ kubectl create -f runtime.yaml \u67e5\u770b dataset \u72b6\u6001 \u00a7 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE s3 180.01GiB 0.00B 261.00GiB 0.0% Bound 62m \u68c0\u67e5 PV\uff0cPVC \u521b\u5efa\u60c5\u51b5 \u00a7 $ kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/s3 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/s3 Bound hadoop 100Gi RWX 58m","title":"\u62e5\u62b1\u4e91\u539f\u751f\uff0cFluid \u7ed3\u5408 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff1a\u52a0\u901f\u5bf9\u8c61\u5b58\u50a8\u4f7f\u7528 S3 \u534f\u8bae"},{"location":"fluid-jindoruntime/jindo_fluid_s3_ufs_example/#fluid-jindofsx-s3","text":"JindoFSx \u63d0\u4f9b\u4e86\u8bbf\u95ee S3 \u534f\u8bae\u7684\u80fd\u529b\uff0c\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e S3 \u4f5c\u4e3a JindoFSx \u7684\u540e\u7aef\u5b58\u50a8\uff0c\u4f7f Fluid \u901a\u8fc7 JindoFSx \u6765\u8bbf\u95ee S3 \u4e0a\u7684\u6570\u636e\uff0c\u540c\u65f6 JindoFSx \u4e5f\u63d0\u4f9b\u4e86\u5bf9 S3 \u4e0a\u7684\u6570\u636e\u4ee5\u53ca\u5143\u6570\u636e\u7684\u7f13\u5b58\u52a0\u901f\u529f\u80fd\u3002 \u672c\u6587\u6863\u5c55\u793a\u4e86\u5982\u4f55\u5728 Fluid \u4e2d\u4ee5\u58f0\u660e\u5f0f\u7684\u65b9\u5f0f\u5b8c\u6210 JindoFSx \u90e8\u7f72\uff0c\u5bf9\u63a5 S3 \u6570\u636e\u6e90\uff08\u5305\u62ec\u63d0\u4f9b S3 \u517c\u5bb9\u7684\u6240\u6709\u5bf9\u8c61\u5b58\u50a8\uff09\u3002","title":"\u62e5\u62b1\u4e91\u539f\u751f\uff0cFluid \u7ed3\u5408 JindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff1a\u52a0\u901f\u5bf9\u8c61\u5b58\u50a8\u4f7f\u7528 S3 \u534f\u8bae"},{"location":"fluid-jindoruntime/jindo_fluid_s3_ufs_example/#_1","text":"kubectl create ns fluid-system","title":"\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"fluid-jindoruntime/jindo_fluid_s3_ufs_example/#fluid","text":"","title":"\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"fluid-jindoruntime/jindo_fluid_s3_ufs_example/#helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"fluid-jindoruntime/jindo_fluid_s3_ufs_example/#fluid_1","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"fluid-jindoruntime/jindo_fluid_s3_ufs_example/#dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: s3 spec: mounts: - mountPoint: s3://<bucket>/<dir> options: fs.s3.accessKeyId: xxx fs.s3.accessKeySecret: xxx fs.s3.endpoint: s3.ap-east-1.amazonaws.com fs.s3.region: ap-east-1 name: s3 path: / accessModes: - ReadOnlyMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: s3 spec: replicas: 2 tieredstore: levels: - mediumtype: MEM path: /dev/shm quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d s3 \u7684\u8def\u5f84\uff0c\u652f\u6301\u6807\u51c6 s3 \u534f\u8bae\u3002 accessModes: \u53ef\u9009 ReadOnlyMany / ReadWriteMany\uff0c\u524d\u8005\u4ee3\u8868\u53ea\u8bfb\uff0c\u540e\u8005\u4ee3\u8868\u53ef\u8bfb\u5199 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u7684 worker \u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c\u5355\u4f4dGi\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b dataset \u7684\u60c5\u51b5 $ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE s3 210MiB 0.00B 180.00GiB 0.0% Bound 1h \u6267\u884c create \u547d\u4ee4 $ kubectl create -f runtime.yaml","title":"\u521b\u5efa dataset \u548c JindoRuntime"},{"location":"fluid-jindoruntime/jindo_fluid_s3_ufs_example/#dataset","text":"$ kubectl get dataset hadoop NAME UFS TOTAL SIZE CACHED CACHE CAPACITY CACHED PERCENTAGE PHASE AGE s3 180.01GiB 0.00B 261.00GiB 0.0% Bound 62m","title":"\u67e5\u770b dataset \u72b6\u6001"},{"location":"fluid-jindoruntime/jindo_fluid_s3_ufs_example/#pvpvc","text":"$ kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/s3 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/s3 Bound hadoop 100Gi RWX 58m","title":"\u68c0\u67e5 PV\uff0cPVC \u521b\u5efa\u60c5\u51b5"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/","text":"Fluid + JindoCache \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u9ad8\u9636\u529f\u80fd\u4f7f\u7528 \u00a7 \u4f7f\u7528\u5217\u8868 \u6302\u8f7d\u70b9\u5728\u6839\u76ee\u5f55\u4e0b Secret \u52a0\u5bc6 AK \u53c2\u6570 Raft 3 master \u6a21\u5f0f \u4f7f\u7528 Placement \u90e8\u7f72\u591a\u4e2a runtime \u4f7f\u7528 NoseSelector \u90e8\u7f72\u8282\u70b9 \u4f7f\u7528 dataset nodeAffinity \u529f\u80fd Worker \u4e2a\u6570\u6269\u7f29\u5bb9 \u4f7f\u7528 tolerations \u529f\u80fd Resource \u8d44\u6e90 Fuse \u56de\u6536\u7b56\u7565 JindoCache Fuse \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528 FuseOnly \u4f7f\u7528\u65b9\u5f0f \u591a\u6302\u8f7d\u70b9 master\u5143\u6570\u636e\u6301\u4e45\u5316 \u66f4\u591a\u7f13\u5b58\u7b56\u7565 \u6302\u8f7d\u70b9\u5728\u6839\u76ee\u5f55\u4e0b \u00a7 \u9ed8\u8ba4\u4f7f\u7528 JindoRuntime \u4f1a\u5728\u6302\u8f7d\u70b9\u591a\u4e00\u5c42 / \u7684\u76ee\u5f55\uff0c\u5982\u679c\u60f3\u6302\u8f7d\u5728\u6839\u76ee\u5f55\u4e0b\u53ef\u4ee5\u5728 dataset \u91cc\u8fdb\u884c\u53c2\u6570\u6307\u5b9a apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase path: / \u6307\u5b9a spec.mounts.path = /\uff0c\u5219\u4f1a\u5c06\u6587\u4ef6\u6302\u8f7d\u5728\u6839\u76ee\u5f55\u4e0b Secret \u52a0\u5bc6 AK \u53c2\u6570 \u00a7 apiVersion: v1 kind: Secret metadata: name: mysecret stringData: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> \u5728 dataset \u91cc\u4f7f\u7528 secret apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://<OSS_BUCKET>/<OSS_DIRECTORY>/ name: hadoop options: fs.oss.endpoint: <OSS_ENDPOINT> encryptOptions: - name: fs.oss.accessKeyId valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeyId - name: fs.oss.accessKeySecret valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeySecret Raft 3 master \u6a21\u5f0f \u00a7 JindoRuntime \u6a21\u5f0f\u542f\u52a8\u4e00\u4e2a master\uff0c\u53ef\u4ee5\u901a\u8fc7\u4f7f\u7528 master.replicas \u6765\u542f\u52a8 3 \u4e2a master \u8fdb\u884c HA \u8f6c\u6362\u3002 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 3 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 2G high: \"0.8\" low: \"0.7\" master: replicas: 3 master.replicas = 3 : \u542f\u52a8 3 \u4e2a master pod \u4f7f\u7528 Placement \u90e8\u7f72\u591a\u4e2a runtime \u00a7 JindoRuntime \u542f\u52a8\u540e\uff0c\u9ed8\u8ba4 worker \u8282\u70b9\u4e0a\u53ea\u80fd\u542f\u52a8\u4e00\u4e2a runtime \u7684worker\uff0c\u5c5e\u4e8e\u72ec\u5360\u6a21\u5f0f\uff0c\u5982\u679c\u60f3\u8981\u5728\u540c\u4e00\u4e2a\u8282\u70b9\u4e0a\u652f\u6301\u90e8\u7f72\u591a\u4e2a worker\uff0c\u53ef\u4ee5\u4f7f\u7528 shared \u6a21\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase placement: \"Shared\" \u4f7f\u7528 NoseSelector \u90e8\u7f72\u8282\u70b9 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /mnt/disk1 quota: 10Gi high: \"0.9\" low: \"0.8\" master: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1 worker: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1 fuse: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1 \u4f7f\u7528 dataset nodeAffinity \u529f\u80fd \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: hbase-cache operator: In values: - \"true\" Worker \u4e2a\u6570\u6269\u7f29\u5bb9 \u00a7 \u4f7f\u7528 kubectl scale \u5bf9 Runtime \u7684 Worker \u6570\u91cf\u8fdb\u884c\u8c03\u6574 $ kubectl scale jindoruntime <runtime_name> --replicas=<replica_num> \u5176\u4e2d * runtime_name\uff1aruntime \u7684\u540d\u5b57 * replica_num\uff1a\u8868\u793a\u6269\u6240\u5bb9\u7684 Worker \u6570\u91cf \u4f7f\u7528 tolerations \u529f\u80fd \u00a7 \u60a8\u53ef\u4ee5\u5728 dataset \u91cc\u5b9a\u4e49 tolerations \u6307\u5b9a worker \u8282\u70b9\u7684\u8c03\u5ea6 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase tolerations: - key: hbase operator: Equal value: \"true\" \u4e5f\u53ef\u4ee5\u4e3a master/worker/fuse \u5355\u72ec\u6307\u5b9a tolerations \u8c03\u5ea6\u7b56\u7565 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 3 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 2Gi high: \"0.8\" low: \"0.7\" master: tolerations: - key: hbase operator: Equal value: \"true\" worker: tolerations: - key: hbase operator: Equal value: \"true\" fuse: tolerations: - key: hbase operator: Equal value: \"true\" Resource \u8d44\u6e90 \u00a7 \u53ef\u4ee5\u6307\u5b9a master/worker \u7b49\u7684 resource \u8d44\u6e90 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" master: resources: limits: cpu: \"4\" memory: \"8Gi\" requests: cpu: \"2\" memory: \"3Gi\" worker: resources: limits: cpu: \"4\" memory: \"8Gi\" requests: cpu: \"2\" memory: \"3Gi\" Fuse \u56de\u6536\u7b56\u7565 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: cleanPolicy: OnDemand fuse.cleanPolicy: onDemand / OnRuntimeDeleted \u5206\u522b\u8868\u793a\u6309\u9700\u542f\u52a8\uff0c\u4efb\u52a1\u7ed3\u675f fuse \u4e5f\u7ed3\u675f\u548c\u5728 runtime \u9500\u6bc1\u7684\u65f6\u5019\u548c\u5176\u4ed6\u7ec4\u4ef6\u4e00\u8d77\u9500\u6bc1\u6389 JindoCache Fuse \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528 \u00a7 \u53c2\u6570\u540d\u79f0 \u53c2\u6570\u8bf4\u660e \u4f7f\u7528\u8303\u4f8b auto_unmount fuse\u8fdb\u7a0b\u9000\u51fa\u540e\u81ea\u52a8umount\u6302\u8f7d\u8282\u70b9\u3002 -oauto_unmount ro \u53ea\u8bfb\u6302\u8f7d\uff0c\u542f\u7528\u53c2\u6570\u540e\u4e0d\u5141\u8bb8\u5199\u64cd\u4f5c\u3002 -oro direct_io \u5f00\u542f\u540e\uff0c\u8bfb\u5199\u6587\u4ef6\u53ef\u4ee5\u7ed5\u8fc7page cache\u3002 -odirect_io kernel_cache \u5f00\u542f\u540e\uff0c\u5229\u7528\u5185\u6838\u7f13\u5b58\u4f18\u5316\u8bfb\u6027\u80fd\u3002 -okernel_cache auto_cache \u9ed8\u8ba4\u5f00\u542f\uff0c\u4e0ekernel_cache \u4e8c\u9009\u4e00\uff0c\u4e0ekernel_cache\u4e0d\u540c\u7684\u662f\uff0c\u5982\u679c\u6587\u4ef6\u5927\u5c0f\u6216\u4fee\u6539\u65f6\u95f4\u53d1\u751f\u53d8\u5316\uff0c\u7f13\u5b58\u5c31\u4f1a\u5931\u6548\u3002 -oauto_cache entry_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-oentry_timeout=60 attr_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-oattr_timeout=60 negative_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u5931\u8d25\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-onegative_timeout=30 \u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: args: - -oro - -oattr_timeout=60 - -oentry_timeout=60 - -onegative_timeout=60 \u4ee5\u6570\u7ec4\u65b9\u5f0f\u5199\u5728 spec.fuse.args \u91cc\uff0c\u6309\u7167\u9700\u8981\u586b\u5199\u5373\u53ef \u5982\u5355\u72ec\u914d\u7f6e\u5ba2\u6237\u7aefpread\u53c2\u6570\uff1a apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: args: - -oro - -ometrics_port=0 - -okernel_cache - -oattr_timeout=7200 - -oentry_timeout=7200 - -onegative_timeout=7200 - -opread FuseOnly \u4f7f\u7528\u65b9\u5f0f \u00a7 \u5982\u60a8\u60f3\u4f7f\u7528 FuseOnly \u65b9\u5f0f\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: master: disabled: true worker: disabled: true \u591a\u6302\u8f7d\u70b9 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: spark - mountPoint: oss://test-bucket-2/dir2/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop \u5728\u6302\u8f7d pvc \u548c fuse \u4ee5\u53ca\u505a dataload \u7684\u65f6\u5019\uff0c\u8fd9\u4e24\u4e2a\u6570\u636e\u6e90\u7684\u65b9\u5f0f\u5c06\u4ee5 mounts.name \u4f5c\u4e3a\u533a\u5206\uff0c\u6bd4\u5982 /spark \u8def\u5f84\u4e0b\u8bbf\u95ee oss://test-bucket-1/dir1/ \u4e0b\u7684\u6587\u4ef6\uff0c/hadoop \u8def\u5f84\u4e0b\u8bbf\u95ee oss://test-bucket-2/dir2/ \u4e0b\u7684\u6587\u4ef6\u5185\u5bb9 master\u5143\u6570\u636e\u6301\u4e45\u5316 \u00a7 \u5982\u679c\u60f3\u901a\u8fc7\u7ed9master\u6302\u8f7dvolume\u7684\u65b9\u5f0f\uff0c\u8ba9master\u7684\u5143\u6570\u636e\u6301\u4e45\u5316\u5230\u6307\u5b9a\u7684\u5b58\u50a8\u4e0a\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" volumes: - name: nas persistentVolumeClaim: claimName: nas-test master: volumeMounts: - name: nas subPath: test mountPath: /var/cache/ properties: namespace.meta-dir: \"/var/cache/\" \u8be5\u4f8b\u5b50\u5c06\u5143\u6570\u636e\u76ee\u5f55 namespace.meta-dir \u6307\u5b9a\u5230\u6302\u8f7d\u7684volume\u4e0a\uff0c\u5e76\u8fdb\u884c\u6301\u4e45\u5316\uff0c\u6ce8\u610f\u6302\u8f7d\u7684vomlume\u5fc5\u987b\u5c31\u6709\u8bfb/\u5199/\u5220\u9664\u6743\u9650\uff0c\u4e0d\u53ef\u6307\u5b9a\u4e3a\u53ea\u8bfb \u66f4\u591a\u7f13\u5b58\u7b56\u7565 \u00a7 \u5143\u6570\u636e\u7f13\u5b58 \u00a7 \u5143\u6570\u636e\u7f13\u5b58\u6709\u4e24\u79cd\u7b56\u7565\uff0c\u53ef\u4ee5 dataset \u7684\u5b9a\u4e49\u4e2d\u8fdb\u884c\u8bbe\u7f6e\uff0c\u6253\u5f00\u548c\u5173\u95ed\u5206\u522b\u5bf9\u5e94 ONCE \u548c ALWAYS\uff0c\u9ed8\u8ba4\u7b56\u7565\u662f\u5173\u95ed\u3002 \u5143\u6570\u636e\u7f13\u5b58\u5f00\u5173 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: metaPolicy: ONCE fs.oss.endpoint: <OSS_ENDPOINT> \u5173\u95ed\u5143\u6570\u636e\u7f13\u5b58\uff08\u9ed8\u8ba4\uff09 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: metaPolicy: ALWAYS fs.oss.endpoint: <OSS_ENDPOINT> \u5199\u65f6\u843d\u7f13\u5b58\uff08WRITE_THROUGH\uff09 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: writePolicy: WRITE_THROUGH metaPolicy: ONCE # \u53ef\u4ee5\u6253\u5f00\u6216\u5173\u95ed fs.oss.endpoint: <OSS_ENDPOINT> \u5199\u4e34\u65f6\u7f13\u5b58\uff08CACHE_ONLY\uff09 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: writePolicy: CACHE_ONLY metaPolicy: ONCE #\u5fc5\u987b\u6253\u5f00\u5143\u6570\u636e\u7f13\u5b58 fs.oss.endpoint: <OSS_ENDPOINT> DHT \u7b56\u7565\uff08\u6d77\u91cf\u5c0f\u6587\u4ef6\u53ea\u8bfb\u573a\u666f\uff09 \u00a7 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: cacheStrategy: DHT metaPolicy: ONCE #\u5fc5\u987b\u6253\u5f00\u5143\u6570\u636e\u7f13\u5b58 fs.oss.endpoint: <OSS_ENDPOINT>","title":"Fluid + JindoCache \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u9ad8\u9636\u529f\u80fd\u4f7f\u7528"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#fluid-jindocache","text":"\u4f7f\u7528\u5217\u8868 \u6302\u8f7d\u70b9\u5728\u6839\u76ee\u5f55\u4e0b Secret \u52a0\u5bc6 AK \u53c2\u6570 Raft 3 master \u6a21\u5f0f \u4f7f\u7528 Placement \u90e8\u7f72\u591a\u4e2a runtime \u4f7f\u7528 NoseSelector \u90e8\u7f72\u8282\u70b9 \u4f7f\u7528 dataset nodeAffinity \u529f\u80fd Worker \u4e2a\u6570\u6269\u7f29\u5bb9 \u4f7f\u7528 tolerations \u529f\u80fd Resource \u8d44\u6e90 Fuse \u56de\u6536\u7b56\u7565 JindoCache Fuse \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528 FuseOnly \u4f7f\u7528\u65b9\u5f0f \u591a\u6302\u8f7d\u70b9 master\u5143\u6570\u636e\u6301\u4e45\u5316 \u66f4\u591a\u7f13\u5b58\u7b56\u7565","title":"Fluid + JindoCache \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\u9ad8\u9636\u529f\u80fd\u4f7f\u7528"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#_1","text":"\u9ed8\u8ba4\u4f7f\u7528 JindoRuntime \u4f1a\u5728\u6302\u8f7d\u70b9\u591a\u4e00\u5c42 / \u7684\u76ee\u5f55\uff0c\u5982\u679c\u60f3\u6302\u8f7d\u5728\u6839\u76ee\u5f55\u4e0b\u53ef\u4ee5\u5728 dataset \u91cc\u8fdb\u884c\u53c2\u6570\u6307\u5b9a apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase path: / \u6307\u5b9a spec.mounts.path = /\uff0c\u5219\u4f1a\u5c06\u6587\u4ef6\u6302\u8f7d\u5728\u6839\u76ee\u5f55\u4e0b","title":"\u6302\u8f7d\u70b9\u5728\u6839\u76ee\u5f55\u4e0b"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#secret-ak","text":"apiVersion: v1 kind: Secret metadata: name: mysecret stringData: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> \u5728 dataset \u91cc\u4f7f\u7528 secret apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://<OSS_BUCKET>/<OSS_DIRECTORY>/ name: hadoop options: fs.oss.endpoint: <OSS_ENDPOINT> encryptOptions: - name: fs.oss.accessKeyId valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeyId - name: fs.oss.accessKeySecret valueFrom: secretKeyRef: name: mysecret key: fs.oss.accessKeySecret","title":"Secret \u52a0\u5bc6 AK \u53c2\u6570"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#raft-3-master","text":"JindoRuntime \u6a21\u5f0f\u542f\u52a8\u4e00\u4e2a master\uff0c\u53ef\u4ee5\u901a\u8fc7\u4f7f\u7528 master.replicas \u6765\u542f\u52a8 3 \u4e2a master \u8fdb\u884c HA \u8f6c\u6362\u3002 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 3 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 2G high: \"0.8\" low: \"0.7\" master: replicas: 3 master.replicas = 3 : \u542f\u52a8 3 \u4e2a master pod","title":"Raft 3 master \u6a21\u5f0f"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#placement-runtime","text":"JindoRuntime \u542f\u52a8\u540e\uff0c\u9ed8\u8ba4 worker \u8282\u70b9\u4e0a\u53ea\u80fd\u542f\u52a8\u4e00\u4e2a runtime \u7684worker\uff0c\u5c5e\u4e8e\u72ec\u5360\u6a21\u5f0f\uff0c\u5982\u679c\u60f3\u8981\u5728\u540c\u4e00\u4e2a\u8282\u70b9\u4e0a\u652f\u6301\u90e8\u7f72\u591a\u4e2a worker\uff0c\u53ef\u4ee5\u4f7f\u7528 shared \u6a21\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase placement: \"Shared\"","title":"\u4f7f\u7528 Placement \u90e8\u7f72\u591a\u4e2a runtime"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#noseselector","text":"apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /mnt/disk1 quota: 10Gi high: \"0.9\" low: \"0.8\" master: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1 worker: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1 fuse: nodeSelector: kubernetes.io/hostname: cn-hangzhou.10.1.1","title":"\u4f7f\u7528 NoseSelector \u90e8\u7f72\u8282\u70b9"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#dataset-nodeaffinity","text":"apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: hbase-cache operator: In values: - \"true\"","title":"\u4f7f\u7528 dataset nodeAffinity \u529f\u80fd"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#worker","text":"\u4f7f\u7528 kubectl scale \u5bf9 Runtime \u7684 Worker \u6570\u91cf\u8fdb\u884c\u8c03\u6574 $ kubectl scale jindoruntime <runtime_name> --replicas=<replica_num> \u5176\u4e2d * runtime_name\uff1aruntime \u7684\u540d\u5b57 * replica_num\uff1a\u8868\u793a\u6269\u6240\u5bb9\u7684 Worker \u6570\u91cf","title":"Worker \u4e2a\u6570\u6269\u7f29\u5bb9"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#tolerations","text":"\u60a8\u53ef\u4ee5\u5728 dataset \u91cc\u5b9a\u4e49 tolerations \u6307\u5b9a worker \u8282\u70b9\u7684\u8c03\u5ea6 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hbase spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hbase tolerations: - key: hbase operator: Equal value: \"true\" \u4e5f\u53ef\u4ee5\u4e3a master/worker/fuse \u5355\u72ec\u6307\u5b9a tolerations \u8c03\u5ea6\u7b56\u7565 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 3 tieredstore: levels: - mediumtype: HDD path: /mnt/disk1 quota: 2Gi high: \"0.8\" low: \"0.7\" master: tolerations: - key: hbase operator: Equal value: \"true\" worker: tolerations: - key: hbase operator: Equal value: \"true\" fuse: tolerations: - key: hbase operator: Equal value: \"true\"","title":"\u4f7f\u7528 tolerations \u529f\u80fd"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#resource","text":"\u53ef\u4ee5\u6307\u5b9a master/worker \u7b49\u7684 resource \u8d44\u6e90 apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" master: resources: limits: cpu: \"4\" memory: \"8Gi\" requests: cpu: \"2\" memory: \"3Gi\" worker: resources: limits: cpu: \"4\" memory: \"8Gi\" requests: cpu: \"2\" memory: \"3Gi\"","title":"Resource \u8d44\u6e90"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#fuse","text":"apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: cleanPolicy: OnDemand fuse.cleanPolicy: onDemand / OnRuntimeDeleted \u5206\u522b\u8868\u793a\u6309\u9700\u542f\u52a8\uff0c\u4efb\u52a1\u7ed3\u675f fuse \u4e5f\u7ed3\u675f\u548c\u5728 runtime \u9500\u6bc1\u7684\u65f6\u5019\u548c\u5176\u4ed6\u7ec4\u4ef6\u4e00\u8d77\u9500\u6bc1\u6389","title":"Fuse \u56de\u6536\u7b56\u7565"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#jindocache-fuse","text":"\u53c2\u6570\u540d\u79f0 \u53c2\u6570\u8bf4\u660e \u4f7f\u7528\u8303\u4f8b auto_unmount fuse\u8fdb\u7a0b\u9000\u51fa\u540e\u81ea\u52a8umount\u6302\u8f7d\u8282\u70b9\u3002 -oauto_unmount ro \u53ea\u8bfb\u6302\u8f7d\uff0c\u542f\u7528\u53c2\u6570\u540e\u4e0d\u5141\u8bb8\u5199\u64cd\u4f5c\u3002 -oro direct_io \u5f00\u542f\u540e\uff0c\u8bfb\u5199\u6587\u4ef6\u53ef\u4ee5\u7ed5\u8fc7page cache\u3002 -odirect_io kernel_cache \u5f00\u542f\u540e\uff0c\u5229\u7528\u5185\u6838\u7f13\u5b58\u4f18\u5316\u8bfb\u6027\u80fd\u3002 -okernel_cache auto_cache \u9ed8\u8ba4\u5f00\u542f\uff0c\u4e0ekernel_cache \u4e8c\u9009\u4e00\uff0c\u4e0ekernel_cache\u4e0d\u540c\u7684\u662f\uff0c\u5982\u679c\u6587\u4ef6\u5927\u5c0f\u6216\u4fee\u6539\u65f6\u95f4\u53d1\u751f\u53d8\u5316\uff0c\u7f13\u5b58\u5c31\u4f1a\u5931\u6548\u3002 -oauto_cache entry_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-oentry_timeout=60 attr_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-oattr_timeout=60 negative_timeout \u9ed8\u8ba4\u503c\uff0c0.1\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u5931\u8d25\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 \u9ed8\u8ba4-onegative_timeout=30 \u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: args: - -oro - -oattr_timeout=60 - -oentry_timeout=60 - -onegative_timeout=60 \u4ee5\u6570\u7ec4\u65b9\u5f0f\u5199\u5728 spec.fuse.args \u91cc\uff0c\u6309\u7167\u9700\u8981\u586b\u5199\u5373\u53ef \u5982\u5355\u72ec\u914d\u7f6e\u5ba2\u6237\u7aefpread\u53c2\u6570\uff1a apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" fuse: args: - -oro - -ometrics_port=0 - -okernel_cache - -oattr_timeout=7200 - -oentry_timeout=7200 - -onegative_timeout=7200 - -opread","title":"JindoCache Fuse \u5ba2\u6237\u7aef\u76f8\u5173\u53c2\u6570\u548c\u4f7f\u7528"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#fuseonly","text":"\u5982\u60a8\u60f3\u4f7f\u7528 FuseOnly \u65b9\u5f0f\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: master: disabled: true worker: disabled: true","title":"FuseOnly \u4f7f\u7528\u65b9\u5f0f"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#_2","text":"apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: spark - mountPoint: oss://test-bucket-2/dir2/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop \u5728\u6302\u8f7d pvc \u548c fuse \u4ee5\u53ca\u505a dataload \u7684\u65f6\u5019\uff0c\u8fd9\u4e24\u4e2a\u6570\u636e\u6e90\u7684\u65b9\u5f0f\u5c06\u4ee5 mounts.name \u4f5c\u4e3a\u533a\u5206\uff0c\u6bd4\u5982 /spark \u8def\u5f84\u4e0b\u8bbf\u95ee oss://test-bucket-1/dir1/ \u4e0b\u7684\u6587\u4ef6\uff0c/hadoop \u8def\u5f84\u4e0b\u8bbf\u95ee oss://test-bucket-2/dir2/ \u4e0b\u7684\u6587\u4ef6\u5185\u5bb9","title":"\u591a\u6302\u8f7d\u70b9"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#master","text":"\u5982\u679c\u60f3\u901a\u8fc7\u7ed9master\u6302\u8f7dvolume\u7684\u65b9\u5f0f\uff0c\u8ba9master\u7684\u5143\u6570\u636e\u6301\u4e45\u5316\u5230\u6307\u5b9a\u7684\u5b58\u50a8\u4e0a\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u914d\u7f6e\u65b9\u5f0f apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: oss spec: replicas: 1 tieredstore: levels: - mediumtype: SSD path: /var/lib/docker/jindo quota: 200Gi high: \"0.9\" low: \"0.8\" volumes: - name: nas persistentVolumeClaim: claimName: nas-test master: volumeMounts: - name: nas subPath: test mountPath: /var/cache/ properties: namespace.meta-dir: \"/var/cache/\" \u8be5\u4f8b\u5b50\u5c06\u5143\u6570\u636e\u76ee\u5f55 namespace.meta-dir \u6307\u5b9a\u5230\u6302\u8f7d\u7684volume\u4e0a\uff0c\u5e76\u8fdb\u884c\u6301\u4e45\u5316\uff0c\u6ce8\u610f\u6302\u8f7d\u7684vomlume\u5fc5\u987b\u5c31\u6709\u8bfb/\u5199/\u5220\u9664\u6743\u9650\uff0c\u4e0d\u53ef\u6307\u5b9a\u4e3a\u53ea\u8bfb","title":"master\u5143\u6570\u636e\u6301\u4e45\u5316"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#_3","text":"","title":"\u66f4\u591a\u7f13\u5b58\u7b56\u7565"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#_4","text":"\u5143\u6570\u636e\u7f13\u5b58\u6709\u4e24\u79cd\u7b56\u7565\uff0c\u53ef\u4ee5 dataset \u7684\u5b9a\u4e49\u4e2d\u8fdb\u884c\u8bbe\u7f6e\uff0c\u6253\u5f00\u548c\u5173\u95ed\u5206\u522b\u5bf9\u5e94 ONCE \u548c ALWAYS\uff0c\u9ed8\u8ba4\u7b56\u7565\u662f\u5173\u95ed\u3002 \u5143\u6570\u636e\u7f13\u5b58\u5f00\u5173 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: metaPolicy: ONCE fs.oss.endpoint: <OSS_ENDPOINT> \u5173\u95ed\u5143\u6570\u636e\u7f13\u5b58\uff08\u9ed8\u8ba4\uff09 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: metaPolicy: ALWAYS fs.oss.endpoint: <OSS_ENDPOINT>","title":"\u5143\u6570\u636e\u7f13\u5b58"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#write_through","text":"apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: writePolicy: WRITE_THROUGH metaPolicy: ONCE # \u53ef\u4ee5\u6253\u5f00\u6216\u5173\u95ed fs.oss.endpoint: <OSS_ENDPOINT>","title":"\u5199\u65f6\u843d\u7f13\u5b58\uff08WRITE_THROUGH\uff09"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#cache_only","text":"apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: writePolicy: CACHE_ONLY metaPolicy: ONCE #\u5fc5\u987b\u6253\u5f00\u5143\u6570\u636e\u7f13\u5b58 fs.oss.endpoint: <OSS_ENDPOINT>","title":"\u5199\u4e34\u65f6\u7f13\u5b58\uff08CACHE_ONLY\uff09"},{"location":"fluid-jindoruntime/jindo_fluid_ways_to_use/#dht","text":"apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: data spec: mounts: - mountPoint: oss://test-bucket-1/dir1/ options: cacheStrategy: DHT metaPolicy: ONCE #\u5fc5\u987b\u6253\u5f00\u5143\u6570\u636e\u7f13\u5b58 fs.oss.endpoint: <OSS_ENDPOINT>","title":"DHT \u7b56\u7565\uff08\u6d77\u91cf\u5c0f\u6587\u4ef6\u53ea\u8bfb\u573a\u666f\uff09"},{"location":"fluid-jindoruntime/case/jindo_fluid_nas_ufs_example/","text":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001\u4e0b\u8f7d\u5b89\u88c5\u5305 \u00a7 \u70b9\u51fb\u4e0b\u8f7d 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-0.7.0.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002 5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907 \u00a7 5.1 \u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: nas spec: mounts: - mountPoint: local:///mnt/nas name: nasData --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: nas spec: replicas: 2 tieredstore: levels: - mediumtype: MEM path: /dev/shm quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d NAS \u7684\u8def\u5f84\uff0c local:// \u4e3a\u672c\u5730\u8def\u5f84\u524d\u7f00\uff0c /mnt/nas \u8868\u793a\u672c\u5730\u771f\u5b9e\u8def\u5f84\uff0c\u8be5\u8def\u5f84\u9700\u8981\u5728\u5bbf\u4e3b\u673a\u4e0a\u53ef\u89c1\u3002 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JindoRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime nas NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE nas Ready Ready 62m \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/nas 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/nas Bound hadoop 100Gi RWX 58m \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 NAS \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: test spec: dataset: name: nas namespace: default target: - path: /nasData replicas: 1 \u5176\u4e2d /nasData \u4e2d nasData \u4e3a dataset \u4e2d\u6302\u8f7d\u70b9\u7684 name\uff0c\u53ef\u4ee5\u4f7f\u7528 /nasData/xxx \u6765\u9884\u70ed\u5176\u4e2d\u7684\u5b50\u76ee\u5f55\u3002 replicas \u8868\u793a\u7f13\u5b58\u7684\u5907\u4efd\u6570\uff0c\u9ed8\u8ba4\u662f1\u3002 \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload test-dataload NAME DATASET PHASE AGE dataload nas Completed 2m13s 6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: nas \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u8bbf\u95ee\u6302\u8f7d\u70b9\u5bf9\u5e94\u7684\u5e95\u5c42\u6587\u4ef6 $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 7\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime nas \u5220\u9664 dataset kubectl delete dataset nas","title":"Jindo fluid nas ufs example"},{"location":"fluid-jindoruntime/case/jindo_fluid_nas_ufs_example/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"fluid-jindoruntime/case/jindo_fluid_nas_ufs_example/#2","text":"\u70b9\u51fb\u4e0b\u8f7d","title":"2\u3001\u4e0b\u8f7d\u5b89\u88c5\u5305"},{"location":"fluid-jindoruntime/case/jindo_fluid_nas_ufs_example/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-0.7.0.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"fluid-jindoruntime/case/jindo_fluid_nas_ufs_example/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s \u5176\u4e2d csi-nodeplugin-fluid-xx \u7684\u6570\u91cf\u5e94\u8be5\u4e0e K8S \u96c6\u7fa4\u4e2d\u8282\u70b9node\u7684\u6570\u91cf\u76f8\u540c\u3002","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"fluid-jindoruntime/case/jindo_fluid_nas_ufs_example/#5","text":"","title":"5\u3001\u6d4b\u8bd5\u6570\u636e\u6e90\u51c6\u5907"},{"location":"fluid-jindoruntime/case/jindo_fluid_nas_ufs_example/#51-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: nas spec: mounts: - mountPoint: local:///mnt/nas name: nasData --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: nas spec: replicas: 2 tieredstore: levels: - mediumtype: MEM path: /dev/shm quota: 100Gi high: \"0.9\" low: \"0.8\" mountPoint\uff1a\u8868\u793a\u6302\u8f7d NAS \u7684\u8def\u5f84\uff0c local:// \u4e3a\u672c\u5730\u8def\u5f84\u524d\u7f00\uff0c /mnt/nas \u8868\u793a\u672c\u5730\u771f\u5b9e\u8def\u5f84\uff0c\u8be5\u8def\u5f84\u9700\u8981\u5728\u5bbf\u4e3b\u673a\u4e0a\u53ef\u89c1\u3002 replicas\uff1a\u8868\u793a\u521b\u5efa JindoFSx \u96c6\u7fa4\u8282\u70b9\u7684\u6570\u91cf\u3002 mediumtype\uff1a JindoFSx \u6682\u53ea\u652f\u6301HDD/SSD/MEM\u4e2d\u7684\u5176\u4e2d\u4e00\u79cd\u3002 path\uff1a\u5b58\u50a8\u8def\u5f84\uff0c\u6682\u53ea\u652f\u6301\u4e00\u5757\u76d8\uff0c\u5f53\u9009\u62e9MEM\u505a\u7f13\u5b58\u4e5f\u9700\u8981\u4e00\u5757\u76d8\u6765\u5b58\u50a8log\u7b49\u6587\u4ef6\u3002 quota\uff1a\u7f13\u5b58\u6700\u5927\u5bb9\u91cf\uff0c100Gi \u8868\u793a\u6700\u5927\u53ef\u7528 100GB \u5927\u5c0f\u7684\u78c1\u76d8\u3002 high\uff1a\u6c34\u4f4d\u4e0a\u9650\u5927\u5c0f / low\uff1a \u6c34\u4f4d\u4e0b\u9650\u5927\u5c0f\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b\u90e8\u7f72\u7684 JindoRuntime \u60c5\u51b5\uff0c\u5176\u4e2d fuse \u4f1a lazy \u542f\u52a8\uff0c\u6839\u636e\u5e94\u7528\u6302\u8f7d\u60c5\u51b5\u6765\u76f8\u5e94\u7684\u8282\u70b9\u4e0a\u542f\u52a8 kubectl get jindoruntime nas NAME MASTER PHASE WORKER PHASE FUSE PHASE AGE nas Ready Ready 62m \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/nas 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/nas Bound hadoop 100Gi RWX 58m \u6b64\u65f6\u6211\u4eec\u5bf9 oss \u4e0a\u6587\u4ef6\u8fdb\u884c\u4e00\u6b21\u6570\u636e\u9884\u70ed\uff0c\u5c06 NAS \u4e0a\u6587\u4ef6\u52a0\u8f7d\u5230\u672c\u5730 apiVersion: data.fluid.io/v1alpha1 kind: DataLoad metadata: name: test spec: dataset: name: nas namespace: default target: - path: /nasData replicas: 1 \u5176\u4e2d /nasData \u4e2d nasData \u4e3a dataset \u4e2d\u6302\u8f7d\u70b9\u7684 name\uff0c\u53ef\u4ee5\u4f7f\u7528 /nasData/xxx \u6765\u9884\u70ed\u5176\u4e2d\u7684\u5b50\u76ee\u5f55\u3002 replicas \u8868\u793a\u7f13\u5b58\u7684\u5907\u4efd\u6570\uff0c\u9ed8\u8ba4\u662f1\u3002 \u6267\u884c kubectl create -f dataload.yaml \u89c2\u5bdf dataload \u4efb\u52a1\u7684\u6267\u884c\u60c5\u51b5 $ kubectl get dataload test-dataload NAME DATASET PHASE AGE dataload nas Completed 2m13s","title":"5.1 \u521b\u5efa dataset \u548c JindoRuntime"},{"location":"fluid-jindoruntime/case/jindo_fluid_nas_ufs_example/#6","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u6765\u4f7f\u7528 JindoFSx \u52a0\u901f\u670d\u52a1\uff0c\u6216\u8005\u8fdb\u884c\u63d0\u4ea4\u673a\u5668\u5b66\u4e60\u4f5c\u4e1a\u6765\u8fdb\u884c\u4f53\u9a8c\u76f8\u5173\u529f\u80fd\u3002 \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u5e94\u7528\u5bb9\u5668 app.yaml \u6765\u4f7f\u7528\u8be5\u6570\u636e\u96c6\uff0c\u6211\u4eec\u5c06\u591a\u6b21\u8bbf\u95ee\u540c\u4e00\u6570\u636e\uff0c\u5e76\u6bd4\u8f83\u8bbf\u95ee\u65f6\u95f4\u6765\u5c55\u793aJindoRuntime \u7684\u52a0\u901f\u6548\u679c\u3002 apiVersion: v1 kind: Pod metadata: name: demo-app spec: containers: - name: demo image: nginx volumeMounts: - mountPath: /data name: hadoop volumes: - name: hadoop persistentVolumeClaim: claimName: nas \u4f7f\u7528kubectl\u5b8c\u6210\u521b\u5efa kubectl create -f app.yaml \u8bbf\u95ee\u6302\u8f7d\u70b9\u5bf9\u5e94\u7684\u5e95\u5c42\u6587\u4ef6 $ kubectl exec -it demo-app -- bash $ du -sh /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2 210M /data/hadoop/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2","title":"6\u3001\u521b\u5efa\u5e94\u7528\u5bb9\u5668\u4f53\u9a8c\u52a0\u901f\u6548\u679c"},{"location":"fluid-jindoruntime/case/jindo_fluid_nas_ufs_example/#7","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime nas \u5220\u9664 dataset kubectl delete dataset nas","title":"7\u3001\u73af\u5883\u6e05\u7406"},{"location":"fluid-jindoruntime/case/jindo_fluid_oss_append_fuseonly/","text":"\u52a0\u901fOSS\u4e0a\u6570\u636e \u00a7 \u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002 1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4 \u00a7 kubectl create ns fluid-system 2\u3001 \u4e0b\u8f7d Fluid \u5b89\u88c5\u5305 \u00a7 3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid \u00a7 helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz 4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001 \u00a7 $ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s 5\u3001\u521b\u5efa dataset \u548c JindoRuntime \u00a7 \u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop path: / accessModes: - ReadWriteMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: master: disabled: true worker: disabled: true mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aoss bucket\u7684endpoint\u4fe1\u606f\uff0c\u516c\u7f51\u6216\u5185\u7f51\u5730\u5740\u7686\u53ef\uff0c\u5982\u60a8\u7684 bucket \u5728\u676d\u5dde Region\uff0c\u90a3\u4e48\u516c\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou.aliyuncs.com\uff0c\u5185\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou-internal.aliyuncs.com(\u5185\u7f51\u5730\u5740\u4f7f\u7528\u6761\u4ef6\u4e3a\u60a8\u7684 K8S \u96c6\u7fa4\u6240\u5728\u533a\u57df\u548coss\u533a\u57df\u76f8\u540c)\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m 6\u3001\u73af\u5883\u6e05\u7406 \u00a7 \u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/case/jindo_fluid_oss_append_fuseonly/#oss","text":"\u4f7f\u7528 JindoRuntime \u6d41\u7a0b\u7b80\u5355\uff0c\u5728\u51c6\u5907\u597d\u57fa\u672c K8S \u548c OSS \u73af\u5883\u7684\u6761\u4ef6\u4e0b\uff0c\u60a8\u53ea\u9700\u8981\u8017\u8d3910\u5206\u949f\u5de6\u53f3\u65f6\u95f4\u5373\u53ef\u90e8\u7f72\u597d\u9700\u8981\u7684 JindoRuntime \u73af\u5883\uff0c\u60a8\u53ef\u4ee5\u6309\u7167\u4e0b\u9762\u7684\u6d41\u7a0b\u8fdb\u884c\u90e8\u7f72\u3002","title":"\u52a0\u901fOSS\u4e0a\u6570\u636e"},{"location":"fluid-jindoruntime/case/jindo_fluid_oss_append_fuseonly/#1","text":"kubectl create ns fluid-system","title":"1\u3001\u521b\u5efa\u547d\u540d\u7a7a\u95f4"},{"location":"fluid-jindoruntime/case/jindo_fluid_oss_append_fuseonly/#2-fluid","text":"","title":"2\u3001\u4e0b\u8f7d Fluid \u5b89\u88c5\u5305"},{"location":"fluid-jindoruntime/case/jindo_fluid_oss_append_fuseonly/#3-helm-fluid","text":"helm install --set runtime.jindo.enabled=true fluid fluid-<version>.tgz","title":"3\u3001\u4f7f\u7528 Helm \u5b89\u88c5 Fluid"},{"location":"fluid-jindoruntime/case/jindo_fluid_oss_append_fuseonly/#4-fluid","text":"$ kubectl get pod -n fluid-system NAME READY STATUS RESTARTS AGE csi-nodeplugin-fluid-2mfcr 2/2 Running 0 108s csi-nodeplugin-fluid-l7lv6 2/2 Running 0 108s dataset-controller-5465c4bbf9-5ds5p 1/1 Running 0 108s jindoruntime-controller-654fb74447-cldsv 1/1 Running 0 108s","title":"4\u3001\u67e5\u770b Fluid \u7684\u8fd0\u884c\u72b6\u6001"},{"location":"fluid-jindoruntime/case/jindo_fluid_oss_append_fuseonly/#5-dataset-jindoruntime","text":"\u521b\u5efa\u4e00\u4e2a resource.yaml \u6587\u4ef6\u91cc\u9762\u5305\u542b\u4e24\u90e8\u5206\uff1a \u9996\u5148\u5305\u542b\u6570\u636e\u96c6\u53ca ufs \u7684 dataset \u4fe1\u606f\uff0c\u521b\u5efa\u4e00\u4e2a Dataset CRD \u5bf9\u8c61\uff0c\u5176\u4e2d\u63cf\u8ff0\u4e86\u6570\u636e\u96c6\u7684\u6765\u6e90\uff0c\u5982\u4f8b\u5b50\u4e2d\u7684 test-bucket\u3002 \u63a5\u4e0b\u6765\u9700\u8981\u521b\u5efa\u4e00\u4e2a JindoRuntime\uff0c\u76f8\u5f53\u4e8e\u542f\u52a8\u4e00\u4e2a JindoFSx \u7684\u96c6\u7fa4\u6765\u63d0\u4f9b\u7f13\u5b58\u670d\u52a1\u3002 apiVersion: data.fluid.io/v1alpha1 kind: Dataset metadata: name: hadoop spec: mounts: - mountPoint: oss://test-bucket/ options: fs.oss.accessKeyId: <OSS_ACCESS_KEY_ID> fs.oss.accessKeySecret: <OSS_ACCESS_KEY_SECRET> fs.oss.endpoint: <OSS_ENDPOINT> name: hadoop path: / accessModes: - ReadWriteMany --- apiVersion: data.fluid.io/v1alpha1 kind: JindoRuntime metadata: name: hadoop spec: master: disabled: true worker: disabled: true mountPoint\uff1a\u8868\u793a\u6302\u8f7dUFS\u7684\u8def\u5f84\uff0c\u8def\u5f84\u4e2d\u4e0d\u9700\u8981\u5305\u542b endpoint \u4fe1\u606f\u3002 fs.oss.accessKeyId/fs.oss.accessKeySecret\uff1aoss bucket\u7684AK\u4fe1\u606f\uff0c\u6709\u6743\u9650\u8bbf\u95ee\u8be5 bucket\u3002 fs.oss.endpoint\uff1aoss bucket\u7684endpoint\u4fe1\u606f\uff0c\u516c\u7f51\u6216\u5185\u7f51\u5730\u5740\u7686\u53ef\uff0c\u5982\u60a8\u7684 bucket \u5728\u676d\u5dde Region\uff0c\u90a3\u4e48\u516c\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou.aliyuncs.com\uff0c\u5185\u7f51\u5730\u5740\u4e3a oss-cn-hangzhou-internal.aliyuncs.com(\u5185\u7f51\u5730\u5740\u4f7f\u7528\u6761\u4ef6\u4e3a\u60a8\u7684 K8S \u96c6\u7fa4\u6240\u5728\u533a\u57df\u548coss\u533a\u57df\u76f8\u540c)\u3002 \u521b\u5efa JindoRuntime kubectl create -f resource.yaml \u67e5\u770b PV\uff0cPVC \u521b\u5efa\u60c5\u51b5\uff0cJindoRuntime\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u4f1a\u81ea\u52a8\u521b\u5efaPV\u548cPVC kubectl get pv,pvc NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/hadoop 100Gi RWX Retain Bound default/hadoop 58m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/hadoop Bound hadoop 100Gi RWX 58m","title":"5\u3001\u521b\u5efa dataset \u548c JindoRuntime"},{"location":"fluid-jindoruntime/case/jindo_fluid_oss_append_fuseonly/#6","text":"\u5220\u9664\u5e94\u7528\u548c\u5e94\u7528\u5bb9\u5668 \u5220\u9664 JindoRuntime kubectl delete jindoruntime hadoop \u5220\u9664 dataset kubectl delete dataset hadoop","title":"6\u3001\u73af\u5883\u6e05\u7406"},{"location":"fluid-jindoruntime/case/jindo_fluid_sdk_connect_example/","text":"\u4f7f\u7528 JindoSDK \u8bbf\u95ee Fluid JindoRuntime \u96c6\u7fa4 \u00a7 \u6b65\u9aa4 \u00a7 \u524d\u63d0\uff1a\u542f\u52a8 Fluid JindoRuntime \u96c6\u7fa4 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e core-site \u00a7 \u767b\u9646\u5230 xxx-jindofs-master-0 \u5bb9\u5668\u91cc kubectl exec -ti xxx-jindofs-master-0 bash \u627e\u5230 /hdfs-3.2.1/etc/hadoop/core-site.xml \u6587\u4ef6\uff0c\u5c06\u6587\u4ef6\u5185\u5bb9\u5168\u90e8\u52a0\u5165\u5230 hadoop \u7684 classpath \u4e0b\u7684 core-site \u6587\u4ef6\u4e2d 4. shell\u547d\u4ee4 \u00a7 \u53ef\u4ee5\u6267\u884c hadoop shell \u547d\u4ee4\u6765\u5224\u65ad\u670d\u52a1\u662f\u5426\u8fde\u63a5\u6210\u529f hadoop fs -ls oss://<Bucket>/<Dir> 6. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoFSx \u53c2\u6570\u8bf4\u660e JindoFSx \u5305\u542b\u4e00\u4e9b\u7f13\u5b58\u76f8\u5173\u7684\u53c2\u6570\uff0c\u8bf7\u53c2\u8003\u6587\u6863 JindoFSx \u5ba2\u6237\u7aef\u914d\u7f6e\u5217\u8868 * JindoSDK \u53c2\u6570\u8bf4\u660e JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 7. \u6743\u9650\u7ba1\u7406 \u00a7 \u81ea\u5efa\u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u963f\u91cc\u4e91 E-MapReduce \u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848","title":"\u4f7f\u7528 JindoSDK \u8bbf\u95ee Fluid JindoRuntime \u96c6\u7fa4"},{"location":"fluid-jindoruntime/case/jindo_fluid_sdk_connect_example/#jindosdk-fluid-jindoruntime","text":"","title":"\u4f7f\u7528 JindoSDK \u8bbf\u95ee Fluid JindoRuntime \u96c6\u7fa4"},{"location":"fluid-jindoruntime/case/jindo_fluid_sdk_connect_example/#_1","text":"\u524d\u63d0\uff1a\u542f\u52a8 Fluid JindoRuntime \u96c6\u7fa4","title":"\u6b65\u9aa4"},{"location":"fluid-jindoruntime/case/jindo_fluid_sdk_connect_example/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"fluid-jindoruntime/case/jindo_fluid_sdk_connect_example/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-4.6.12 \u4e3a\u4f8b: cp jindosdk-4.6.12/lib/jindo-core-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-4.6.12/lib/jindo-sdk-4.6.12.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"fluid-jindoruntime/case/jindo_fluid_sdk_connect_example/#3-core-site","text":"\u767b\u9646\u5230 xxx-jindofs-master-0 \u5bb9\u5668\u91cc kubectl exec -ti xxx-jindofs-master-0 bash \u627e\u5230 /hdfs-3.2.1/etc/hadoop/core-site.xml \u6587\u4ef6\uff0c\u5c06\u6587\u4ef6\u5185\u5bb9\u5168\u90e8\u52a0\u5165\u5230 hadoop \u7684 classpath \u4e0b\u7684 core-site \u6587\u4ef6\u4e2d","title":"3. \u914d\u7f6e core-site"},{"location":"fluid-jindoruntime/case/jindo_fluid_sdk_connect_example/#4-shell","text":"\u53ef\u4ee5\u6267\u884c hadoop shell \u547d\u4ee4\u6765\u5224\u65ad\u670d\u52a1\u662f\u5426\u8fde\u63a5\u6210\u529f hadoop fs -ls oss://<Bucket>/<Dir>","title":"4. shell\u547d\u4ee4"},{"location":"fluid-jindoruntime/case/jindo_fluid_sdk_connect_example/#6","text":"JindoFSx \u53c2\u6570\u8bf4\u660e JindoFSx \u5305\u542b\u4e00\u4e9b\u7f13\u5b58\u76f8\u5173\u7684\u53c2\u6570\uff0c\u8bf7\u53c2\u8003\u6587\u6863 JindoFSx \u5ba2\u6237\u7aef\u914d\u7f6e\u5217\u8868 * JindoSDK \u53c2\u6570\u8bf4\u660e JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"6. \u53c2\u6570\u8c03\u4f18"},{"location":"fluid-jindoruntime/case/jindo_fluid_sdk_connect_example/#7","text":"\u81ea\u5efa\u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 \u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848 \u963f\u91cc\u4e91 E-MapReduce \u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4\u4e2d\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848","title":"7. \u6743\u9650\u7ba1\u7406"},{"location":"fluid-jindoruntime/case/jindo_fluid_update_version_example/","text":"Fluid JindoRuntime \u955c\u50cf\u5347\u7ea7 \u00a7 1\u3001\u4fee\u6539 jindoruntime-controller \u7684\u4fe1\u606f kubectl edit deployment jindoruntime-controller -n fluid-system 2\u3001\u7f16\u8f91 controller\uff0c\u627e\u5230\u5982\u4e0b\u5185\u5bb9 - name: JINDO_SMARTDATA_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:4.5.2 - name: JINDO_FUSE_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:4.5.2 3\u3001\u4f8b\u5982\u4ece 4.5.1 \u5347\u7ea7\u5230 4.5.2\uff0c\u4fee\u6539\u5982\u4e0b\u955c\u50cf\u5373\u53ef - name: JINDO_SMARTDATA_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:4.5.2 - name: JINDO_FUSE_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:4.5.2 \u5982\u679c\u662f fuseOnly \u6a21\u5f0f\uff0c\u90a3\u4e48\u53ea\u9700\u4fee\u6539 JINDO_FUSE_IMAGE_ENV \u5b57\u6bb5\u5373\u53ef 4\u3001\u4fdd\u5b58\u4fee\u6539\uff0c\u91cd\u65b0\u521b\u5efa\u96c6\u7fa4\u5373\u53ef","title":"Fluid JindoRuntime \u955c\u50cf\u5347\u7ea7"},{"location":"fluid-jindoruntime/case/jindo_fluid_update_version_example/#fluid-jindoruntime","text":"1\u3001\u4fee\u6539 jindoruntime-controller \u7684\u4fe1\u606f kubectl edit deployment jindoruntime-controller -n fluid-system 2\u3001\u7f16\u8f91 controller\uff0c\u627e\u5230\u5982\u4e0b\u5185\u5bb9 - name: JINDO_SMARTDATA_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:4.5.2 - name: JINDO_FUSE_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:4.5.2 3\u3001\u4f8b\u5982\u4ece 4.5.1 \u5347\u7ea7\u5230 4.5.2\uff0c\u4fee\u6539\u5982\u4e0b\u955c\u50cf\u5373\u53ef - name: JINDO_SMARTDATA_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:4.5.2 - name: JINDO_FUSE_IMAGE_ENV value: registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:4.5.2 \u5982\u679c\u662f fuseOnly \u6a21\u5f0f\uff0c\u90a3\u4e48\u53ea\u9700\u4fee\u6539 JINDO_FUSE_IMAGE_ENV \u5b57\u6bb5\u5373\u53ef 4\u3001\u4fdd\u5b58\u4fee\u6539\uff0c\u91cd\u65b0\u521b\u5efa\u96c6\u7fa4\u5373\u53ef","title":"Fluid JindoRuntime \u955c\u50cf\u5347\u7ea7"},{"location":"jindoauth/jindoauth_emr-next_kerberos/","text":"\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\uff08\u9ad8\u5b89\u5168 EMR \u96c6\u7fa4\uff09 \u00a7 \u672c\u6587\u4ecb\u7ecd\u5728 E-MapReduce \u65b0\u7248\u63a7\u5236\u53f0( EMR-5.15.0/EMR-3.49.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4)\u914d\u7f6e\u963f\u91cc\u4e91 OSS \u548c OSS-HDFS \u670d\u52a1\u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\u3002 \u80cc\u666f \u00a7 Apache Ranger \u63d0\u4f9b\u96c6\u4e2d\u5f0f\u7684\u6743\u9650\u7ba1\u7406\u6846\u67b6\uff0c\u53ef\u4ee5\u5bf9 Hadoop \u751f\u6001\u4e2d\u7684\u591a\u4e2a\u7ec4\u4ef6\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u8bbf\u95ee\u63a7\u5236\u3002\u5f53\u7528\u6237\u5c06\u6570\u636e\u5b58\u653e\u5728\u963f\u91cc\u4e91 OSS \u65f6\uff0c\u5219\u662f\u901a\u8fc7\u5728\u963f\u91cc\u4e91 RAM \u4ea7\u54c1\u521b\u5efa\u6216\u7ba1\u7406 RAM \u7528\u6237\uff0c\u5b9e\u73b0\u5bf9 OSS \u8d44\u6e90\u7684\u8bbf\u95ee\u63a7\u5236\u3002 \u4e3a\u7ef4\u6301\u5927\u6570\u636e\u5ba2\u6237\u7684\u4f7f\u7528\u4e60\u60ef\uff0c\u901a\u8fc7 JindoAuth \u63a5\u5165 Ranger \u7684\u5ba2\u6237\u7aef\uff0c\u65b9\u4fbf\u7528\u6237\u7edf\u4e00\u7ba1\u7406\u5927\u6570\u636e\u7ec4\u4ef6\u6743\u9650\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u521b\u5efa E-MapReduce EMR-5.15.0/EMR-3.49.0 \u6216\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\uff0c\u5e76\u4e14\u9009\u62e9\u4e86 Ranger \u548c Ranger-plugin \u670d\u52a1\u3002 \u542f\u7528 Kerberos \u8ba4\u8bc1\u6ee1\u8db3\u8ba4\u8bc1\u9700\u6c42\u3002 \u82e5\u4f7f\u7528\u96c6\u7fa4\u90fd\u662f\u53ef\u4fe1\u7528\u6237\uff0c\u5219\u53ef\u4e0d\u542f\u7528 Kerberos \u8ba4\u8bc1\u3002 \u6ce8\u610f\uff1aEMR \u96c6\u7fa4\u5f00\u542f Kerberos \u540e\uff0c\u5c06\u65e0\u6cd5\u5173\u95ed Kerberos \u8ba4\u8bc1\uff0c\u8bf7\u8c28\u614e\u9009\u62e9\u3002\u5f00\u542f Kerberos \u540e\uff0c\u96c6\u7fa4\u6240\u6709\u5927\u6570\u636e\u7ec4\u4ef6\u7684\u670d\u52a1\u5747\u9700\u8981\u7ecf\u8fc7 Kerberos \u8ba4\u8bc1\uff0c\u5927\u6570\u636e\u4f5c\u4e1a\u63d0\u4ea4\u5230\u96c6\u7fa4\u4f1a\u5148\u7ecf\u8fc7\u8eab\u4efd\u8ba4\u8bc1\u3002Kerberos \u8be6\u60c5\u4fe1\u606f\uff0c\u8bf7\u53c2\u89c1 \u9ad8\u5b89\u5168\u7c7b\u578b\u96c6\u7fa4 1. \u8fdb\u5165\u96c6\u7fa4\u670d\u52a1\u9875\u9762\u3002 \u00a7 a. \u767b\u5f55EMR on ECS\u63a7\u5236\u53f0\u3002 b. \u9876\u90e8\u83dc\u5355\u680f\u5904\uff0c\u6839\u636e\u5b9e\u9645\u60c5\u51b5\u9009\u62e9\u5730\u57df\u548c\u8d44\u6e90\u7ec4\u3002 c. \u5728\u96c6\u7fa4\u7ba1\u7406\u9875\u9762\uff0c\u5355\u51fb\u76ee\u6807\u96c6\u7fa4\u64cd\u4f5c\u5217\u7684\u96c6\u7fa4\u670d\u52a1\u3002 2. Ranger\u542f\u7528OSS\u3002 \u00a7 a. \u5728\u96c6\u7fa4\u670d\u52a1\u9875\u9762\uff0c\u5355\u51fbRanger-plugin\u670d\u52a1\u533a\u57df\u7684\u72b6\u6001\u3002 b. \u5728\u670d\u52a1\u6982\u8ff0\u533a\u57df\uff0c\u6253\u5f00enableOSS\u5f00\u5173\u3002 c. \u5728\u5f39\u51fa\u7684\u5bf9\u8bdd\u6846\u4e2d\uff0c\u5355\u51fb\u786e\u5b9a\u3002 3. \u90e8\u7f72\u5ba2\u6237\u7aef\u914d\u7f6e\u3002 \u00a7 a. \u5728\u96c6\u7fa4\u670d\u52a1\u9875\u9762\uff0c\u5355\u51fb HADOOP-COMMON \u670d\u52a1\u9875\u9762\u3002 b. \u5355\u51fb\u914d\u7f6e\u3002 c. \u5355\u51fb\u90e8\u7f72\u5ba2\u6237\u7aef\u914d\u7f6e\u3002 4. \u91cd\u542f HiveServer2 \u7b49\u5e38\u9a7b\u670d\u52a1\u3002 \u00a7 a. \u5728\u96c6\u7fa4\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9more > Hive\u3002 b. \u5728\u7ec4\u4ef6\u5217\u8868\u533a\u57df\uff0c\u5355\u51fbHiveServer\u64cd\u4f5c\u5217\u7684\u91cd\u542f\u3002 c. \u5728\u5f39\u51fa\u7684\u5bf9\u8bdd\u6846\u4e2d\uff0c\u8f93\u5165\u6267\u884c\u539f\u56e0\uff0c\u5355\u51fb\u786e\u5b9a\u3002 d. \u5728\u786e\u8ba4\u5bf9\u8bdd\u6846\u4e2d\uff0c\u5355\u51fb\u786e\u5b9a\u3002 5. \u6743\u9650\u914d\u7f6e\u793a\u4f8b\u3002 \u00a7 Ranger\u542f\u7528OSS\u540e\uff0c\u5df2\u9ed8\u8ba4\u6dfb\u52a0\u597d\u4e86OSS Service\u3002 \u4f8b\uff1a\u914d\u7f6e\u7528\u6237test\u62e5\u6709\u8bbf\u95ee oss://jindoauth-runjob-cn-shanghai/user/test \u76ee\u5f55\u7684\u6240\u4ee5\u6743\u9650\u7684\u6b65\u9aa4\uff1a \u914d\u7f6e test \u7528\u6237\u8bbf\u95ee oss://jindoauth-runjob-cn-shanghai/user/test \u76ee\u5f55\u7684\u8bbf\u95ee\u6743\u9650\u4e3a ALL\u3002 \u00a7 \u8bf4\u660e\uff1a \u00a7 \u89c4\u5219\u914d\u7f6e\u9875\u9762\u4e2d\uff0c\u914d\u7f6e\u7684 path \u6ca1\u6709 oss:// \u7684\u524d\u7f00\u3002 recursive\u6309\u94ae\u4e0d\u53ef\u5173\u95ed\u3002 \u8def\u5f84\u672b\u5c3e\u65e0\u9700\u5e26\u6b63\u659c\u7ebf\uff08/\uff09\u3002 6. \u8bbf\u95eeOSS\u3002 \u00a7 \u82e5\u7528\u6237\u8bbf\u95ee Ranger \u6ca1\u6709\u6388\u6743\u7684\u8def\u5f84\uff0c\u5219\u4f1a\u62a5\u5982\u4e0b\u9519\u8bef\uff1a org.apache.hadoop.security.AccessControlException: Permission denied: user=test, access=READ_EXECUTE, resourcePath=\"bucket-test-hangzhou/\"","title":"JindoAuth \u5feb\u901f\u5165\u95e8"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#oss-ranger-emr","text":"\u672c\u6587\u4ecb\u7ecd\u5728 E-MapReduce \u65b0\u7248\u63a7\u5236\u53f0( EMR-5.15.0/EMR-3.49.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4)\u914d\u7f6e\u963f\u91cc\u4e91 OSS \u548c OSS-HDFS \u670d\u52a1\u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\u3002","title":"\u963f\u91cc\u4e91 OSS \u4f7f\u7528 Ranger \u7684\u9274\u6743\u65b9\u6848\uff08\u9ad8\u5b89\u5168 EMR \u96c6\u7fa4\uff09"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#_1","text":"Apache Ranger \u63d0\u4f9b\u96c6\u4e2d\u5f0f\u7684\u6743\u9650\u7ba1\u7406\u6846\u67b6\uff0c\u53ef\u4ee5\u5bf9 Hadoop \u751f\u6001\u4e2d\u7684\u591a\u4e2a\u7ec4\u4ef6\u8fdb\u884c\u7ec6\u7c92\u5ea6\u7684\u6743\u9650\u8bbf\u95ee\u63a7\u5236\u3002\u5f53\u7528\u6237\u5c06\u6570\u636e\u5b58\u653e\u5728\u963f\u91cc\u4e91 OSS \u65f6\uff0c\u5219\u662f\u901a\u8fc7\u5728\u963f\u91cc\u4e91 RAM \u4ea7\u54c1\u521b\u5efa\u6216\u7ba1\u7406 RAM \u7528\u6237\uff0c\u5b9e\u73b0\u5bf9 OSS \u8d44\u6e90\u7684\u8bbf\u95ee\u63a7\u5236\u3002 \u4e3a\u7ef4\u6301\u5927\u6570\u636e\u5ba2\u6237\u7684\u4f7f\u7528\u4e60\u60ef\uff0c\u901a\u8fc7 JindoAuth \u63a5\u5165 Ranger \u7684\u5ba2\u6237\u7aef\uff0c\u65b9\u4fbf\u7528\u6237\u7edf\u4e00\u7ba1\u7406\u5927\u6570\u636e\u7ec4\u4ef6\u6743\u9650\u3002","title":"\u80cc\u666f"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#_2","text":"\u5df2\u521b\u5efa E-MapReduce EMR-5.15.0/EMR-3.49.0 \u6216\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\uff0c\u5e76\u4e14\u9009\u62e9\u4e86 Ranger \u548c Ranger-plugin \u670d\u52a1\u3002 \u542f\u7528 Kerberos \u8ba4\u8bc1\u6ee1\u8db3\u8ba4\u8bc1\u9700\u6c42\u3002 \u82e5\u4f7f\u7528\u96c6\u7fa4\u90fd\u662f\u53ef\u4fe1\u7528\u6237\uff0c\u5219\u53ef\u4e0d\u542f\u7528 Kerberos \u8ba4\u8bc1\u3002 \u6ce8\u610f\uff1aEMR \u96c6\u7fa4\u5f00\u542f Kerberos \u540e\uff0c\u5c06\u65e0\u6cd5\u5173\u95ed Kerberos \u8ba4\u8bc1\uff0c\u8bf7\u8c28\u614e\u9009\u62e9\u3002\u5f00\u542f Kerberos \u540e\uff0c\u96c6\u7fa4\u6240\u6709\u5927\u6570\u636e\u7ec4\u4ef6\u7684\u670d\u52a1\u5747\u9700\u8981\u7ecf\u8fc7 Kerberos \u8ba4\u8bc1\uff0c\u5927\u6570\u636e\u4f5c\u4e1a\u63d0\u4ea4\u5230\u96c6\u7fa4\u4f1a\u5148\u7ecf\u8fc7\u8eab\u4efd\u8ba4\u8bc1\u3002Kerberos \u8be6\u60c5\u4fe1\u606f\uff0c\u8bf7\u53c2\u89c1 \u9ad8\u5b89\u5168\u7c7b\u578b\u96c6\u7fa4","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#1","text":"a. \u767b\u5f55EMR on ECS\u63a7\u5236\u53f0\u3002 b. \u9876\u90e8\u83dc\u5355\u680f\u5904\uff0c\u6839\u636e\u5b9e\u9645\u60c5\u51b5\u9009\u62e9\u5730\u57df\u548c\u8d44\u6e90\u7ec4\u3002 c. \u5728\u96c6\u7fa4\u7ba1\u7406\u9875\u9762\uff0c\u5355\u51fb\u76ee\u6807\u96c6\u7fa4\u64cd\u4f5c\u5217\u7684\u96c6\u7fa4\u670d\u52a1\u3002","title":"1. \u8fdb\u5165\u96c6\u7fa4\u670d\u52a1\u9875\u9762\u3002"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#2-rangeross","text":"a. \u5728\u96c6\u7fa4\u670d\u52a1\u9875\u9762\uff0c\u5355\u51fbRanger-plugin\u670d\u52a1\u533a\u57df\u7684\u72b6\u6001\u3002 b. \u5728\u670d\u52a1\u6982\u8ff0\u533a\u57df\uff0c\u6253\u5f00enableOSS\u5f00\u5173\u3002 c. \u5728\u5f39\u51fa\u7684\u5bf9\u8bdd\u6846\u4e2d\uff0c\u5355\u51fb\u786e\u5b9a\u3002","title":"2. Ranger\u542f\u7528OSS\u3002"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#3","text":"a. \u5728\u96c6\u7fa4\u670d\u52a1\u9875\u9762\uff0c\u5355\u51fb HADOOP-COMMON \u670d\u52a1\u9875\u9762\u3002 b. \u5355\u51fb\u914d\u7f6e\u3002 c. \u5355\u51fb\u90e8\u7f72\u5ba2\u6237\u7aef\u914d\u7f6e\u3002","title":"3. \u90e8\u7f72\u5ba2\u6237\u7aef\u914d\u7f6e\u3002"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#4-hiveserver2","text":"a. \u5728\u96c6\u7fa4\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9more > Hive\u3002 b. \u5728\u7ec4\u4ef6\u5217\u8868\u533a\u57df\uff0c\u5355\u51fbHiveServer\u64cd\u4f5c\u5217\u7684\u91cd\u542f\u3002 c. \u5728\u5f39\u51fa\u7684\u5bf9\u8bdd\u6846\u4e2d\uff0c\u8f93\u5165\u6267\u884c\u539f\u56e0\uff0c\u5355\u51fb\u786e\u5b9a\u3002 d. \u5728\u786e\u8ba4\u5bf9\u8bdd\u6846\u4e2d\uff0c\u5355\u51fb\u786e\u5b9a\u3002","title":"4. \u91cd\u542f HiveServer2 \u7b49\u5e38\u9a7b\u670d\u52a1\u3002"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#5","text":"Ranger\u542f\u7528OSS\u540e\uff0c\u5df2\u9ed8\u8ba4\u6dfb\u52a0\u597d\u4e86OSS Service\u3002 \u4f8b\uff1a\u914d\u7f6e\u7528\u6237test\u62e5\u6709\u8bbf\u95ee oss://jindoauth-runjob-cn-shanghai/user/test \u76ee\u5f55\u7684\u6240\u4ee5\u6743\u9650\u7684\u6b65\u9aa4\uff1a","title":"5. \u6743\u9650\u914d\u7f6e\u793a\u4f8b\u3002"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#test-ossjindoauth-runjob-cn-shanghaiusertest-all","text":"","title":"\u914d\u7f6e test \u7528\u6237\u8bbf\u95eeoss://jindoauth-runjob-cn-shanghai/user/test\u76ee\u5f55\u7684\u8bbf\u95ee\u6743\u9650\u4e3a ALL\u3002"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#_3","text":"\u89c4\u5219\u914d\u7f6e\u9875\u9762\u4e2d\uff0c\u914d\u7f6e\u7684 path \u6ca1\u6709 oss:// \u7684\u524d\u7f00\u3002 recursive\u6309\u94ae\u4e0d\u53ef\u5173\u95ed\u3002 \u8def\u5f84\u672b\u5c3e\u65e0\u9700\u5e26\u6b63\u659c\u7ebf\uff08/\uff09\u3002","title":"\u8bf4\u660e\uff1a"},{"location":"jindoauth/jindoauth_emr-next_kerberos/#6-oss","text":"\u82e5\u7528\u6237\u8bbf\u95ee Ranger \u6ca1\u6709\u6388\u6743\u7684\u8def\u5f84\uff0c\u5219\u4f1a\u62a5\u5982\u4e0b\u9519\u8bef\uff1a org.apache.hadoop.security.AccessControlException: Permission denied: user=test, access=READ_EXECUTE, resourcePath=\"bucket-test-hangzhou/\"","title":"6. \u8bbf\u95eeOSS\u3002"},{"location":"jindofs/jindofs_acls/","text":"\u80cc\u666f \u00a7 OSS-HDFS \u5b9e\u73b0\u4e86\u4e00\u4e2aPOSIX\u7cfb\u7edf\u7c7b\u4f3c\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u6743\u9650\u6a21\u578b\uff0c\u6bcf\u4e2a\u76ee\u5f55\u548c\u6587\u4ef6\u90fd\u5b58\u5728owner\u3001group\u3001others \u4e09\u7c7b\u6743\u9650\uff0c\u76ee\u5f55\u548c\u6587\u4ef6\u5bf9\u4e8e\u4e0d\u540c\u7684\u7528\u6237\u4ee5\u53ca\u7528\u6237\u7ec4\u6709\u7740\u4e0d\u540c\u7684\u6743\u9650\u3002\u6587\u4ef6\u6216\u76ee\u5f55\u7684\u6743\u9650\u5c31\u662f\u5b83\u7684\u6a21\u5f0f\uff08mode\uff09\u3002OSS-HDFS\u91c7\u7528\u4e86Unix\u8868\u793a\u548c\u663e\u793a\u6a21\u5f0f\u7684\u4e60\u60ef\uff0c\u5305\u62ec\u4f7f\u7528\u516b\u8fdb\u5236\u6570\u6765\u8868\u793a\u6743\u9650\u3002 \u5f53\u7528\u6237\u8bbf\u95eeOSS-HDFS\u7684\u6587\u4ef6\u6216\u8005\u76ee\u5f55\uff0cOSS-HDFS\u4f1a\u5bf9\u5176\u8fdb\u884c\u6743\u9650\u68c0\u67e5\uff0c\u68c0\u67e5\u6d41\u7a0b\u5982\u4e0b\uff1a - \u8bbf\u95ee\u7528\u6237\u4e0e\u6587\u4ef6/\u76ee\u5f55owner\u4e00\u81f4\uff0c\u68c0\u67e5owner\u7684\u6743\u9650 - \u8bbf\u95ee\u7528\u6237\u5173\u8054\u7684\u7ec4\u540d\u5728\u7ec4\u540d\u5217\u8868\u4e2d\u51fa\u73b0\uff0c\u68c0\u67e5group\u5bf9\u5e94\u8bbf\u95ee\u6743\u9650 - \u5426\u5219\u68c0\u67e5others \u5bf9\u5e94\u7684\u8bbf\u95ee\u6743\u9650 \u5982\u679c\u6743\u9650\u68c0\u67e5\u5931\u8d25\uff0c\u5219\u5ba2\u6237\u64cd\u4f5c\u4f1a\u5931\u8d25 \u5f00\u901a\u6743\u9650\u7ba1\u7406 \u00a7 \u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 \u6743\u9650\u7ba1\u7406\u7684\u5f00\u901a\u9700\u8981\u7528\u5230\u7684\u662f -putConfig \u4e0e -getConfig \u547d\u4ee4\uff0c\u5177\u4f53\u7528\u6cd5\u5c06\u5728\u4e0b\u6587\u4ecb\u7ecd\u3002 \u914d\u7f6e\u8bbf\u95ee AK \u00a7 \u6743\u9650\u7ba1\u7406\u9700\u8981\u914d\u7f6e\u8fde\u63a5 OSS-HDFS \u6240\u9700\u7684 AK \u4fe1\u606f\u3002\u6709\u4e24\u79cd\u65b9\u6cd5\u53ef\u4ee5\u8fdb\u884c\u914d\u7f6e\uff1a \u914d\u7f6e\u6587\u4ef6\uff08\u63a8\u8350\uff09 \u00a7 \u8bbe\u7f6e\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6307\u5411\u67d0\u4e2a\u76ee\u5f55\uff0c\u7136\u540e\u5728\u76ee\u5f55\u4e0b\u521b\u5efa\u540d\u4e3a jindofs.cfg \u7684\u6587\u4ef6\uff0c\u7136\u540e\u5728\u6587\u4ef6\u91cc\u5199\u5165\u5982\u4e0b\u5185\u5bb9\uff1a [client] fs.oss.accessKeyId = <key> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK key fs.oss.accessKeySecret = <secret> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK secret fs.oss.bucket.<bucket>.accessKeyId = <key> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK key\uff0c\u4f18\u5148\u7ea7\u9ad8\u4e8e\u9ed8\u8ba4 fs.oss.bucket.<bucket>.accessKeySecret = <secret> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK secret --extraConf \u00a7 \u547d\u4ee4\u884c\u5de5\u5177\u7684\u6240\u6709\u6307\u4ee4\u5747\u53ef\u901a\u8fc7 --extraConf \u9009\u9879\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u914d\u7f6e\u6587\u4ef6\uff0c\u4f8b\u5982\uff1a jindofs admin -getConfig -dlsUri <path> -name <keys> \\ --extraConf fs.oss.accessKeyId=<AK key> \\ --extraConf fs.oss.accessKeySecret=<AK secret> \u5c06\u7ed9\u547d\u4ee4\u63d0\u4f9b\u989d\u5916\u7684 AK key \u4e0e AK secret \u914d\u7f6e\u3002 \u6253\u5f00\u6743\u9650\u7ba1\u7406 \u00a7 \u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf namespace.permissions.enabled=true \u67e5\u770b\u6743\u9650\u7ba1\u7406 \u00a7 \u547d\u4ee4\uff1a jindofs admin -getConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -name namespace.permissions.enabled \u4e0a\u8ff0\u547d\u4ee4\u8fd4\u56de\u4e86\u540d\u4e3a \\<bucket>\uff08endpoint \u4e3a \\<oss-hdfs-endpoint>\uff09\u7684\u6876\u7684\u7ba1\u7406\u6743\u9650\u7684\u6807\u7b7e","title":"Jindofs acls"},{"location":"jindofs/jindofs_acls/#_1","text":"OSS-HDFS \u5b9e\u73b0\u4e86\u4e00\u4e2aPOSIX\u7cfb\u7edf\u7c7b\u4f3c\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u6743\u9650\u6a21\u578b\uff0c\u6bcf\u4e2a\u76ee\u5f55\u548c\u6587\u4ef6\u90fd\u5b58\u5728owner\u3001group\u3001others \u4e09\u7c7b\u6743\u9650\uff0c\u76ee\u5f55\u548c\u6587\u4ef6\u5bf9\u4e8e\u4e0d\u540c\u7684\u7528\u6237\u4ee5\u53ca\u7528\u6237\u7ec4\u6709\u7740\u4e0d\u540c\u7684\u6743\u9650\u3002\u6587\u4ef6\u6216\u76ee\u5f55\u7684\u6743\u9650\u5c31\u662f\u5b83\u7684\u6a21\u5f0f\uff08mode\uff09\u3002OSS-HDFS\u91c7\u7528\u4e86Unix\u8868\u793a\u548c\u663e\u793a\u6a21\u5f0f\u7684\u4e60\u60ef\uff0c\u5305\u62ec\u4f7f\u7528\u516b\u8fdb\u5236\u6570\u6765\u8868\u793a\u6743\u9650\u3002 \u5f53\u7528\u6237\u8bbf\u95eeOSS-HDFS\u7684\u6587\u4ef6\u6216\u8005\u76ee\u5f55\uff0cOSS-HDFS\u4f1a\u5bf9\u5176\u8fdb\u884c\u6743\u9650\u68c0\u67e5\uff0c\u68c0\u67e5\u6d41\u7a0b\u5982\u4e0b\uff1a - \u8bbf\u95ee\u7528\u6237\u4e0e\u6587\u4ef6/\u76ee\u5f55owner\u4e00\u81f4\uff0c\u68c0\u67e5owner\u7684\u6743\u9650 - \u8bbf\u95ee\u7528\u6237\u5173\u8054\u7684\u7ec4\u540d\u5728\u7ec4\u540d\u5217\u8868\u4e2d\u51fa\u73b0\uff0c\u68c0\u67e5group\u5bf9\u5e94\u8bbf\u95ee\u6743\u9650 - \u5426\u5219\u68c0\u67e5others \u5bf9\u5e94\u7684\u8bbf\u95ee\u6743\u9650 \u5982\u679c\u6743\u9650\u68c0\u67e5\u5931\u8d25\uff0c\u5219\u5ba2\u6237\u64cd\u4f5c\u4f1a\u5931\u8d25","title":"\u80cc\u666f"},{"location":"jindofs/jindofs_acls/#_2","text":"\u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 \u6743\u9650\u7ba1\u7406\u7684\u5f00\u901a\u9700\u8981\u7528\u5230\u7684\u662f -putConfig \u4e0e -getConfig \u547d\u4ee4\uff0c\u5177\u4f53\u7528\u6cd5\u5c06\u5728\u4e0b\u6587\u4ecb\u7ecd\u3002","title":"\u5f00\u901a\u6743\u9650\u7ba1\u7406"},{"location":"jindofs/jindofs_acls/#ak","text":"\u6743\u9650\u7ba1\u7406\u9700\u8981\u914d\u7f6e\u8fde\u63a5 OSS-HDFS \u6240\u9700\u7684 AK \u4fe1\u606f\u3002\u6709\u4e24\u79cd\u65b9\u6cd5\u53ef\u4ee5\u8fdb\u884c\u914d\u7f6e\uff1a","title":"\u914d\u7f6e\u8bbf\u95ee AK"},{"location":"jindofs/jindofs_acls/#_3","text":"\u8bbe\u7f6e\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6307\u5411\u67d0\u4e2a\u76ee\u5f55\uff0c\u7136\u540e\u5728\u76ee\u5f55\u4e0b\u521b\u5efa\u540d\u4e3a jindofs.cfg \u7684\u6587\u4ef6\uff0c\u7136\u540e\u5728\u6587\u4ef6\u91cc\u5199\u5165\u5982\u4e0b\u5185\u5bb9\uff1a [client] fs.oss.accessKeyId = <key> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK key fs.oss.accessKeySecret = <secret> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK secret fs.oss.bucket.<bucket>.accessKeyId = <key> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK key\uff0c\u4f18\u5148\u7ea7\u9ad8\u4e8e\u9ed8\u8ba4 fs.oss.bucket.<bucket>.accessKeySecret = <secret> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK secret","title":"\u914d\u7f6e\u6587\u4ef6\uff08\u63a8\u8350\uff09"},{"location":"jindofs/jindofs_acls/#-extraconf","text":"\u547d\u4ee4\u884c\u5de5\u5177\u7684\u6240\u6709\u6307\u4ee4\u5747\u53ef\u901a\u8fc7 --extraConf \u9009\u9879\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u914d\u7f6e\u6587\u4ef6\uff0c\u4f8b\u5982\uff1a jindofs admin -getConfig -dlsUri <path> -name <keys> \\ --extraConf fs.oss.accessKeyId=<AK key> \\ --extraConf fs.oss.accessKeySecret=<AK secret> \u5c06\u7ed9\u547d\u4ee4\u63d0\u4f9b\u989d\u5916\u7684 AK key \u4e0e AK secret \u914d\u7f6e\u3002","title":"--extraConf"},{"location":"jindofs/jindofs_acls/#_4","text":"\u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf namespace.permissions.enabled=true","title":"\u6253\u5f00\u6743\u9650\u7ba1\u7406"},{"location":"jindofs/jindofs_acls/#_5","text":"\u547d\u4ee4\uff1a jindofs admin -getConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -name namespace.permissions.enabled \u4e0a\u8ff0\u547d\u4ee4\u8fd4\u56de\u4e86\u540d\u4e3a \\<bucket>\uff08endpoint \u4e3a \\<oss-hdfs-endpoint>\uff09\u7684\u6876\u7684\u7ba1\u7406\u6743\u9650\u7684\u6807\u7b7e","title":"\u67e5\u770b\u6743\u9650\u7ba1\u7406"},{"location":"jindofs/jindofs_archive_direct_read/","text":"\u80cc\u666f \u00a7 OSS-HDFS \u5f52\u6863\u76f4\u8bfb\u5f53\u524d\u53ef\u4ee5\u901a\u8fc7\u547d\u4ee4\u884c\u547d\u4ee4\u7684\u65b9\u5f0f\u8fdb\u884c\u64cd\u4f5c\u3002\u5c06\u6765\u8fd8\u4f1a\u901a\u8fc7\u4ea7\u54c1\u754c\u9762\u7684\u65b9\u5f0f\uff0c\u63d0\u4f9b\u66f4\u52a0\u4fbf\u6377\u6613\u7528\u7684\u67e5\u770b\u548c\u8bbe\u7f6e\u65b9\u6cd5\u3002 \u5f52\u6863\u76f4\u8bfb\u529f\u80fd\u63cf\u8ff0 \u00a7 \u7528\u6237\u9700\u8981\u8bfb\u53d6\u5f52\u6863\u7c7b\u578b\u7684\u6587\u4ef6\u65f6\u5019\uff0c\u901a\u5e38\u9700\u8981\u901a\u8fc7\u89e3\u51bb\u64cd\u4f5c\u5c06\u5f52\u6863\u6587\u4ef6\u89e3\u51bb\u5230\u53ef\u8bfb\u53d6\u7684\u72b6\u6001\uff0c\u8fd9\u79cd\u89e3\u51bb\u64cd\u4f5c\u53ef\u80fd\u6bd4\u8f83\u8017\u65f6\uff0cOSS-HDFS \u5f52\u6863\u76f4\u8bfb\u529f\u80fd\u53ef\u4ee5\u907f\u514d\u89e3\u51bb\u64cd\u4f5c\uff0c\u76f4\u63a5\u8bfb\u53d6\u5f52\u6863\u6587\u4ef6 JindoFS \u547d\u4ee4\u884c\u7b80\u4ecb \u00a7 \u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 \u5f52\u6863\u76f4\u8bfb\u529f\u80fd\u9700\u8981\u7528\u5230\u7684\u662f -putConfig \u4e0e -getConfig \u547d\u4ee4\uff0c\u5177\u4f53\u7528\u6cd5\u5c06\u5728\u4e0b\u6587\u4ecb\u7ecd\u3002 \u914d\u7f6e\u8bbf\u95ee AK \u00a7 \u5f52\u6863\u76f4\u8bfb\u529f\u80fd\u9700\u8981\u914d\u7f6e\u8fde\u63a5 OSS-HDFS \u6240\u9700\u7684 AK \u4fe1\u606f\u3002\u6709\u4e24\u79cd\u65b9\u6cd5\u53ef\u4ee5\u8fdb\u884c\u914d\u7f6e\uff1a \u914d\u7f6e\u6587\u4ef6\uff08\u63a8\u8350\uff09 \u00a7 \u8bbe\u7f6e\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6307\u5411\u67d0\u4e2a\u76ee\u5f55\uff0c\u7136\u540e\u5728\u76ee\u5f55\u4e0b\u521b\u5efa\u540d\u4e3a jindofs.cfg \u7684\u6587\u4ef6\uff0c\u7136\u540e\u5728\u6587\u4ef6\u91cc\u5199\u5165\u5982\u4e0b\u5185\u5bb9\uff1a [client] fs.oss.accessKeyId = <key> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK key fs.oss.accessKeySecret = <secret> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK secret fs.oss.bucket.<bucket>.accessKeyId = <key> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK key\uff0c\u4f18\u5148\u7ea7\u9ad8\u4e8e\u9ed8\u8ba4 fs.oss.bucket.<bucket>.accessKeySecret = <secret> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK secret --extraConf \u00a7 \u547d\u4ee4\u884c\u5de5\u5177\u7684\u6240\u6709\u6307\u4ee4\u5747\u53ef\u901a\u8fc7 --extraConf \u9009\u9879\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u914d\u7f6e\u6587\u4ef6\uff0c\u4f8b\u5982\uff1a jindofs admin -getConfig -dlsUri <path> -name <keys> \\ --extraConf fs.oss.accessKeyId=<AK key> \\ --extraConf fs.oss.accessKeySecret=<AK secret> \u5c06\u7ed9\u547d\u4ee4\u63d0\u4f9b\u989d\u5916\u7684 AK key \u4e0e AK secret \u914d\u7f6e\u3002 \u8bbe\u7f6e\u5f52\u6863\u76f4\u8bfb \u00a7 \u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf namespace.archive.directread.enable=true \u67e5\u770b\u5f52\u6863\u76f4\u8bfb \u00a7 \u547d\u4ee4\uff1a jindofs admin -getConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -name namespace.archive.directread.enable \u4e0a\u8ff0\u547d\u4ee4\u8fd4\u56de\u4e86\u540d\u4e3a \\<bucket>\uff08endpoint \u4e3a \\<oss-hdfs-endpoint>\uff09\u7684\u6876\u7684\u5f52\u6863\u76f4\u8bfb\u6807\u7b7e","title":"Jindofs archive direct read"},{"location":"jindofs/jindofs_archive_direct_read/#_1","text":"OSS-HDFS \u5f52\u6863\u76f4\u8bfb\u5f53\u524d\u53ef\u4ee5\u901a\u8fc7\u547d\u4ee4\u884c\u547d\u4ee4\u7684\u65b9\u5f0f\u8fdb\u884c\u64cd\u4f5c\u3002\u5c06\u6765\u8fd8\u4f1a\u901a\u8fc7\u4ea7\u54c1\u754c\u9762\u7684\u65b9\u5f0f\uff0c\u63d0\u4f9b\u66f4\u52a0\u4fbf\u6377\u6613\u7528\u7684\u67e5\u770b\u548c\u8bbe\u7f6e\u65b9\u6cd5\u3002","title":"\u80cc\u666f"},{"location":"jindofs/jindofs_archive_direct_read/#_2","text":"\u7528\u6237\u9700\u8981\u8bfb\u53d6\u5f52\u6863\u7c7b\u578b\u7684\u6587\u4ef6\u65f6\u5019\uff0c\u901a\u5e38\u9700\u8981\u901a\u8fc7\u89e3\u51bb\u64cd\u4f5c\u5c06\u5f52\u6863\u6587\u4ef6\u89e3\u51bb\u5230\u53ef\u8bfb\u53d6\u7684\u72b6\u6001\uff0c\u8fd9\u79cd\u89e3\u51bb\u64cd\u4f5c\u53ef\u80fd\u6bd4\u8f83\u8017\u65f6\uff0cOSS-HDFS \u5f52\u6863\u76f4\u8bfb\u529f\u80fd\u53ef\u4ee5\u907f\u514d\u89e3\u51bb\u64cd\u4f5c\uff0c\u76f4\u63a5\u8bfb\u53d6\u5f52\u6863\u6587\u4ef6","title":"\u5f52\u6863\u76f4\u8bfb\u529f\u80fd\u63cf\u8ff0"},{"location":"jindofs/jindofs_archive_direct_read/#jindofs","text":"\u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 \u5f52\u6863\u76f4\u8bfb\u529f\u80fd\u9700\u8981\u7528\u5230\u7684\u662f -putConfig \u4e0e -getConfig \u547d\u4ee4\uff0c\u5177\u4f53\u7528\u6cd5\u5c06\u5728\u4e0b\u6587\u4ecb\u7ecd\u3002","title":"JindoFS \u547d\u4ee4\u884c\u7b80\u4ecb"},{"location":"jindofs/jindofs_archive_direct_read/#ak","text":"\u5f52\u6863\u76f4\u8bfb\u529f\u80fd\u9700\u8981\u914d\u7f6e\u8fde\u63a5 OSS-HDFS \u6240\u9700\u7684 AK \u4fe1\u606f\u3002\u6709\u4e24\u79cd\u65b9\u6cd5\u53ef\u4ee5\u8fdb\u884c\u914d\u7f6e\uff1a","title":"\u914d\u7f6e\u8bbf\u95ee AK"},{"location":"jindofs/jindofs_archive_direct_read/#_3","text":"\u8bbe\u7f6e\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6307\u5411\u67d0\u4e2a\u76ee\u5f55\uff0c\u7136\u540e\u5728\u76ee\u5f55\u4e0b\u521b\u5efa\u540d\u4e3a jindofs.cfg \u7684\u6587\u4ef6\uff0c\u7136\u540e\u5728\u6587\u4ef6\u91cc\u5199\u5165\u5982\u4e0b\u5185\u5bb9\uff1a [client] fs.oss.accessKeyId = <key> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK key fs.oss.accessKeySecret = <secret> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK secret fs.oss.bucket.<bucket>.accessKeyId = <key> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK key\uff0c\u4f18\u5148\u7ea7\u9ad8\u4e8e\u9ed8\u8ba4 fs.oss.bucket.<bucket>.accessKeySecret = <secret> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK secret","title":"\u914d\u7f6e\u6587\u4ef6\uff08\u63a8\u8350\uff09"},{"location":"jindofs/jindofs_archive_direct_read/#-extraconf","text":"\u547d\u4ee4\u884c\u5de5\u5177\u7684\u6240\u6709\u6307\u4ee4\u5747\u53ef\u901a\u8fc7 --extraConf \u9009\u9879\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u914d\u7f6e\u6587\u4ef6\uff0c\u4f8b\u5982\uff1a jindofs admin -getConfig -dlsUri <path> -name <keys> \\ --extraConf fs.oss.accessKeyId=<AK key> \\ --extraConf fs.oss.accessKeySecret=<AK secret> \u5c06\u7ed9\u547d\u4ee4\u63d0\u4f9b\u989d\u5916\u7684 AK key \u4e0e AK secret \u914d\u7f6e\u3002","title":"--extraConf"},{"location":"jindofs/jindofs_archive_direct_read/#_4","text":"\u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf namespace.archive.directread.enable=true","title":"\u8bbe\u7f6e\u5f52\u6863\u76f4\u8bfb"},{"location":"jindofs/jindofs_archive_direct_read/#_5","text":"\u547d\u4ee4\uff1a jindofs admin -getConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -name namespace.archive.directread.enable \u4e0a\u8ff0\u547d\u4ee4\u8fd4\u56de\u4e86\u540d\u4e3a \\<bucket>\uff08endpoint \u4e3a \\<oss-hdfs-endpoint>\uff09\u7684\u6876\u7684\u5f52\u6863\u76f4\u8bfb\u6807\u7b7e","title":"\u67e5\u770b\u5f52\u6863\u76f4\u8bfb"},{"location":"jindofs/jindofs_callerContext/","text":"\u5f00\u901aCallerContext \u00a7 \u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 CallerContext\u7684\u5f00\u901a\u9700\u8981\u7528\u5230\u7684\u662f -putConfig \u4e0e -getConfig \u547d\u4ee4\uff0c\u5177\u4f53\u7528\u6cd5\u5c06\u5728\u4e0b\u6587\u4ecb\u7ecd\u3002 \u914d\u7f6e\u8bbf\u95ee AK \u00a7 CallerContext\u9700\u8981\u914d\u7f6e\u8fde\u63a5 OSS-HDFS \u6240\u9700\u7684 AK \u4fe1\u606f\u3002\u6709\u4e24\u79cd\u65b9\u6cd5\u53ef\u4ee5\u8fdb\u884c\u914d\u7f6e\uff1a \u914d\u7f6e\u6587\u4ef6\uff08\u63a8\u8350\uff09 \u00a7 \u8bbe\u7f6e\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6307\u5411\u67d0\u4e2a\u76ee\u5f55\uff0c\u7136\u540e\u5728\u76ee\u5f55\u4e0b\u521b\u5efa\u540d\u4e3a jindofs.cfg \u7684\u6587\u4ef6\uff0c\u7136\u540e\u5728\u6587\u4ef6\u91cc\u5199\u5165\u5982\u4e0b\u5185\u5bb9\uff1a [client] fs.oss.accessKeyId = <key> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK key fs.oss.accessKeySecret = <secret> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK secret fs.oss.bucket.<bucket>.accessKeyId = <key> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK key\uff0c\u4f18\u5148\u7ea7\u9ad8\u4e8e\u9ed8\u8ba4 fs.oss.bucket.<bucket>.accessKeySecret = <secret> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK secret --extraConf \u00a7 \u547d\u4ee4\u884c\u5de5\u5177\u7684\u6240\u6709\u6307\u4ee4\u5747\u53ef\u901a\u8fc7 --extraConf \u9009\u9879\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u914d\u7f6e\u6587\u4ef6\uff0c\u4f8b\u5982\uff1a jindofs admin -getConfig -dlsUri <path> -name <keys> \\ --extraConf fs.oss.accessKeyId=<AK key> \\ --extraConf fs.oss.accessKeySecret=<AK secret> \u5c06\u7ed9\u547d\u4ee4\u63d0\u4f9b\u989d\u5916\u7684 AK key \u4e0e AK secret \u914d\u7f6e\u3002 \u6253\u5f00CallerContext\u7ba1\u7406 \u00a7 \u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf namespace.caller.context.enabled=true \u67e5\u770bCallerContext\u7ba1\u7406 \u00a7 \u547d\u4ee4\uff1a jindofs admin -getConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -name namespace.caller.context.enabled \u4e0a\u8ff0\u547d\u4ee4\u8fd4\u56de\u4e86\u540d\u4e3a \\<bucket>\uff08endpoint \u4e3a \\<oss-hdfs-endpoint>\uff09\u7684\u6876\u7684CallerContext\u7684\u6807\u7b7e","title":"jindofs callerContext"},{"location":"jindofs/jindofs_callerContext/#callercontext","text":"\u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 CallerContext\u7684\u5f00\u901a\u9700\u8981\u7528\u5230\u7684\u662f -putConfig \u4e0e -getConfig \u547d\u4ee4\uff0c\u5177\u4f53\u7528\u6cd5\u5c06\u5728\u4e0b\u6587\u4ecb\u7ecd\u3002","title":"\u5f00\u901aCallerContext"},{"location":"jindofs/jindofs_callerContext/#ak","text":"CallerContext\u9700\u8981\u914d\u7f6e\u8fde\u63a5 OSS-HDFS \u6240\u9700\u7684 AK \u4fe1\u606f\u3002\u6709\u4e24\u79cd\u65b9\u6cd5\u53ef\u4ee5\u8fdb\u884c\u914d\u7f6e\uff1a","title":"\u914d\u7f6e\u8bbf\u95ee AK"},{"location":"jindofs/jindofs_callerContext/#_1","text":"\u8bbe\u7f6e\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6307\u5411\u67d0\u4e2a\u76ee\u5f55\uff0c\u7136\u540e\u5728\u76ee\u5f55\u4e0b\u521b\u5efa\u540d\u4e3a jindofs.cfg \u7684\u6587\u4ef6\uff0c\u7136\u540e\u5728\u6587\u4ef6\u91cc\u5199\u5165\u5982\u4e0b\u5185\u5bb9\uff1a [client] fs.oss.accessKeyId = <key> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK key fs.oss.accessKeySecret = <secret> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK secret fs.oss.bucket.<bucket>.accessKeyId = <key> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK key\uff0c\u4f18\u5148\u7ea7\u9ad8\u4e8e\u9ed8\u8ba4 fs.oss.bucket.<bucket>.accessKeySecret = <secret> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK secret","title":"\u914d\u7f6e\u6587\u4ef6\uff08\u63a8\u8350\uff09"},{"location":"jindofs/jindofs_callerContext/#-extraconf","text":"\u547d\u4ee4\u884c\u5de5\u5177\u7684\u6240\u6709\u6307\u4ee4\u5747\u53ef\u901a\u8fc7 --extraConf \u9009\u9879\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u914d\u7f6e\u6587\u4ef6\uff0c\u4f8b\u5982\uff1a jindofs admin -getConfig -dlsUri <path> -name <keys> \\ --extraConf fs.oss.accessKeyId=<AK key> \\ --extraConf fs.oss.accessKeySecret=<AK secret> \u5c06\u7ed9\u547d\u4ee4\u63d0\u4f9b\u989d\u5916\u7684 AK key \u4e0e AK secret \u914d\u7f6e\u3002","title":"--extraConf"},{"location":"jindofs/jindofs_callerContext/#callercontext_1","text":"\u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf namespace.caller.context.enabled=true","title":"\u6253\u5f00CallerContext\u7ba1\u7406"},{"location":"jindofs/jindofs_callerContext/#callercontext_2","text":"\u547d\u4ee4\uff1a jindofs admin -getConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -name namespace.caller.context.enabled \u4e0a\u8ff0\u547d\u4ee4\u8fd4\u56de\u4e86\u540d\u4e3a \\<bucket>\uff08endpoint \u4e3a \\<oss-hdfs-endpoint>\uff09\u7684\u6876\u7684CallerContext\u7684\u6807\u7b7e","title":"\u67e5\u770bCallerContext\u7ba1\u7406"},{"location":"jindofs/jindofs_client_tools/","text":"JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 \u00a7 \u4ecb\u7ecd \u00a7 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u662f\u8bbf\u95ee OSS-HDFS \u7684\u53ef\u6267\u884c\u7a0b\u5e8f\uff0c\u529f\u80fd\u7c7b\u4f3c\u4e8e\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u7684 ossutil\u3002\u65e2\u652f\u6301\u5e38\u7528\u7684\u6587\u4ef6\u5143\u6570\u636e\u548c\u8bfb\u5199\u6d41\u64cd\u4f5c\uff0c\u5982\u521b\u5efa\u76ee\u5f55\u3001\u62f7\u8d1d\u6587\u4ef6\uff1b\u4e5f\u652f\u6301 OSS-HDFS \u7279\u6709\u7684\u65b9\u6cd5\uff0c\u5982\u6e05\u5355\u5bfc\u51fa\u7b49\u3002\u91c7\u7528\u4e86 Native \u5b9e\u73b0\uff0c\u76ee\u6807\u662f\u5bf9 OSS-HDFS \u63d0\u4f9b\u9ad8\u6548\u6613\u7528\u7684\u547d\u4ee4\u884c\u652f\u6301\u3002 \u4e0e JindoSDK \u63d0\u4f9b\u7684\u547d\u4ee4\u884c\u5de5\u5177\u76f8\u6bd4\uff0cJindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4e0d\u4ec5\u63d0\u4f9b\u5bf9\u6807 Hdfs shell \u7684\u5e38\u7528\u547d\u4ee4\uff0c\u8fd8\u63d0\u4f9b OSS-HDFS \u7684\u7279\u6709\u65b9\u6cd5\u3002\u7136\u800c\uff0cJindoFS \u547d\u4ee4\u884c\u5de5\u5177\u53ea\u652f\u6301 OSS-HDFS\uff0c\u4e3b\u8981\u4f5c\u4e3a OSS-HDFS \u7684\u7279\u6709\u5de5\u5177\u4f7f\u7528\u3002JindoSDK \u7684\u547d\u4ee4\u884c\u5de5\u5177\u53c2\u89c1 \u300aJindo CLI \u4f7f\u7528\u6307\u5357\u300b \u83b7\u53d6 \u00a7 \u76ee\u524d\u53ea\u63d0\u4f9b\u4e86 Linux \u7248\u672c\u7684\u547d\u4ee4\u884c\u5de5\u5177\uff0c \u4e0b\u8f7d\u94fe\u63a5 \u4e0b\u8f7d\u540e\u89e3\u538b\u7f29\uff0c\u5728 bin/ \u76ee\u5f55\u4e0b\u5373\u53ef\u627e\u5230\u547d\u4ee4\u884c\u5de5\u5177\u7684\u4e8c\u8fdb\u5236\u6587\u4ef6\uff0c\u6587\u4ef6\u540d\u4e3a jindofs \u3002 \u914d\u7f6e \u00a7 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528 JINDOSDK_CONF_DIR \u4e3a\u73af\u5883\u53d8\u91cf\u3002\u4f7f\u8be5\u53d8\u91cf\u6307\u5411\u4e00\u4e2a\u672c\u5730\u76ee\u5f55\uff0c\u5728\u76ee\u5f55\u4e0b\u653e\u7f6e\u914d\u7f6e\u6587\u4ef6 jindofs.cfg \uff0c\u4fbf\u53ef\u4ee5\u4e3a\u547d\u4ee4\u884c\u5de5\u5177\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\u3002 jindofs.cfg \u7684\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a [client] fs.oss.accessKeyId = <access key> fs.oss.accessKeySecret = <access secret> fs.oss.endpoint = <OSS-HDFS endpoint> fs.oss.data.lake.storage.enable = true # fs.oss.provider.endpoint = ECS_ROLE # EMR Credential provider for password-free access # fs.oss.provider.format = JSON \u5728\u89e3\u538b\u540e\u7684 conf/ \u76ee\u5f55\u4e0b\u4e5f\u53ef\u4ee5\u627e\u5230\u914d\u7f6e\u793a\u4f8b\u6587\u4ef6 jindofs.cfg.template \u3002 \u7528\u6cd5 \u00a7 \u5728\u547d\u4ee4\u884c\u4e2d\u6267\u884c jindofs \u4e8c\u8fdb\u5236\u6587\u4ef6\uff08\u4f8b\u5982\uff0c cd \u81f3\u6240\u5728\u76ee\u5f55\uff0c\u6267\u884c ./jindofs \u6216\u8005 ./jindofs -help \uff09\uff0c\u5c06\u663e\u793a\u6240\u6709\u652f\u6301\u7684\u547d\u4ee4\uff1a jindofs [-help] [-version] jindofs admin [-help] [-addProxyUser -dlsUri <path> -proxyUser <value> -users|-groups <value> -hosts <value>] [-allowSnapshot -dlsUri <path>] [-addUserGroupsMapping -dlsUri <path> -user <value> -groups <value>] [-disallowSnapshot -dlsUri <path>] [-deleteProxyUser -dlsUri <path> -proxyUser <value>] [-deleteUserGroupsMapping -dlsUri <path> -user <value>] [-dumpMetaInfo -dlsUri <path>] [-dumpInventory <path>] [-getJobProgress <jobId> -dlsUri <path>] [-listJobs -dlsUri <path> [-stages <stages>] [-createTime <createTime>] [-type <type>]] [-listProxyUsers -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-listUserGroupsMappings -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-snapshotDiff -dlsUri <path> -fromSnapshot <value> -toSnapshot <value>] [-setRootPolicy <dlsRootPath> <accessRootPath>] [-unsetRootPolicy <dlsRootPath> <accessRootPath>] [-listAccessPolicies <dlsRootPath>] [-putConfig -dlsUri <path> -conf <key1=value1> -conf <key2=value2> ...] [-getConfig -dlsUri <path> -name <keys>] jindofs fs [-help] [-cat <path>] [-count [-h] <path>] [-cp [-f] <src> <dst>] [-checksum [-mode COMPOSITE_CRC(default)/MD5MD5CRC] [-blockSize <size in bytes (only used in MD5MD5CRC mode, 128MB by default)>] <src> ...] [-chgrp [-R] <group> <path>] [-chmod [-R] <mode> <path>] [-chown [-R] [owner][:[group]] <path>] [-copyFromLocal [-f] <localsrc> <dst>] [-copyToLocal [-f] <src> <localdst>] [-createSnapshot <snapshotDir> [<snapshotName>]] [-checkStoragePolicy -path <path>] [-deleteSnapshot <snapshotDir> <snapshotName>] [-du [-s] [-h] <path> ...] [-expunge [-immediate] <path>] [-get [-f] <src> <localdst>] [-getfacl [-R] <path>] [-getfattr [-R] {-n name | -d} <path>] [-getStoragePolicy -path <path>] [-ls [-R] <path>] [-listPolicies] [-mkdir [-p] <path>] [-mv <src> ... <dst>] [-moveFromLocal <localsrc> ... <dst>] [-moveToLocal <src> <localdst>] [-put [-f] <localsrc> <dst>] [-rm [-skipTrash] [-f] [-r|-R] [-safely] <path>] [-rmdir <path>] [-test -[defsz] <path>] [-renameSnapshot <snapshotDir> <oldName> <newName>] [-stat [format] <path>] [-setStoragePolicy -path <path> -policy <policy> [-restoreDays <restoreDays>]] [-setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>]] [-setfattr {-n name [-v value] | -x name} <path>] [-touchz <path>] [-truncate [-w] <length> <path>] [-unsetStoragePolicy -path <path>] \u53e6\u5916\uff0c\u6240\u6709\u547d\u4ee4\u5747\u652f\u6301 --extraConf key=value \u7684\u65b9\u5f0f\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u4ec5\u5bf9\u8be5\u6b21\u6267\u884c\u6709\u6548\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u5199\u5728\u914d\u7f6e\u6587\u4ef6\u4e2d\u3002 \u793a\u4f8b \u00a7 \u4e0a\u4f20\u6587\u4ef6 \u00a7 ./jindofs fs -put /local/file/path oss://<bucket-name>.<oss-hdfs-endpoint>/remote/file/path \u4f5c\u7528\u662f\u628a\u672c\u5730\u6587\u4ef6 /local/file/path \u4e0a\u4f20\u5230\u8fdc\u7aef\u8def\u5f84 oss://<bucket-name>.<oss-hdfs-endpoint>/remote/file/path \u3002 \u6e05\u5355\u5bfc\u51fa \u00a7 ./jindofs admin -dumpInventory oss://<bucket-name>.<oss-hdfs-endpoint>/ --extraConf fs.oss.accessKeyId=<access key> --extraConf fs.oss.accessKeySecret=<access secret> \u4e0a\u8ff0\u547d\u4ee4\u5411 OSS-HDFS \u670d\u52a1\u53d1\u8d77\u4e86\u4e00\u6b21\u6e05\u5355\u5bfc\u51fa\u4efb\u52a1\uff0c\u5e76\u4e14\u8fd8\u901a\u8fc7 --extraConf key=value \u7684\u65b9\u5f0f\u4e3a\u672c\u6b21\u6267\u884c\u63d0\u4f9b\u4e86\u989d\u5916\u7684\u4e24\u6761\u914d\u7f6e\u4fe1\u606f\u3002JindoFS \u7684\u6240\u6709\u547d\u4ee4\u5747\u652f\u6301 --extraConf \u53c2\u6570\u3002 \u5982\u679c\u547d\u4ee4\u6210\u529f\u6267\u884c\uff0c\u5c06\u8fd4\u56de\uff1a =============Dump Inventory============= Job Id: <job id> Data Location: <oss-hdfs path to inventory file> ....................... FINISHED. \u968f\u540e\u53ef\u5728 Data Location \u6307\u5411\u7684 OSS-HDFS \u8def\u5f84\u627e\u5230\u6e05\u5355\u5bfc\u51fa\u7684\u7ed3\u679c\u6587\u4ef6\uff0c\u6309\u666e\u901a\u6587\u4ef6\u8bbf\u95ee\u5373\u53ef\u3002","title":"JindoFS Cli \u5feb\u901f\u5165\u95e8"},{"location":"jindofs/jindofs_client_tools/#jindofs","text":"","title":"JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357"},{"location":"jindofs/jindofs_client_tools/#_1","text":"JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u662f\u8bbf\u95ee OSS-HDFS \u7684\u53ef\u6267\u884c\u7a0b\u5e8f\uff0c\u529f\u80fd\u7c7b\u4f3c\u4e8e\u963f\u91cc\u4e91 OSS \u63d0\u4f9b\u7684 ossutil\u3002\u65e2\u652f\u6301\u5e38\u7528\u7684\u6587\u4ef6\u5143\u6570\u636e\u548c\u8bfb\u5199\u6d41\u64cd\u4f5c\uff0c\u5982\u521b\u5efa\u76ee\u5f55\u3001\u62f7\u8d1d\u6587\u4ef6\uff1b\u4e5f\u652f\u6301 OSS-HDFS \u7279\u6709\u7684\u65b9\u6cd5\uff0c\u5982\u6e05\u5355\u5bfc\u51fa\u7b49\u3002\u91c7\u7528\u4e86 Native \u5b9e\u73b0\uff0c\u76ee\u6807\u662f\u5bf9 OSS-HDFS \u63d0\u4f9b\u9ad8\u6548\u6613\u7528\u7684\u547d\u4ee4\u884c\u652f\u6301\u3002 \u4e0e JindoSDK \u63d0\u4f9b\u7684\u547d\u4ee4\u884c\u5de5\u5177\u76f8\u6bd4\uff0cJindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4e0d\u4ec5\u63d0\u4f9b\u5bf9\u6807 Hdfs shell \u7684\u5e38\u7528\u547d\u4ee4\uff0c\u8fd8\u63d0\u4f9b OSS-HDFS \u7684\u7279\u6709\u65b9\u6cd5\u3002\u7136\u800c\uff0cJindoFS \u547d\u4ee4\u884c\u5de5\u5177\u53ea\u652f\u6301 OSS-HDFS\uff0c\u4e3b\u8981\u4f5c\u4e3a OSS-HDFS \u7684\u7279\u6709\u5de5\u5177\u4f7f\u7528\u3002JindoSDK \u7684\u547d\u4ee4\u884c\u5de5\u5177\u53c2\u89c1 \u300aJindo CLI \u4f7f\u7528\u6307\u5357\u300b","title":"\u4ecb\u7ecd"},{"location":"jindofs/jindofs_client_tools/#_2","text":"\u76ee\u524d\u53ea\u63d0\u4f9b\u4e86 Linux \u7248\u672c\u7684\u547d\u4ee4\u884c\u5de5\u5177\uff0c \u4e0b\u8f7d\u94fe\u63a5 \u4e0b\u8f7d\u540e\u89e3\u538b\u7f29\uff0c\u5728 bin/ \u76ee\u5f55\u4e0b\u5373\u53ef\u627e\u5230\u547d\u4ee4\u884c\u5de5\u5177\u7684\u4e8c\u8fdb\u5236\u6587\u4ef6\uff0c\u6587\u4ef6\u540d\u4e3a jindofs \u3002","title":"\u83b7\u53d6"},{"location":"jindofs/jindofs_client_tools/#_3","text":"JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528 JINDOSDK_CONF_DIR \u4e3a\u73af\u5883\u53d8\u91cf\u3002\u4f7f\u8be5\u53d8\u91cf\u6307\u5411\u4e00\u4e2a\u672c\u5730\u76ee\u5f55\uff0c\u5728\u76ee\u5f55\u4e0b\u653e\u7f6e\u914d\u7f6e\u6587\u4ef6 jindofs.cfg \uff0c\u4fbf\u53ef\u4ee5\u4e3a\u547d\u4ee4\u884c\u5de5\u5177\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\u3002 jindofs.cfg \u7684\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a [client] fs.oss.accessKeyId = <access key> fs.oss.accessKeySecret = <access secret> fs.oss.endpoint = <OSS-HDFS endpoint> fs.oss.data.lake.storage.enable = true # fs.oss.provider.endpoint = ECS_ROLE # EMR Credential provider for password-free access # fs.oss.provider.format = JSON \u5728\u89e3\u538b\u540e\u7684 conf/ \u76ee\u5f55\u4e0b\u4e5f\u53ef\u4ee5\u627e\u5230\u914d\u7f6e\u793a\u4f8b\u6587\u4ef6 jindofs.cfg.template \u3002","title":"\u914d\u7f6e"},{"location":"jindofs/jindofs_client_tools/#_4","text":"\u5728\u547d\u4ee4\u884c\u4e2d\u6267\u884c jindofs \u4e8c\u8fdb\u5236\u6587\u4ef6\uff08\u4f8b\u5982\uff0c cd \u81f3\u6240\u5728\u76ee\u5f55\uff0c\u6267\u884c ./jindofs \u6216\u8005 ./jindofs -help \uff09\uff0c\u5c06\u663e\u793a\u6240\u6709\u652f\u6301\u7684\u547d\u4ee4\uff1a jindofs [-help] [-version] jindofs admin [-help] [-addProxyUser -dlsUri <path> -proxyUser <value> -users|-groups <value> -hosts <value>] [-allowSnapshot -dlsUri <path>] [-addUserGroupsMapping -dlsUri <path> -user <value> -groups <value>] [-disallowSnapshot -dlsUri <path>] [-deleteProxyUser -dlsUri <path> -proxyUser <value>] [-deleteUserGroupsMapping -dlsUri <path> -user <value>] [-dumpMetaInfo -dlsUri <path>] [-dumpInventory <path>] [-getJobProgress <jobId> -dlsUri <path>] [-listJobs -dlsUri <path> [-stages <stages>] [-createTime <createTime>] [-type <type>]] [-listProxyUsers -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-listUserGroupsMappings -dlsUri <path> [-maxKeys <value>] [-marker <value>]] [-snapshotDiff -dlsUri <path> -fromSnapshot <value> -toSnapshot <value>] [-setRootPolicy <dlsRootPath> <accessRootPath>] [-unsetRootPolicy <dlsRootPath> <accessRootPath>] [-listAccessPolicies <dlsRootPath>] [-putConfig -dlsUri <path> -conf <key1=value1> -conf <key2=value2> ...] [-getConfig -dlsUri <path> -name <keys>] jindofs fs [-help] [-cat <path>] [-count [-h] <path>] [-cp [-f] <src> <dst>] [-checksum [-mode COMPOSITE_CRC(default)/MD5MD5CRC] [-blockSize <size in bytes (only used in MD5MD5CRC mode, 128MB by default)>] <src> ...] [-chgrp [-R] <group> <path>] [-chmod [-R] <mode> <path>] [-chown [-R] [owner][:[group]] <path>] [-copyFromLocal [-f] <localsrc> <dst>] [-copyToLocal [-f] <src> <localdst>] [-createSnapshot <snapshotDir> [<snapshotName>]] [-checkStoragePolicy -path <path>] [-deleteSnapshot <snapshotDir> <snapshotName>] [-du [-s] [-h] <path> ...] [-expunge [-immediate] <path>] [-get [-f] <src> <localdst>] [-getfacl [-R] <path>] [-getfattr [-R] {-n name | -d} <path>] [-getStoragePolicy -path <path>] [-ls [-R] <path>] [-listPolicies] [-mkdir [-p] <path>] [-mv <src> ... <dst>] [-moveFromLocal <localsrc> ... <dst>] [-moveToLocal <src> <localdst>] [-put [-f] <localsrc> <dst>] [-rm [-skipTrash] [-f] [-r|-R] [-safely] <path>] [-rmdir <path>] [-test -[defsz] <path>] [-renameSnapshot <snapshotDir> <oldName> <newName>] [-stat [format] <path>] [-setStoragePolicy -path <path> -policy <policy> [-restoreDays <restoreDays>]] [-setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>]] [-setfattr {-n name [-v value] | -x name} <path>] [-touchz <path>] [-truncate [-w] <length> <path>] [-unsetStoragePolicy -path <path>] \u53e6\u5916\uff0c\u6240\u6709\u547d\u4ee4\u5747\u652f\u6301 --extraConf key=value \u7684\u65b9\u5f0f\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u4ec5\u5bf9\u8be5\u6b21\u6267\u884c\u6709\u6548\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u5199\u5728\u914d\u7f6e\u6587\u4ef6\u4e2d\u3002","title":"\u7528\u6cd5"},{"location":"jindofs/jindofs_client_tools/#_5","text":"","title":"\u793a\u4f8b"},{"location":"jindofs/jindofs_client_tools/#_6","text":"./jindofs fs -put /local/file/path oss://<bucket-name>.<oss-hdfs-endpoint>/remote/file/path \u4f5c\u7528\u662f\u628a\u672c\u5730\u6587\u4ef6 /local/file/path \u4e0a\u4f20\u5230\u8fdc\u7aef\u8def\u5f84 oss://<bucket-name>.<oss-hdfs-endpoint>/remote/file/path \u3002","title":"\u4e0a\u4f20\u6587\u4ef6"},{"location":"jindofs/jindofs_client_tools/#_7","text":"./jindofs admin -dumpInventory oss://<bucket-name>.<oss-hdfs-endpoint>/ --extraConf fs.oss.accessKeyId=<access key> --extraConf fs.oss.accessKeySecret=<access secret> \u4e0a\u8ff0\u547d\u4ee4\u5411 OSS-HDFS \u670d\u52a1\u53d1\u8d77\u4e86\u4e00\u6b21\u6e05\u5355\u5bfc\u51fa\u4efb\u52a1\uff0c\u5e76\u4e14\u8fd8\u901a\u8fc7 --extraConf key=value \u7684\u65b9\u5f0f\u4e3a\u672c\u6b21\u6267\u884c\u63d0\u4f9b\u4e86\u989d\u5916\u7684\u4e24\u6761\u914d\u7f6e\u4fe1\u606f\u3002JindoFS \u7684\u6240\u6709\u547d\u4ee4\u5747\u652f\u6301 --extraConf \u53c2\u6570\u3002 \u5982\u679c\u547d\u4ee4\u6210\u529f\u6267\u884c\uff0c\u5c06\u8fd4\u56de\uff1a =============Dump Inventory============= Job Id: <job id> Data Location: <oss-hdfs path to inventory file> ....................... FINISHED. \u968f\u540e\u53ef\u5728 Data Location \u6307\u5411\u7684 OSS-HDFS \u8def\u5f84\u627e\u5230\u6e05\u5355\u5bfc\u51fa\u7684\u7ed3\u679c\u6587\u4ef6\uff0c\u6309\u666e\u901a\u6587\u4ef6\u8bbf\u95ee\u5373\u53ef\u3002","title":"\u6e05\u5355\u5bfc\u51fa"},{"location":"jindofs/jindofs_java_sdk/","text":"JindoFS Java SDK \u4f7f\u7528\u6307\u5357 \u00a7 \u4ecb\u7ecd \u00a7 JindoFS Java SDK \u63d0\u4f9b\u4e86\u8bbf\u95ee OSS-HDFS \u6570\u636e\u6e56\u5b58\u50a8\u670d\u52a1\u7684 Java API\u3002\u5b83\u652f\u6301\u5e38\u89c1\u7684\u6587\u4ef6\u7cfb\u7edf\u5143\u6570\u636e\u63a5\u53e3\u548c\u6d41\u5f0f\u6570\u636e\u64cd\u4f5c\uff0c\u5982\u521b\u5efa\u76ee\u5f55\u3001\u8bfb\u5199\u6216\u5220\u9664\u6587\u4ef6\uff1b\u540c\u65f6\u4e5f\u652f\u6301 OSS-HDFS \u7279\u6709\u7684\u65b9\u6cd5\uff0c\u5982\u6e05\u5355\u5bfc\u51fa\u7b49\u3002 JindoFS Java SDK \u5e76\u672a\u63d0\u4f9b\u57fa\u4e8e Hadoop-Compatible FileSystem\uff08HCFS\uff09\u7684\u5b9e\u73b0\u3002\u5982\u679c\u5e0c\u671b\u5728 Hadoop \u751f\u6001\u4e2d\u4ee5 HCFS \u6807\u51c6\u63a5\u53e3\u8bbf\u95ee OSS-HDFS\uff0c\u8bf7\u53c2\u8003 \u300aHadoop \u73af\u5883\u4f7f\u7528 JindoSDK \u5feb\u901f\u5165\u95e8\u300b \u672c\u6587\u6863\u57fa\u4e8e JindoFS Java SDK 6.2.5 \u7248\u672c\u7f16\u5199\u3002 \u4e0e JindoSDK \u7684\u533a\u522b \u00a7 JindoSDK \u540c\u6837\u63d0\u4f9b\u4e86\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684 Java API\u3002\u4e0e\u4e4b\u76f8\u6bd4\uff0cJindoFS Java SDK \u6709\u4e09\u4e2a\u4e3b\u8981\u533a\u522b\uff1a 1. JindoFS Java SDK \u4e0d\u8003\u8651\u539f\u751f\u652f\u6301 Hadoop \u751f\u6001\uff0c\u4e0d\u63d0\u4f9b\u57fa\u4e8e HCFS \u7684\u5b9e\u73b0\uff0c\u4e5f\u4e0d\u4f1a\u4ece Hadoop core-site \u8bfb\u53d6\u914d\u7f6e\u3002 2. JindoFS Java SDK \u53ea\u652f\u6301 OSS-HDFS\uff0c\u800c JindoSDK \u9664\u4e86\u652f\u6301 OSS-HDFS \u4ee5\u5916\uff0c\u8fd8\u53ef\u4ee5\u652f\u6301\u666e\u901a OSS\u3001S3\u3001HDFS \u7b49\u591a\u79cd\u5b58\u50a8\u670d\u52a1\u3002 3. JindoFS Java SDK \u63d0\u4f9b\u4e86\u4e00\u4e9b OSS-HDFS \u7279\u6709\u7684\u63a5\u53e3\uff0c\u8fd9\u4e9b\u63a5\u53e3\u53ef\u80fd\u662f JindoSDK \u672a\u63d0\u4f9b\u7684\uff0c\u4f8b\u5982\u6e05\u5355\u5bfc\u51fa\u7b49\u3002 \u5982\u679c\u5e0c\u671b\u4f7f\u7528 HCFS \u7684\u6587\u4ef6\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u300aHadoop \u73af\u5883\u4f7f\u7528 JindoSDK \u5feb\u901f\u5165\u95e8\u300b \uff0c\u5bf9 Hadoop \u751f\u6001 HCFS \u7684\u4ecb\u7ecd\u8bf7\u53c2\u9605 Apache Hadoop \u6587\u6863\u3002 \u83b7\u53d6 \u00a7 JindoFS Java SDK \u7684\u4f7f\u7528\u6709\u5e73\u53f0\u9650\u5236\uff0c\u76ee\u524d\u53ea\u63d0\u4f9b\u4e86 Linux x86 \u5e73\u53f0\u7684\u7248\u672c\uff0c \u4e0b\u8f7d\u94fe\u63a5 \u6ce8\uff1a\u5982\u679c\u5e0c\u671b\u5728 MacOS\u3001Ubuntu\u3001\u6216\u8005 Arch \u67b6\u6784\u4e0b\u4f7f\u7528 JindoFS Java SDK\uff0c\u8bf7\u8054\u7cfb JindoFS \u56e2\u961f\u63d0\u4f9b\u652f\u6301\u3002 \u4e0b\u8f7d\u540e\u89e3\u538b\u7f29\uff0c\u5728 lib/ \u76ee\u5f55\u4e0b\u5373\u53ef\u627e\u5230 JindoFS Java SDK \u7684\u8d44\u6e90\uff1a - jindofs-core-${version}.jar - jindofs-sdk-${version}.jar Maven \u00a7 \u76ee\u524d\u6682\u672a\u63d0\u4f9b\u53ef\u4ee5\u516c\u5171\u8bbf\u95ee\u7684 Maven \u4ed3\u5e93\u3002\u60a8\u53ef\u4ee5\u672c\u5730\u4f9d\u8d56\u8fd9\u4e24\u4e2a jar \u6587\u4ef6\uff0c\u6216\u8005\u5c06\u5b83\u4eec\u653e\u7f6e\u5728\u4e00\u4e2a\u60a8\u65b9\u4fbf\u8bbf\u95ee\u7684 Maven \u4ed3\u5e93\u4e2d\u3002 Maven \u4f9d\u8d56\u5199\u4f5c\uff1a <dependencies> <dependency> <groupId>com.aliyun.jindodata.jindofs</groupId> <artifactId>jindofs-core</artifactId> <version>${version}</version> </dependency> <dependency> <groupId>com.aliyun.jindodata.jindofs</groupId> <artifactId>jindofs-sdk</artifactId> <version>${version}</version> </dependency> </dependencies> \u5176\u4e2d ${version} \u6839\u636e\u5b9e\u9645\u7684\u7248\u672c\u53f7\u586b\u5199\u3002 \u4f7f\u7528 \u00a7 JindoFS Java SDK \u63d0\u4f9b\u4e86 JfsFileSystem \u7684\u63a5\u53e3\uff0c\u4ee5\u53ca\u4e00\u4e2a JfsFileSystemImpl \u7684\u5b9e\u73b0\u3002\u5ba2\u6237\u7aef\u521d\u59cb\u5316\u7684\u65b9\u5f0f\u4e3a\uff1a JfsFileSystem fs = new JfsFileSystemImpl(); fs.initialize(uri, user, conf); \u5176\u4e2d\uff1a 1. uri \u6307\u5411 OSS-HDFS \u7684\u521d\u59cb\u5316\u5730\u5740\uff0c\u4f8b\u5982 oss://<your-oss-hdfs-bucket>.<oss-hdfs-endpoint>/init-path 2. user \u8868\u793a\u5ba2\u6237\u7aef\u4ee3\u8868\u7684\u7528\u6237\u540d\u79f0\u3002 3. conf \u662f\u4e00\u4e2a JfsConfiguration \u5bf9\u8c61\u3002 JfsConfiguration \u662f\u5ba2\u6237\u7aef\u63a5\u53d7\u914d\u7f6e\u7684\u552f\u4e00\u9014\u5f84\uff0cJindoFS Java SDK \u672c\u8eab\u4e0d\u4f1a\u8bfb\u53d6\u4efb\u4f55\u914d\u7f6e\u6587\u4ef6\u3002\u901a\u8fc7 JfsConfiguration \u53ef\u4ee5\u8bbe\u7f6e\u591a\u79cd\u914d\u7f6e\u53c2\u6570\uff0c\u5176\u4e2d JfsConfiguration.JfsAccessInfo \u5305\u542b\u8bbf\u95ee OSS-HDFS \u5fc5\u8981\u7684\u4fe1\u606f\uff0c\u5176\u4ed6\u914d\u7f6e\u9879\u5747\u53ef\u4ee5\u91c7\u7528\u9ed8\u8ba4\u503c\u3002 \u6784\u9020 JfsConfiguration.JfsAccessInfo \u5e76\u8fdb\u4e00\u6b65\u521b\u5efa JfsConfiguration \u7684\u65b9\u6cd5\u5982\u4e0b\uff1a JfsConfiguration.JfsAccessInfo accessInfo = new JfsConfiguration.JfsAccessInfo(); accessInfo.accessKeyId = \"your access key ID\"; accessInfo.accessKeySecret = \"your access key secret\"; accessInfo.endpoint = \"oss-hdfs-endpoint\"; JfsConfiguration conf = new JfsConfiguration.Builder() .setDefaultAccessInfo(accessInfo) .build(); \u6ce8\u610f\u4e8b\u9879 \u00a7 \u4e0e Hadoop \u751f\u6001\u7684\u4f7f\u7528\u4e60\u60ef\u76f8\u6096\u7684\u662f\uff0c JfsFileSystem \u7684\u5b9e\u4f8b\u6ca1\u6709\u5168\u5c40\u5171\u4eab\u7684\u673a\u5236\uff0c\u4e5f\u4e0d\u63d0\u4f9b\u4ece\u8def\u5f84\u83b7\u53d6 JfsFileSystem \u5b9e\u4f8b\u7684\u65b9\u6cd5\u3002\u5e76\u4e14\uff0c\u6bcf\u4e00\u4e2a\u65b0\u521b\u5efa\u7684 JfsFileSystem \u5b9e\u4f8b\u5728\u4f7f\u7528\u5b8c\u4e4b\u540e\u90fd\u9700\u8981\u4e3b\u52a8\u8c03\u7528 close \u5173\u95ed\uff0c\u5426\u5219\u53ef\u80fd\u4f1a\u4ea7\u751f\u8d44\u6e90\u6cc4\u9732\u3002\u53e6\u4e00\u65b9\u9762\uff0c\u5bf9\u4e8e\u8bfb\u5199\u6d41 JfsInputStream \u4e0e JfsOutputStream \u7684\u5bf9\u8c61\uff0c\u5982\u679c\u4e0d\u8c03\u7528 close \u5e76\u6ca1\u6709\u8d44\u6e90\u6cc4\u9732\u4e4b\u865e\u3002 \u793a\u4f8b \u00a7 \u4e0b\u9762\u662f\u4e00\u4e2a\u5b8c\u6574\u7684\u4ee3\u7801\u793a\u4f8b\uff0c\u7528\u6765\u6f14\u793a\u5982\u4f55\u521d\u59cb\u5316\u4e00\u4e2a JfsFileSystem \u5bf9\u8c61\uff0c\u5e76\u4e14\u5728 OSS-HDFS \u4e2d\u521b\u5efa\u4e00\u4e2a\u76ee\u5f55\uff1a package com.aliyun.jindodata.jfssdk.demo; import com.aliyun.jindodata.jfssdk.JfsConfiguration; import com.aliyun.jindodata.jfssdk.JfsFileSystem; import com.aliyun.jindodata.jfssdk.JfsFileSystemImpl; import java.io.IOException; public class Demo { public static void main(String[] args) throws IOException { JfsConfiguration.JfsAccessInfo accessInfo = new JfsConfiguration.JfsAccessInfo(); accessInfo.accessKeyId = \"your access key ID\"; accessInfo.accessKeySecret = \"your access key secret\"; accessInfo.endpoint = \"oss-hdfs-endpoint\"; JfsConfiguration conf = new JfsConfiguration.Builder() .setDefaultAccessInfo(accessInfo) .build(); String uri = \"oss://<your-oss-hdfs-bucket>.<oss-hdfs-endpoint>/init-uri\"; try (JfsFileSystem fs = new JfsFileSystemImpl()) { fs.initialize(uri, \"hadoop\", conf); fs.mkdirs(\"oss://<your-oss-hdfs-bucket>.<oss-hdfs-endpoint>/a-new-dir\"); } } } \u5176\u4e2d\uff0c your-oss-hdfs-bucket \u3001 your access key ID \u3001 your access key secret \u3001 oss-hdfs-endpoint \u6839\u636e\u5b9e\u9645\u503c\u586b\u5199\u3002","title":"JindoFS Java SDK \u4f7f\u7528\u6307\u5357"},{"location":"jindofs/jindofs_java_sdk/#jindofs-java-sdk","text":"","title":"JindoFS Java SDK \u4f7f\u7528\u6307\u5357"},{"location":"jindofs/jindofs_java_sdk/#_1","text":"JindoFS Java SDK \u63d0\u4f9b\u4e86\u8bbf\u95ee OSS-HDFS \u6570\u636e\u6e56\u5b58\u50a8\u670d\u52a1\u7684 Java API\u3002\u5b83\u652f\u6301\u5e38\u89c1\u7684\u6587\u4ef6\u7cfb\u7edf\u5143\u6570\u636e\u63a5\u53e3\u548c\u6d41\u5f0f\u6570\u636e\u64cd\u4f5c\uff0c\u5982\u521b\u5efa\u76ee\u5f55\u3001\u8bfb\u5199\u6216\u5220\u9664\u6587\u4ef6\uff1b\u540c\u65f6\u4e5f\u652f\u6301 OSS-HDFS \u7279\u6709\u7684\u65b9\u6cd5\uff0c\u5982\u6e05\u5355\u5bfc\u51fa\u7b49\u3002 JindoFS Java SDK \u5e76\u672a\u63d0\u4f9b\u57fa\u4e8e Hadoop-Compatible FileSystem\uff08HCFS\uff09\u7684\u5b9e\u73b0\u3002\u5982\u679c\u5e0c\u671b\u5728 Hadoop \u751f\u6001\u4e2d\u4ee5 HCFS \u6807\u51c6\u63a5\u53e3\u8bbf\u95ee OSS-HDFS\uff0c\u8bf7\u53c2\u8003 \u300aHadoop \u73af\u5883\u4f7f\u7528 JindoSDK \u5feb\u901f\u5165\u95e8\u300b \u672c\u6587\u6863\u57fa\u4e8e JindoFS Java SDK 6.2.5 \u7248\u672c\u7f16\u5199\u3002","title":"\u4ecb\u7ecd"},{"location":"jindofs/jindofs_java_sdk/#jindosdk","text":"JindoSDK \u540c\u6837\u63d0\u4f9b\u4e86\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684 Java API\u3002\u4e0e\u4e4b\u76f8\u6bd4\uff0cJindoFS Java SDK \u6709\u4e09\u4e2a\u4e3b\u8981\u533a\u522b\uff1a 1. JindoFS Java SDK \u4e0d\u8003\u8651\u539f\u751f\u652f\u6301 Hadoop \u751f\u6001\uff0c\u4e0d\u63d0\u4f9b\u57fa\u4e8e HCFS \u7684\u5b9e\u73b0\uff0c\u4e5f\u4e0d\u4f1a\u4ece Hadoop core-site \u8bfb\u53d6\u914d\u7f6e\u3002 2. JindoFS Java SDK \u53ea\u652f\u6301 OSS-HDFS\uff0c\u800c JindoSDK \u9664\u4e86\u652f\u6301 OSS-HDFS \u4ee5\u5916\uff0c\u8fd8\u53ef\u4ee5\u652f\u6301\u666e\u901a OSS\u3001S3\u3001HDFS \u7b49\u591a\u79cd\u5b58\u50a8\u670d\u52a1\u3002 3. JindoFS Java SDK \u63d0\u4f9b\u4e86\u4e00\u4e9b OSS-HDFS \u7279\u6709\u7684\u63a5\u53e3\uff0c\u8fd9\u4e9b\u63a5\u53e3\u53ef\u80fd\u662f JindoSDK \u672a\u63d0\u4f9b\u7684\uff0c\u4f8b\u5982\u6e05\u5355\u5bfc\u51fa\u7b49\u3002 \u5982\u679c\u5e0c\u671b\u4f7f\u7528 HCFS \u7684\u6587\u4ef6\u7cfb\u7edf\uff0c\u8bf7\u53c2\u8003 \u300aHadoop \u73af\u5883\u4f7f\u7528 JindoSDK \u5feb\u901f\u5165\u95e8\u300b \uff0c\u5bf9 Hadoop \u751f\u6001 HCFS \u7684\u4ecb\u7ecd\u8bf7\u53c2\u9605 Apache Hadoop \u6587\u6863\u3002","title":"\u4e0e JindoSDK \u7684\u533a\u522b"},{"location":"jindofs/jindofs_java_sdk/#_2","text":"JindoFS Java SDK \u7684\u4f7f\u7528\u6709\u5e73\u53f0\u9650\u5236\uff0c\u76ee\u524d\u53ea\u63d0\u4f9b\u4e86 Linux x86 \u5e73\u53f0\u7684\u7248\u672c\uff0c \u4e0b\u8f7d\u94fe\u63a5 \u6ce8\uff1a\u5982\u679c\u5e0c\u671b\u5728 MacOS\u3001Ubuntu\u3001\u6216\u8005 Arch \u67b6\u6784\u4e0b\u4f7f\u7528 JindoFS Java SDK\uff0c\u8bf7\u8054\u7cfb JindoFS \u56e2\u961f\u63d0\u4f9b\u652f\u6301\u3002 \u4e0b\u8f7d\u540e\u89e3\u538b\u7f29\uff0c\u5728 lib/ \u76ee\u5f55\u4e0b\u5373\u53ef\u627e\u5230 JindoFS Java SDK \u7684\u8d44\u6e90\uff1a - jindofs-core-${version}.jar - jindofs-sdk-${version}.jar","title":"\u83b7\u53d6"},{"location":"jindofs/jindofs_java_sdk/#maven","text":"\u76ee\u524d\u6682\u672a\u63d0\u4f9b\u53ef\u4ee5\u516c\u5171\u8bbf\u95ee\u7684 Maven \u4ed3\u5e93\u3002\u60a8\u53ef\u4ee5\u672c\u5730\u4f9d\u8d56\u8fd9\u4e24\u4e2a jar \u6587\u4ef6\uff0c\u6216\u8005\u5c06\u5b83\u4eec\u653e\u7f6e\u5728\u4e00\u4e2a\u60a8\u65b9\u4fbf\u8bbf\u95ee\u7684 Maven \u4ed3\u5e93\u4e2d\u3002 Maven \u4f9d\u8d56\u5199\u4f5c\uff1a <dependencies> <dependency> <groupId>com.aliyun.jindodata.jindofs</groupId> <artifactId>jindofs-core</artifactId> <version>${version}</version> </dependency> <dependency> <groupId>com.aliyun.jindodata.jindofs</groupId> <artifactId>jindofs-sdk</artifactId> <version>${version}</version> </dependency> </dependencies> \u5176\u4e2d ${version} \u6839\u636e\u5b9e\u9645\u7684\u7248\u672c\u53f7\u586b\u5199\u3002","title":"Maven"},{"location":"jindofs/jindofs_java_sdk/#_3","text":"JindoFS Java SDK \u63d0\u4f9b\u4e86 JfsFileSystem \u7684\u63a5\u53e3\uff0c\u4ee5\u53ca\u4e00\u4e2a JfsFileSystemImpl \u7684\u5b9e\u73b0\u3002\u5ba2\u6237\u7aef\u521d\u59cb\u5316\u7684\u65b9\u5f0f\u4e3a\uff1a JfsFileSystem fs = new JfsFileSystemImpl(); fs.initialize(uri, user, conf); \u5176\u4e2d\uff1a 1. uri \u6307\u5411 OSS-HDFS \u7684\u521d\u59cb\u5316\u5730\u5740\uff0c\u4f8b\u5982 oss://<your-oss-hdfs-bucket>.<oss-hdfs-endpoint>/init-path 2. user \u8868\u793a\u5ba2\u6237\u7aef\u4ee3\u8868\u7684\u7528\u6237\u540d\u79f0\u3002 3. conf \u662f\u4e00\u4e2a JfsConfiguration \u5bf9\u8c61\u3002 JfsConfiguration \u662f\u5ba2\u6237\u7aef\u63a5\u53d7\u914d\u7f6e\u7684\u552f\u4e00\u9014\u5f84\uff0cJindoFS Java SDK \u672c\u8eab\u4e0d\u4f1a\u8bfb\u53d6\u4efb\u4f55\u914d\u7f6e\u6587\u4ef6\u3002\u901a\u8fc7 JfsConfiguration \u53ef\u4ee5\u8bbe\u7f6e\u591a\u79cd\u914d\u7f6e\u53c2\u6570\uff0c\u5176\u4e2d JfsConfiguration.JfsAccessInfo \u5305\u542b\u8bbf\u95ee OSS-HDFS \u5fc5\u8981\u7684\u4fe1\u606f\uff0c\u5176\u4ed6\u914d\u7f6e\u9879\u5747\u53ef\u4ee5\u91c7\u7528\u9ed8\u8ba4\u503c\u3002 \u6784\u9020 JfsConfiguration.JfsAccessInfo \u5e76\u8fdb\u4e00\u6b65\u521b\u5efa JfsConfiguration \u7684\u65b9\u6cd5\u5982\u4e0b\uff1a JfsConfiguration.JfsAccessInfo accessInfo = new JfsConfiguration.JfsAccessInfo(); accessInfo.accessKeyId = \"your access key ID\"; accessInfo.accessKeySecret = \"your access key secret\"; accessInfo.endpoint = \"oss-hdfs-endpoint\"; JfsConfiguration conf = new JfsConfiguration.Builder() .setDefaultAccessInfo(accessInfo) .build();","title":"\u4f7f\u7528"},{"location":"jindofs/jindofs_java_sdk/#_4","text":"\u4e0e Hadoop \u751f\u6001\u7684\u4f7f\u7528\u4e60\u60ef\u76f8\u6096\u7684\u662f\uff0c JfsFileSystem \u7684\u5b9e\u4f8b\u6ca1\u6709\u5168\u5c40\u5171\u4eab\u7684\u673a\u5236\uff0c\u4e5f\u4e0d\u63d0\u4f9b\u4ece\u8def\u5f84\u83b7\u53d6 JfsFileSystem \u5b9e\u4f8b\u7684\u65b9\u6cd5\u3002\u5e76\u4e14\uff0c\u6bcf\u4e00\u4e2a\u65b0\u521b\u5efa\u7684 JfsFileSystem \u5b9e\u4f8b\u5728\u4f7f\u7528\u5b8c\u4e4b\u540e\u90fd\u9700\u8981\u4e3b\u52a8\u8c03\u7528 close \u5173\u95ed\uff0c\u5426\u5219\u53ef\u80fd\u4f1a\u4ea7\u751f\u8d44\u6e90\u6cc4\u9732\u3002\u53e6\u4e00\u65b9\u9762\uff0c\u5bf9\u4e8e\u8bfb\u5199\u6d41 JfsInputStream \u4e0e JfsOutputStream \u7684\u5bf9\u8c61\uff0c\u5982\u679c\u4e0d\u8c03\u7528 close \u5e76\u6ca1\u6709\u8d44\u6e90\u6cc4\u9732\u4e4b\u865e\u3002","title":"\u6ce8\u610f\u4e8b\u9879"},{"location":"jindofs/jindofs_java_sdk/#_5","text":"\u4e0b\u9762\u662f\u4e00\u4e2a\u5b8c\u6574\u7684\u4ee3\u7801\u793a\u4f8b\uff0c\u7528\u6765\u6f14\u793a\u5982\u4f55\u521d\u59cb\u5316\u4e00\u4e2a JfsFileSystem \u5bf9\u8c61\uff0c\u5e76\u4e14\u5728 OSS-HDFS \u4e2d\u521b\u5efa\u4e00\u4e2a\u76ee\u5f55\uff1a package com.aliyun.jindodata.jfssdk.demo; import com.aliyun.jindodata.jfssdk.JfsConfiguration; import com.aliyun.jindodata.jfssdk.JfsFileSystem; import com.aliyun.jindodata.jfssdk.JfsFileSystemImpl; import java.io.IOException; public class Demo { public static void main(String[] args) throws IOException { JfsConfiguration.JfsAccessInfo accessInfo = new JfsConfiguration.JfsAccessInfo(); accessInfo.accessKeyId = \"your access key ID\"; accessInfo.accessKeySecret = \"your access key secret\"; accessInfo.endpoint = \"oss-hdfs-endpoint\"; JfsConfiguration conf = new JfsConfiguration.Builder() .setDefaultAccessInfo(accessInfo) .build(); String uri = \"oss://<your-oss-hdfs-bucket>.<oss-hdfs-endpoint>/init-uri\"; try (JfsFileSystem fs = new JfsFileSystemImpl()) { fs.initialize(uri, \"hadoop\", conf); fs.mkdirs(\"oss://<your-oss-hdfs-bucket>.<oss-hdfs-endpoint>/a-new-dir\"); } } } \u5176\u4e2d\uff0c your-oss-hdfs-bucket \u3001 your access key ID \u3001 your access key secret \u3001 oss-hdfs-endpoint \u6839\u636e\u5b9e\u9645\u503c\u586b\u5199\u3002","title":"\u793a\u4f8b"},{"location":"jindofs/jindofs_protected_directories/","text":"\u80cc\u666f \u00a7 JindoFS \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u5f53\u524d\u53ef\u4ee5\u901a\u8fc7\u547d\u4ee4\u884c\u547d\u4ee4\u7684\u65b9\u5f0f\u8fdb\u884c\u64cd\u4f5c\uff0c\u53ef\u4ee5\u663e\u793a\u6b63\u5728\u4fdd\u62a4\u7684\u76ee\u5f55\u5217\u8868\uff0c\u6216\u8005\u4fee\u6539\u4fdd\u62a4\u7684\u76ee\u5f55\u3002\u5c06\u6765\u8fd8\u4f1a\u901a\u8fc7\u4ea7\u54c1\u754c\u9762\u7684\u65b9\u5f0f\uff0c\u63d0\u4f9b\u66f4\u52a0\u4fbf\u6377\u6613\u7528\u7684\u67e5\u770b\u548c\u8bbe\u7f6e\u65b9\u6cd5\u3002 \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u63cf\u8ff0 \u00a7 JindoFS \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\uff0c\u4e3b\u8981\u9632\u6b62\u7528\u6237\u8bef\u5220\u76ee\u5f55\uff0c\u88ab\u4fdd\u62a4\u7684\u76ee\u5f55\u53ea\u8981\u8be5\u76ee\u5f55\u4e0b\u5b58\u5728\u76ee\u5f55\u6216\u8005\u6587\u4ef6\uff0c\u4fdd\u62a4\u76ee\u5f55\u7684\u5220\u9664\u4ee5\u53ca\u91cd\u547d\u540d\u64cd\u4f5c\u4f1a\u62a5\u9519\uff0c\u53ea\u6709\u5f53\u88ab\u4fdd\u62a4\u7684\u76ee\u5f55\u4e0b\u4e0d\u5b58\u5728\u4efb\u4f55\u76ee\u5f55\u6216\u8005\u6587\u4ef6\uff0c\u4fdd\u62a4\u76ee\u5f55\u624d\u4f1a\u88ab\u5141\u8bb8\u5220\u9664\u6216\u8005\u91cd\u547d\u540d\u3002 JindoFS \u547d\u4ee4\u884c\u7b80\u4ecb \u00a7 \u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u9700\u8981\u7528\u5230\u7684\u662f -putConfig \u4e0e -getConfig \u547d\u4ee4\uff0c\u5177\u4f53\u7528\u6cd5\u5c06\u5728\u4e0b\u6587\u4ecb\u7ecd\u3002 \u914d\u7f6e\u8bbf\u95ee AK \u00a7 \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u9700\u8981\u914d\u7f6e\u8fde\u63a5 OSS-HDFS \u6240\u9700\u7684 AK \u4fe1\u606f\u3002\u6709\u4e24\u79cd\u65b9\u6cd5\u53ef\u4ee5\u8fdb\u884c\u914d\u7f6e\uff1a \u914d\u7f6e\u6587\u4ef6\uff08\u63a8\u8350\uff09 \u00a7 \u8bbe\u7f6e\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6307\u5411\u67d0\u4e2a\u76ee\u5f55\uff0c\u7136\u540e\u5728\u76ee\u5f55\u4e0b\u521b\u5efa\u540d\u4e3a jindofs.cfg \u7684\u6587\u4ef6\uff0c\u7136\u540e\u5728\u6587\u4ef6\u91cc\u5199\u5165\u5982\u4e0b\u5185\u5bb9\uff1a [client] fs.oss.accessKeyId = <key> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK key fs.oss.accessKeySecret = <secret> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK secret fs.oss.bucket.<bucket>.accessKeyId = <key> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK key\uff0c\u4f18\u5148\u7ea7\u9ad8\u4e8e\u9ed8\u8ba4 fs.oss.bucket.<bucket>.accessKeySecret = <secret> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK secret --extraConf \u00a7 \u547d\u4ee4\u884c\u5de5\u5177\u7684\u6240\u6709\u6307\u4ee4\u5747\u53ef\u901a\u8fc7 --extraConf \u9009\u9879\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u914d\u7f6e\u6587\u4ef6\uff0c\u4f8b\u5982\uff1a jindofs admin -getConfig -dlsUri <path> -name <keys> \\ --extraConf fs.oss.accessKeyId=<AK key> \\ --extraConf fs.oss.accessKeySecret=<AK secret> \u5c06\u7ed9\u547d\u4ee4\u63d0\u4f9b\u989d\u5916\u7684 AK key \u4e0e AK secret \u914d\u7f6e\u3002 \u8bbe\u7f6e\u4fdd\u62a4\u76ee\u5f55 \u00a7 \u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf fs.protected.directories=/path/to/dir1,/path/to/dir2 \u4e0a\u8ff0\u547d\u4ee4\u5bf9\u540d\u4e3a \uff08endpoint \u4e3a \uff09\u7684\u6876\u914d\u7f6e\u4e86\u4e24\u4e2a\u4fdd\u62a4\u76ee\u5f55\uff1a /path/to/dir1 /path/to/dir2 \u5982\u793a\u4f8b\u4e2d\u7684\u683c\u5f0f\uff0c\u8fd9\u4e9b\u4fdd\u62a4\u76ee\u5f55\u7684\u8def\u5f84\u9700\u8981\u5199\u5728\u4e00\u8d77\uff0c\u9017\u53f7\u9694\u5f00\u3002\u8def\u5f84\u4e0d\u4ee5 oss:// \u5f00\u5934\uff0c\u800c\u662f\u4f7f\u7528\u6876\u5185\u4ee5 / \u5f00\u5934\u7684\u7edd\u5bf9\u8def\u5f84\u3002\u5982\u679c\u547d\u4ee4\u5931\u8d25\uff0c\u5c06\u663e\u793a\u5931\u8d25\u4fe1\u606f\u3002\u5982\u679c\u6ca1\u6709\u4efb\u4f55\u4fe1\u606f\u663e\u793a\uff0c\u8868\u660e\u6267\u884c\u6210\u529f\u3002\u547d\u4ee4\u6267\u884c\u6210\u529f\u4e4b\u540e\uff0c\u5bf9\u8fd9\u4e9b\u76ee\u5f55\u7684\u4fdd\u62a4\u529f\u80fd\u5c06\u5728 30 \u79d2\u5185\u6b63\u5f0f\u751f\u6548\u3002 \u67e5\u770b\u4fdd\u62a4\u76ee\u5f55 \u00a7 \u547d\u4ee4\uff1a jindofs admin -getConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -name fs.protected.directories \u4e0a\u8ff0\u547d\u4ee4\u8fd4\u56de\u4e86\u540d\u4e3a \uff08endpoint \u4e3a \uff09\u7684\u6876\u7684\u4fdd\u62a4\u76ee\u5f55\u5217\u8868\uff0c\u8fd4\u56de\u503c\u793a\u4f8b\uff1a fs.protected.directories: /path/to/dir1,/path/to/dir2 \u8868\u660e\u8fd9\u4e9b\u76ee\u5f55\u76ee\u524d\u6b63\u5728\u4fdd\u62a4\u5f53\u4e2d\uff1a /path/to/dir1 /path/to/dir2 \u5220\u9664\u6240\u6709\u4fdd\u62a4\u76ee\u5f55 \u00a7 \u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf fs.protected.directories= \u4e0a\u8ff0\u547d\u4ee4\u5220\u9664\u4e86\u540d\u4e3a \\<bucket>\uff08endpoint \u4e3a \\<oss-hdfs-endpoint>\uff09\u7684\u6876\u7684\u6240\u6709\u4fdd\u62a4\u76ee\u5f55\uff0c\u6b63\u5f0f\u751f\u6548\u540e\uff08\u547d\u4ee4\u8fd4\u56de\u540e 30 \u79d2\u5185\uff09\uff0c\u8be5\u6876\u5c06\u4e0d\u518d\u6709\u4efb\u4f55\u76ee\u5f55\u53d7\u5230\u4fdd\u62a4\u3002","title":"Jindofs protected directories"},{"location":"jindofs/jindofs_protected_directories/#_1","text":"JindoFS \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u5f53\u524d\u53ef\u4ee5\u901a\u8fc7\u547d\u4ee4\u884c\u547d\u4ee4\u7684\u65b9\u5f0f\u8fdb\u884c\u64cd\u4f5c\uff0c\u53ef\u4ee5\u663e\u793a\u6b63\u5728\u4fdd\u62a4\u7684\u76ee\u5f55\u5217\u8868\uff0c\u6216\u8005\u4fee\u6539\u4fdd\u62a4\u7684\u76ee\u5f55\u3002\u5c06\u6765\u8fd8\u4f1a\u901a\u8fc7\u4ea7\u54c1\u754c\u9762\u7684\u65b9\u5f0f\uff0c\u63d0\u4f9b\u66f4\u52a0\u4fbf\u6377\u6613\u7528\u7684\u67e5\u770b\u548c\u8bbe\u7f6e\u65b9\u6cd5\u3002","title":"\u80cc\u666f"},{"location":"jindofs/jindofs_protected_directories/#_2","text":"JindoFS \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\uff0c\u4e3b\u8981\u9632\u6b62\u7528\u6237\u8bef\u5220\u76ee\u5f55\uff0c\u88ab\u4fdd\u62a4\u7684\u76ee\u5f55\u53ea\u8981\u8be5\u76ee\u5f55\u4e0b\u5b58\u5728\u76ee\u5f55\u6216\u8005\u6587\u4ef6\uff0c\u4fdd\u62a4\u76ee\u5f55\u7684\u5220\u9664\u4ee5\u53ca\u91cd\u547d\u540d\u64cd\u4f5c\u4f1a\u62a5\u9519\uff0c\u53ea\u6709\u5f53\u88ab\u4fdd\u62a4\u7684\u76ee\u5f55\u4e0b\u4e0d\u5b58\u5728\u4efb\u4f55\u76ee\u5f55\u6216\u8005\u6587\u4ef6\uff0c\u4fdd\u62a4\u76ee\u5f55\u624d\u4f1a\u88ab\u5141\u8bb8\u5220\u9664\u6216\u8005\u91cd\u547d\u540d\u3002","title":"\u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u63cf\u8ff0"},{"location":"jindofs/jindofs_protected_directories/#jindofs","text":"\u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u9700\u8981\u7528\u5230\u7684\u662f -putConfig \u4e0e -getConfig \u547d\u4ee4\uff0c\u5177\u4f53\u7528\u6cd5\u5c06\u5728\u4e0b\u6587\u4ecb\u7ecd\u3002","title":"JindoFS \u547d\u4ee4\u884c\u7b80\u4ecb"},{"location":"jindofs/jindofs_protected_directories/#ak","text":"\u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u9700\u8981\u914d\u7f6e\u8fde\u63a5 OSS-HDFS \u6240\u9700\u7684 AK \u4fe1\u606f\u3002\u6709\u4e24\u79cd\u65b9\u6cd5\u53ef\u4ee5\u8fdb\u884c\u914d\u7f6e\uff1a","title":"\u914d\u7f6e\u8bbf\u95ee AK"},{"location":"jindofs/jindofs_protected_directories/#_3","text":"\u8bbe\u7f6e\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6307\u5411\u67d0\u4e2a\u76ee\u5f55\uff0c\u7136\u540e\u5728\u76ee\u5f55\u4e0b\u521b\u5efa\u540d\u4e3a jindofs.cfg \u7684\u6587\u4ef6\uff0c\u7136\u540e\u5728\u6587\u4ef6\u91cc\u5199\u5165\u5982\u4e0b\u5185\u5bb9\uff1a [client] fs.oss.accessKeyId = <key> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK key fs.oss.accessKeySecret = <secret> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK secret fs.oss.bucket.<bucket>.accessKeyId = <key> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK key\uff0c\u4f18\u5148\u7ea7\u9ad8\u4e8e\u9ed8\u8ba4 fs.oss.bucket.<bucket>.accessKeySecret = <secret> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK secret","title":"\u914d\u7f6e\u6587\u4ef6\uff08\u63a8\u8350\uff09"},{"location":"jindofs/jindofs_protected_directories/#-extraconf","text":"\u547d\u4ee4\u884c\u5de5\u5177\u7684\u6240\u6709\u6307\u4ee4\u5747\u53ef\u901a\u8fc7 --extraConf \u9009\u9879\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u914d\u7f6e\u6587\u4ef6\uff0c\u4f8b\u5982\uff1a jindofs admin -getConfig -dlsUri <path> -name <keys> \\ --extraConf fs.oss.accessKeyId=<AK key> \\ --extraConf fs.oss.accessKeySecret=<AK secret> \u5c06\u7ed9\u547d\u4ee4\u63d0\u4f9b\u989d\u5916\u7684 AK key \u4e0e AK secret \u914d\u7f6e\u3002","title":"--extraConf"},{"location":"jindofs/jindofs_protected_directories/#_4","text":"\u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf fs.protected.directories=/path/to/dir1,/path/to/dir2 \u4e0a\u8ff0\u547d\u4ee4\u5bf9\u540d\u4e3a \uff08endpoint \u4e3a \uff09\u7684\u6876\u914d\u7f6e\u4e86\u4e24\u4e2a\u4fdd\u62a4\u76ee\u5f55\uff1a /path/to/dir1 /path/to/dir2 \u5982\u793a\u4f8b\u4e2d\u7684\u683c\u5f0f\uff0c\u8fd9\u4e9b\u4fdd\u62a4\u76ee\u5f55\u7684\u8def\u5f84\u9700\u8981\u5199\u5728\u4e00\u8d77\uff0c\u9017\u53f7\u9694\u5f00\u3002\u8def\u5f84\u4e0d\u4ee5 oss:// \u5f00\u5934\uff0c\u800c\u662f\u4f7f\u7528\u6876\u5185\u4ee5 / \u5f00\u5934\u7684\u7edd\u5bf9\u8def\u5f84\u3002\u5982\u679c\u547d\u4ee4\u5931\u8d25\uff0c\u5c06\u663e\u793a\u5931\u8d25\u4fe1\u606f\u3002\u5982\u679c\u6ca1\u6709\u4efb\u4f55\u4fe1\u606f\u663e\u793a\uff0c\u8868\u660e\u6267\u884c\u6210\u529f\u3002\u547d\u4ee4\u6267\u884c\u6210\u529f\u4e4b\u540e\uff0c\u5bf9\u8fd9\u4e9b\u76ee\u5f55\u7684\u4fdd\u62a4\u529f\u80fd\u5c06\u5728 30 \u79d2\u5185\u6b63\u5f0f\u751f\u6548\u3002","title":"\u8bbe\u7f6e\u4fdd\u62a4\u76ee\u5f55"},{"location":"jindofs/jindofs_protected_directories/#_5","text":"\u547d\u4ee4\uff1a jindofs admin -getConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -name fs.protected.directories \u4e0a\u8ff0\u547d\u4ee4\u8fd4\u56de\u4e86\u540d\u4e3a \uff08endpoint \u4e3a \uff09\u7684\u6876\u7684\u4fdd\u62a4\u76ee\u5f55\u5217\u8868\uff0c\u8fd4\u56de\u503c\u793a\u4f8b\uff1a fs.protected.directories: /path/to/dir1,/path/to/dir2 \u8868\u660e\u8fd9\u4e9b\u76ee\u5f55\u76ee\u524d\u6b63\u5728\u4fdd\u62a4\u5f53\u4e2d\uff1a /path/to/dir1 /path/to/dir2","title":"\u67e5\u770b\u4fdd\u62a4\u76ee\u5f55"},{"location":"jindofs/jindofs_protected_directories/#_6","text":"\u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf fs.protected.directories= \u4e0a\u8ff0\u547d\u4ee4\u5220\u9664\u4e86\u540d\u4e3a \\<bucket>\uff08endpoint \u4e3a \\<oss-hdfs-endpoint>\uff09\u7684\u6876\u7684\u6240\u6709\u4fdd\u62a4\u76ee\u5f55\uff0c\u6b63\u5f0f\u751f\u6548\u540e\uff08\u547d\u4ee4\u8fd4\u56de\u540e 30 \u79d2\u5185\uff09\uff0c\u8be5\u6876\u5c06\u4e0d\u518d\u6709\u4efb\u4f55\u76ee\u5f55\u53d7\u5230\u4fdd\u62a4\u3002","title":"\u5220\u9664\u6240\u6709\u4fdd\u62a4\u76ee\u5f55"},{"location":"jindofs/jindofs_storagepolicy_restore/","text":"\u4f7f\u7528 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u5b9e\u73b0\u51b7\u70ed\u5206\u5c42\u5b58\u50a8 \u00a7 \u6982\u8ff0 \u00a7 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u662f\u8bbf\u95ee OSS-HDFS \u7684\u53ef\u6267\u884c\u7a0b\u5e8f\uff0c\u652f\u6301\u5e38\u89c1\u6587\u4ef6\u5143\u6570\u636e\u53ca\u8bfb\u5199\u64cd\u4f5c\uff0c\u4ee5\u53ca\u670d\u52a1\u7279\u6709\u65b9\u6cd5\u3002\u66f4\u591a\u4ecb\u7ecd\uff0c\u4ee5\u53ca\u5de5\u5177\u7684\u83b7\u53d6\u548c\u914d\u7f6e\u65b9\u6cd5\uff0c\u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 OSS-HDFS \u670d\u52a1\u7684\u4ecb\u7ecd\uff0c\u53c2\u89c1 OSS-HDFS\u670d\u52a1\u6982\u8ff0 \u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u4f7f\u7528 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u5b9e\u73b0 OSS-HDFS \u670d\u52a1\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8\u529f\u80fd\uff0c\u4ee5\u52a9\u4e8e\u66f4\u597d\u5730\u6cbb\u7406\u6570\u636e\uff0c\u7ba1\u7406\u6210\u672c\u3002 \u5b58\u50a8\u7c7b\u578b \u00a7 OSS-HDFS \u670d\u52a1\u76ee\u524d\u652f\u6301\u516d\u79cd\u5b58\u50a8\u7c7b\u578b\uff0c\u4f9d\u6b21\u4e3a\uff1a \u5b58\u50a8\u7c7b\u578b \u63cf\u8ff0 CLOUD_STD \u6807\u51c6\u5b58\u50a8\uff0c\u9ed8\u8ba4\u7684\u5b58\u50a8\u7c7b\u578b\uff0c\u70ed\u6570\u636e\uff0c\u5b58\u50a8\u8d39\u7528\u6700\u9ad8 CLOUD_IA \u4f4e\u9891\u5b58\u50a8\uff0c\u504f\u51b7\u7684\u6570\u636e\uff0c\u53ef\u76f4\u63a5\u8bbf\u95ee\uff0c\u5b58\u50a8\u8d39\u7528\u8f83\u4f4e\uff0c\u8bbf\u95ee\u8d39\u7528\u8f83\u9ad8 CLOUD_AR \u5f52\u6863\u5b58\u50a8\uff0c\u51b7\u6570\u636e\uff0c\u53ea\u6709\u4e34\u65f6\u89e3\u51bb\u540e\u624d\u80fd\u8bbf\u95ee\uff0c\u5b58\u50a8\u8d39\u7528\u4f4e CLOUD_COLD_AR \u51b7\u5f52\u6863\u5b58\u50a8\uff0c\u6bd4\u5f52\u6863\u66f4\u51b7\u7684\u7c7b\u578b\uff0c\u53ea\u6709\u4e34\u65f6\u89e3\u51bb\u540e\u624d\u80fd\u8bbf\u95ee\uff0c\u5b58\u50a8\u8d39\u7528\u5f88\u4f4e CLOUD_AR_RESTORED \u5f52\u6863\u5b58\u50a8\u4e34\u65f6\u89e3\u51bb\u540e\u7684\u7c7b\u578b\uff0c\u6709\u8fc7\u671f\u65f6\u95f4 CLOUD_COLD_AR_RESTORED \u51b7\u5f52\u6863\u5b58\u50a8\u4e34\u65f6\u89e3\u51bb\u540e\u7684\u7c7b\u578b\uff0c\u6709\u8fc7\u671f\u65f6\u95f4 \u6570\u636e\u5f52\u6863 \u00a7 \u5bf9\u4e8e\u539f\u672c\u5b58\u50a8\u7c7b\u578b\u4e3a \u201c\u6807\u51c6\u201d \u6216 \u201c\u4f4e\u9891\u201d \u7684\u6570\u636e\uff0c\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0b\u547d\u4ee4\u89e6\u53d1\u5bf9\u6570\u636e\u7684\u5f52\u6863\u6307\u4ee4\uff1a jindofs fs -setStoragePolicy -path <path> -policy <policy> \u5176\u4e2d\uff1a * <path> \u53ef\u4ee5\u662f\u6587\u4ef6\u6216\u76ee\u5f55\uff0c\u4e0e\u5f00\u6e90 HDFS \u7684 setStoragePolicy \u529f\u80fd\u4e00\u81f4\u3002 * <policy> \u4e3a\u76ee\u6807\u5b58\u50a8\u7c7b\u578b\uff0c CLOUD_AR \u4e3a \u201c\u5f52\u6863\u201d\uff0c CLOUD_COLD_AR \u4e3a \u201c\u51b7\u5f52\u6863\u201d\u3002 \u5982\u679c\u547d\u4ee4\u6b63\u786e\u6267\u884c\uff0c\u5c06\u8fd4\u56de\u5982\u4e0b\u4fe1\u606f\uff1a Successfully Set StoragePolicy for <path> with policy: <policy> \u68c0\u67e5\u5f52\u6863\u4efb\u52a1 \u00a7 \u4e0a\u4e00\u5219\u547d\u4ee4\u89e6\u53d1\u4e86\u6570\u636e\u5f52\u6863\u7684\u540e\u53f0\u4efb\u52a1\u3002\u53ef\u4ee5\u901a\u8fc7\u4e0b\u5217\u6307\u4ee4\u68c0\u67e5\u8fd9\u4e2a\u4efb\u52a1\u662f\u5426\u5b8c\u6210\uff1a jindofs fs -checkStoragePolicy -path <path> \u8fd4\u56de\u7684\u5185\u5bb9\u793a\u4f8b\u4e3a\uff1a The status storage policy set/unset job for <path> is FINALIZED \u5176\u4e2d FINALIZED \u5373\u4e3a \u201c\u5df2\u5b8c\u6210\u201d\u3002\u5176\u4ed6\u53ef\u80fd\u7684\u72b6\u6001\u8fd8\u6709 PENDING \u3001 PROCESSING \u3001 SUBMITTED \uff0c\u5747\u4e3a\u672a\u5b8c\u6210\u72b6\u6001\uff0c\u9700\u7ee7\u7eed\u7b49\u5f85\u3002 \u6570\u636e\u89e3\u51bb \u00a7 \u5bf9\u4e8e\u5f52\u6863\u7684\u6570\u636e\uff0c\u5982\u679c\u9700\u8981\u8bbf\u95ee\uff0c\u5fc5\u987b\u5148\u8fdb\u884c\u89e3\u51bb\u3002\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0b\u547d\u4ee4\u89e6\u53d1\u5bf9\u6570\u636e\u7684\u4e34\u65f6\u89e3\u51bb\uff1a jindofs fs -setStoragePolicy -path <path> -policy <policy> -restoreDays <restoreDays> \u5176\u4e2d\uff1a * \u5982\u679c\u6570\u636e\u539f\u672c\u4e3a\u5f52\u6863\uff08 CLOUD_AR \uff09\uff0c\u90a3\u4e48 <policy> \u5fc5\u987b\u4e3a CLOUD_AR_RESTORED \uff1b\u5982\u679c\u539f\u672c\u4e3a\u51b7\u5f52\u6863\uff08 CLOUD_COLD_AR \uff09\uff0c\u5219 <policy> \u5fc5\u987b\u4e3a CLOUD_COLD_AR_RESTORED \u3002 * <restoreDays> \u6307\u5b9a\u89e3\u51bb\u7684\u5929\u6570\u3002\u5f52\u6863\u652f\u6301 1 - 7 \u5929\uff0c\u51b7\u5f52\u6863\u652f\u6301 1 - 365 \u5929\u3002\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4 1 \u5929\u3002 * \u53ef\u4ee5\u7528 jindofs fs -checkStoragePolicy -path <path> \u67e5\u770b\u4e34\u65f6\u89e3\u51bb\u4efb\u52a1\u662f\u5426\u5b8c\u6210\u3002 * \u6570\u636e\u4ece\u5f52\u6863\uff08 CLOUD_AR \uff09\u6216\u8005\u51b7\u5f52\u6863( CLOUD_COLD_AR ) \u6062\u590d\u6210\u4f4e\u9891\uff08 CLOUD_IA \uff09\u6216\u8005\u6807\u51c6 ( CLOUD_STD ) \u5b58\u5728\u9650\u5236\uff0c\u5355\u6b21\u63d0\u4ea4\u6570\u636e\u91cf\u4e0d\u80fd\u8d85\u8fc75TB\uff0c \u540c\u65f6\u5904\u4e8e\u6267\u884c\u72b6\u6001\u7684\u6570\u636e\u91cf\u4e0d\u80fd\u8d85\u8fc750TB\u3002 \u4e34\u65f6\u89e3\u51bb\u6709\u4e0b\u5217\u6ce8\u610f\u4e8b\u9879\uff1a * \u7528 CLOUD_AR \u6216\u8005 CLOUD_COLD_AR \u7684\u7b56\u7565\u8fdb\u884c\u5f52\u6863\u4e4b\u540e\uff0c\u5fc5\u987b\u95f4\u9694\u8d85\u8fc7 2 \u5929\u624d\u53ef\u8fdb\u884c\u89e3\u51bb\u3002 * \u4e34\u65f6\u89e3\u51bb\u4efb\u52a1\u5b8c\u6210\u540e\uff0c\u6570\u636e\u4ecd\u4e0d\u80fd\u7acb\u523b\u53ef\u8bfb\u3002\u901a\u5e38\u5f52\u6863\u7c7b\u578b\u9700\u8981\u6570\u5206\u949f\u540e\u53ef\u8bfb\uff0c\u51b7\u5f52\u6863\u7c7b\u578b\u5219\u9700\u8981\u6570\u5c0f\u65f6\u3002 * \u4e34\u65f6\u89e3\u51bb\u6709\u5929\u6570\u9650\u5236\uff0c\u7531 <restoreDays> \u6307\u5b9a\uff0c\u8d85\u51fa\u5219\u56de\u5230\u4e0d\u53ef\u8bfb\u72b6\u6001\u3002 * \u5904\u4e8e\u4e34\u65f6\u89e3\u51bb\u72b6\u6001\u65f6\uff0c\u4ecd\u53ef\u4ee5\u518d\u6b21\u8fdb\u884c\u89e3\u51bb\uff0c\u4f46\u662f\u95f4\u9694\u5fc5\u987b\u8d85\u8fc7 2 \u5929\u4ee5\u4e0a\u3002","title":"\u4f7f\u7528 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u5b9e\u73b0\u51b7\u70ed\u5206\u5c42\u5b58\u50a8"},{"location":"jindofs/jindofs_storagepolicy_restore/#jindofs","text":"","title":"\u4f7f\u7528 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u5b9e\u73b0\u51b7\u70ed\u5206\u5c42\u5b58\u50a8"},{"location":"jindofs/jindofs_storagepolicy_restore/#_1","text":"JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u662f\u8bbf\u95ee OSS-HDFS \u7684\u53ef\u6267\u884c\u7a0b\u5e8f\uff0c\u652f\u6301\u5e38\u89c1\u6587\u4ef6\u5143\u6570\u636e\u53ca\u8bfb\u5199\u64cd\u4f5c\uff0c\u4ee5\u53ca\u670d\u52a1\u7279\u6709\u65b9\u6cd5\u3002\u66f4\u591a\u4ecb\u7ecd\uff0c\u4ee5\u53ca\u5de5\u5177\u7684\u83b7\u53d6\u548c\u914d\u7f6e\u65b9\u6cd5\uff0c\u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 OSS-HDFS \u670d\u52a1\u7684\u4ecb\u7ecd\uff0c\u53c2\u89c1 OSS-HDFS\u670d\u52a1\u6982\u8ff0 \u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u4f7f\u7528 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u5b9e\u73b0 OSS-HDFS \u670d\u52a1\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8\u529f\u80fd\uff0c\u4ee5\u52a9\u4e8e\u66f4\u597d\u5730\u6cbb\u7406\u6570\u636e\uff0c\u7ba1\u7406\u6210\u672c\u3002","title":"\u6982\u8ff0"},{"location":"jindofs/jindofs_storagepolicy_restore/#_2","text":"OSS-HDFS \u670d\u52a1\u76ee\u524d\u652f\u6301\u516d\u79cd\u5b58\u50a8\u7c7b\u578b\uff0c\u4f9d\u6b21\u4e3a\uff1a \u5b58\u50a8\u7c7b\u578b \u63cf\u8ff0 CLOUD_STD \u6807\u51c6\u5b58\u50a8\uff0c\u9ed8\u8ba4\u7684\u5b58\u50a8\u7c7b\u578b\uff0c\u70ed\u6570\u636e\uff0c\u5b58\u50a8\u8d39\u7528\u6700\u9ad8 CLOUD_IA \u4f4e\u9891\u5b58\u50a8\uff0c\u504f\u51b7\u7684\u6570\u636e\uff0c\u53ef\u76f4\u63a5\u8bbf\u95ee\uff0c\u5b58\u50a8\u8d39\u7528\u8f83\u4f4e\uff0c\u8bbf\u95ee\u8d39\u7528\u8f83\u9ad8 CLOUD_AR \u5f52\u6863\u5b58\u50a8\uff0c\u51b7\u6570\u636e\uff0c\u53ea\u6709\u4e34\u65f6\u89e3\u51bb\u540e\u624d\u80fd\u8bbf\u95ee\uff0c\u5b58\u50a8\u8d39\u7528\u4f4e CLOUD_COLD_AR \u51b7\u5f52\u6863\u5b58\u50a8\uff0c\u6bd4\u5f52\u6863\u66f4\u51b7\u7684\u7c7b\u578b\uff0c\u53ea\u6709\u4e34\u65f6\u89e3\u51bb\u540e\u624d\u80fd\u8bbf\u95ee\uff0c\u5b58\u50a8\u8d39\u7528\u5f88\u4f4e CLOUD_AR_RESTORED \u5f52\u6863\u5b58\u50a8\u4e34\u65f6\u89e3\u51bb\u540e\u7684\u7c7b\u578b\uff0c\u6709\u8fc7\u671f\u65f6\u95f4 CLOUD_COLD_AR_RESTORED \u51b7\u5f52\u6863\u5b58\u50a8\u4e34\u65f6\u89e3\u51bb\u540e\u7684\u7c7b\u578b\uff0c\u6709\u8fc7\u671f\u65f6\u95f4","title":"\u5b58\u50a8\u7c7b\u578b"},{"location":"jindofs/jindofs_storagepolicy_restore/#_3","text":"\u5bf9\u4e8e\u539f\u672c\u5b58\u50a8\u7c7b\u578b\u4e3a \u201c\u6807\u51c6\u201d \u6216 \u201c\u4f4e\u9891\u201d \u7684\u6570\u636e\uff0c\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0b\u547d\u4ee4\u89e6\u53d1\u5bf9\u6570\u636e\u7684\u5f52\u6863\u6307\u4ee4\uff1a jindofs fs -setStoragePolicy -path <path> -policy <policy> \u5176\u4e2d\uff1a * <path> \u53ef\u4ee5\u662f\u6587\u4ef6\u6216\u76ee\u5f55\uff0c\u4e0e\u5f00\u6e90 HDFS \u7684 setStoragePolicy \u529f\u80fd\u4e00\u81f4\u3002 * <policy> \u4e3a\u76ee\u6807\u5b58\u50a8\u7c7b\u578b\uff0c CLOUD_AR \u4e3a \u201c\u5f52\u6863\u201d\uff0c CLOUD_COLD_AR \u4e3a \u201c\u51b7\u5f52\u6863\u201d\u3002 \u5982\u679c\u547d\u4ee4\u6b63\u786e\u6267\u884c\uff0c\u5c06\u8fd4\u56de\u5982\u4e0b\u4fe1\u606f\uff1a Successfully Set StoragePolicy for <path> with policy: <policy>","title":"\u6570\u636e\u5f52\u6863"},{"location":"jindofs/jindofs_storagepolicy_restore/#_4","text":"\u4e0a\u4e00\u5219\u547d\u4ee4\u89e6\u53d1\u4e86\u6570\u636e\u5f52\u6863\u7684\u540e\u53f0\u4efb\u52a1\u3002\u53ef\u4ee5\u901a\u8fc7\u4e0b\u5217\u6307\u4ee4\u68c0\u67e5\u8fd9\u4e2a\u4efb\u52a1\u662f\u5426\u5b8c\u6210\uff1a jindofs fs -checkStoragePolicy -path <path> \u8fd4\u56de\u7684\u5185\u5bb9\u793a\u4f8b\u4e3a\uff1a The status storage policy set/unset job for <path> is FINALIZED \u5176\u4e2d FINALIZED \u5373\u4e3a \u201c\u5df2\u5b8c\u6210\u201d\u3002\u5176\u4ed6\u53ef\u80fd\u7684\u72b6\u6001\u8fd8\u6709 PENDING \u3001 PROCESSING \u3001 SUBMITTED \uff0c\u5747\u4e3a\u672a\u5b8c\u6210\u72b6\u6001\uff0c\u9700\u7ee7\u7eed\u7b49\u5f85\u3002","title":"\u68c0\u67e5\u5f52\u6863\u4efb\u52a1"},{"location":"jindofs/jindofs_storagepolicy_restore/#_5","text":"\u5bf9\u4e8e\u5f52\u6863\u7684\u6570\u636e\uff0c\u5982\u679c\u9700\u8981\u8bbf\u95ee\uff0c\u5fc5\u987b\u5148\u8fdb\u884c\u89e3\u51bb\u3002\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0b\u547d\u4ee4\u89e6\u53d1\u5bf9\u6570\u636e\u7684\u4e34\u65f6\u89e3\u51bb\uff1a jindofs fs -setStoragePolicy -path <path> -policy <policy> -restoreDays <restoreDays> \u5176\u4e2d\uff1a * \u5982\u679c\u6570\u636e\u539f\u672c\u4e3a\u5f52\u6863\uff08 CLOUD_AR \uff09\uff0c\u90a3\u4e48 <policy> \u5fc5\u987b\u4e3a CLOUD_AR_RESTORED \uff1b\u5982\u679c\u539f\u672c\u4e3a\u51b7\u5f52\u6863\uff08 CLOUD_COLD_AR \uff09\uff0c\u5219 <policy> \u5fc5\u987b\u4e3a CLOUD_COLD_AR_RESTORED \u3002 * <restoreDays> \u6307\u5b9a\u89e3\u51bb\u7684\u5929\u6570\u3002\u5f52\u6863\u652f\u6301 1 - 7 \u5929\uff0c\u51b7\u5f52\u6863\u652f\u6301 1 - 365 \u5929\u3002\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4 1 \u5929\u3002 * \u53ef\u4ee5\u7528 jindofs fs -checkStoragePolicy -path <path> \u67e5\u770b\u4e34\u65f6\u89e3\u51bb\u4efb\u52a1\u662f\u5426\u5b8c\u6210\u3002 * \u6570\u636e\u4ece\u5f52\u6863\uff08 CLOUD_AR \uff09\u6216\u8005\u51b7\u5f52\u6863( CLOUD_COLD_AR ) \u6062\u590d\u6210\u4f4e\u9891\uff08 CLOUD_IA \uff09\u6216\u8005\u6807\u51c6 ( CLOUD_STD ) \u5b58\u5728\u9650\u5236\uff0c\u5355\u6b21\u63d0\u4ea4\u6570\u636e\u91cf\u4e0d\u80fd\u8d85\u8fc75TB\uff0c \u540c\u65f6\u5904\u4e8e\u6267\u884c\u72b6\u6001\u7684\u6570\u636e\u91cf\u4e0d\u80fd\u8d85\u8fc750TB\u3002 \u4e34\u65f6\u89e3\u51bb\u6709\u4e0b\u5217\u6ce8\u610f\u4e8b\u9879\uff1a * \u7528 CLOUD_AR \u6216\u8005 CLOUD_COLD_AR \u7684\u7b56\u7565\u8fdb\u884c\u5f52\u6863\u4e4b\u540e\uff0c\u5fc5\u987b\u95f4\u9694\u8d85\u8fc7 2 \u5929\u624d\u53ef\u8fdb\u884c\u89e3\u51bb\u3002 * \u4e34\u65f6\u89e3\u51bb\u4efb\u52a1\u5b8c\u6210\u540e\uff0c\u6570\u636e\u4ecd\u4e0d\u80fd\u7acb\u523b\u53ef\u8bfb\u3002\u901a\u5e38\u5f52\u6863\u7c7b\u578b\u9700\u8981\u6570\u5206\u949f\u540e\u53ef\u8bfb\uff0c\u51b7\u5f52\u6863\u7c7b\u578b\u5219\u9700\u8981\u6570\u5c0f\u65f6\u3002 * \u4e34\u65f6\u89e3\u51bb\u6709\u5929\u6570\u9650\u5236\uff0c\u7531 <restoreDays> \u6307\u5b9a\uff0c\u8d85\u51fa\u5219\u56de\u5230\u4e0d\u53ef\u8bfb\u72b6\u6001\u3002 * \u5904\u4e8e\u4e34\u65f6\u89e3\u51bb\u72b6\u6001\u65f6\uff0c\u4ecd\u53ef\u4ee5\u518d\u6b21\u8fdb\u884c\u89e3\u51bb\uff0c\u4f46\u662f\u95f4\u9694\u5fc5\u987b\u8d85\u8fc7 2 \u5929\u4ee5\u4e0a\u3002","title":"\u6570\u636e\u89e3\u51bb"},{"location":"jindofs/jindofs_trash_quickstart/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u56de\u6536\u7ad9\u4f7f\u7528\u8bf4\u660e \u00a7 (\u4ece JindoData 4.5.1 \u5f00\u59cb\u652f\u6301) \u4ecb\u7ecd \u00a7 \u5f53\u4ece OSS-HDFS \u670d\u52a1\u5220\u9664\u6587\u4ef6\u65f6\uff0c\u6587\u4ef6\u4e0d\u662f\u7acb\u5373\u88ab\u7acb\u5373\u5220\u9664\u3002\u88ab\u5220\u9664\u7684\u6587\u4ef6\u88ab\u632a\u5230\u4e86 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\u3002\u5f53\u7ecf\u8fc730\u5206\u949f\u540e\uff0cCurrent\u76ee\u5f55\u4f1a\u88ab\u632a\u5230 ```bash /user/ /.Trash/ \u4e5f\u5c31\u662f\u8bf4\u67d0\u4e00\u65f6\u95f4\u6bb5\u5185\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u4f1a\u88ab\u5f52\u7c7b\u5230\u4e00\u4e2a\u5e26\u65f6\u95f4\u6233\u7684\u76ee\u5f55\u4e0b\uff0c\u8868\u793a\u662f\u5728\u8fd9\u4e2a\u65f6\u95f4\u5185\u88ab\u5220\u9664\u7684\uff0c\u76f8\u5f53\u4e8e\u4e00\u6b21checkpoint\u3002\u5f53\u7ecf\u8fc7**7\u5929**\u540e\uff0c\u8fd9\u4e2a<timestamp>\u76ee\u5f55\u5c06\u4f1a\u88ab\u6c38\u8fdc\u5220\u9664\u3002\u56e0\u6b64\u57287\u5929\u5185\uff0c\u60a8\u6709\u673a\u4f1a\u4ece.Trash\u76ee\u5f55\u4e0b\u627e\u5230\u5bf9\u5e94\u65f6\u523b\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u5c06\u5176\u4ece.Trash\u632a\u51fa\uff0c\u4ece\u800c\u6062\u590d\u5b83\u3002 \u6ce8\u610f\uff1a\u8fd9\u9879\u529f\u80fd\uff0c\u662f\u7531\u5ba2\u6237\u7aef\u548c\u670d\u52a1\u7aef\u914d\u5408\uff0c\u65b9\u80fd\u5f62\u6210\u56de\u6536\u7ad9\u7684\u529f\u80fd\u3002\u670d\u52a1\u7aef\u53ea\u8d1f\u8d23\u7ef4\u62a4 ```/user/<username>/.Trash``` \u7684\u5b9a\u65f6\u6e05\u7406\uff0c\u670d\u52a1\u7aef\u7684\u5b9a\u65f6\u6e05\u7406\u662f\u9ed8\u8ba4\u5f00\u542f\u7684\u3002\u5ba2\u6237\u7aef\u8d1f\u8d23\u5c06\u5f85\u5220\u9664\u6587\u4ef6\u632a\u5230.Trash\u76ee\u5f55\u4e0b\u3002 ## Hadoop FileSystem Shell \u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd ```bash hadoop fs -rm oss://bucket/a/b/c \u5ba2\u6237\u7aef\u7684 Hadoop Shell \u547d\u4ee4\u9ed8\u8ba4\u4e0d\u5f00\u542fTrash\u529f\u80fd\uff0c\u56e0\u6b64\u9700\u8981\u5728 core-site.xml \u91cc\u6dfb\u52a0\u4e00\u6761\u914d\u7f6e <property> <name>fs.trash.interval</name> <value>1440</value> </property> (\u8be5\u503c\u53ea\u9700\u5927\u4e8e0\u5373\u53ef) \u6b64\u65f6\u5728\u5ba2\u6237\u7aef\u4f1a\u81ea\u52a8\u5c06rm\u547d\u4ee4\u8f6c\u6362\u4e3a\u4e00\u6761 hadoop fs -mv oss://bucket/a/b/c /user/<username>/.Trash/Current/a/b/c \u547d\u4ee4\uff0c\u56e0\u6b64\uff0c\u60a8\u4e0d\u9700\u8981\u611f\u77e5\u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u670d\u52a1\u7aef\u4f1a\u8d1f\u8d23\u6e05\u7406\u3002 \u5982\u679c\u60a8\u60f3\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\uff0c\u91ca\u653e\u7a7a\u95f4\uff0c\u53ef\u4ee5\u6dfb\u52a0 -skipTrash \u53c2\u6570\uff0c\u6b64\u65f6\u5c06\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\u3002 Hadoop \u751f\u6001\u7ec4\u4ef6\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd \u00a7 Hive/Spark/Flink \u7b49\u7ec4\u4ef6\u5e76\u4e0d\u611f\u77e5 OSS-HDFS \u670d\u52a1\u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u4f7f\u7528 FileSystem(HCFS) \u7684 delete \u63a5\u53e3\u610f\u5473\u7740\u7acb\u5373\u5220\u9664\u3002\u56e0\u6b64\u5982\u679c\u60f3\u8981\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd\uff0c\u9700\u8981\u663e\u5f0f\u5730\u8c03\u7528 FileSystem \u7684 rename \u63a5\u53e3\uff0c\u5c06\u76ee\u6807\u6587\u4ef6\u624b\u52a8\u632a\u5230 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\uff0c\u7531 OSS-HDFS \u670d\u52a1\u7aef\u8d1f\u8d23\u5b9a\u671f\u5220\u9664\u3002\u8fd9\u4e00\u70b9\u4e0a OSS-HDFS \u91c7\u53d6\u4e86\u8ddf\u5f00\u6e90 Hadoop \u76f8\u4f3c\u7684\u7b56\u7565\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u56de\u6536\u7ad9\u4f7f\u7528\u8bf4\u660e"},{"location":"jindofs/jindofs_trash_quickstart/#oss-hdfs-jindofs","text":"(\u4ece JindoData 4.5.1 \u5f00\u59cb\u652f\u6301)","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u56de\u6536\u7ad9\u4f7f\u7528\u8bf4\u660e"},{"location":"jindofs/jindofs_trash_quickstart/#_1","text":"\u5f53\u4ece OSS-HDFS \u670d\u52a1\u5220\u9664\u6587\u4ef6\u65f6\uff0c\u6587\u4ef6\u4e0d\u662f\u7acb\u5373\u88ab\u7acb\u5373\u5220\u9664\u3002\u88ab\u5220\u9664\u7684\u6587\u4ef6\u88ab\u632a\u5230\u4e86 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\u3002\u5f53\u7ecf\u8fc730\u5206\u949f\u540e\uff0cCurrent\u76ee\u5f55\u4f1a\u88ab\u632a\u5230 ```bash /user/ /.Trash/ \u4e5f\u5c31\u662f\u8bf4\u67d0\u4e00\u65f6\u95f4\u6bb5\u5185\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u4f1a\u88ab\u5f52\u7c7b\u5230\u4e00\u4e2a\u5e26\u65f6\u95f4\u6233\u7684\u76ee\u5f55\u4e0b\uff0c\u8868\u793a\u662f\u5728\u8fd9\u4e2a\u65f6\u95f4\u5185\u88ab\u5220\u9664\u7684\uff0c\u76f8\u5f53\u4e8e\u4e00\u6b21checkpoint\u3002\u5f53\u7ecf\u8fc7**7\u5929**\u540e\uff0c\u8fd9\u4e2a<timestamp>\u76ee\u5f55\u5c06\u4f1a\u88ab\u6c38\u8fdc\u5220\u9664\u3002\u56e0\u6b64\u57287\u5929\u5185\uff0c\u60a8\u6709\u673a\u4f1a\u4ece.Trash\u76ee\u5f55\u4e0b\u627e\u5230\u5bf9\u5e94\u65f6\u523b\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u5c06\u5176\u4ece.Trash\u632a\u51fa\uff0c\u4ece\u800c\u6062\u590d\u5b83\u3002 \u6ce8\u610f\uff1a\u8fd9\u9879\u529f\u80fd\uff0c\u662f\u7531\u5ba2\u6237\u7aef\u548c\u670d\u52a1\u7aef\u914d\u5408\uff0c\u65b9\u80fd\u5f62\u6210\u56de\u6536\u7ad9\u7684\u529f\u80fd\u3002\u670d\u52a1\u7aef\u53ea\u8d1f\u8d23\u7ef4\u62a4 ```/user/<username>/.Trash``` \u7684\u5b9a\u65f6\u6e05\u7406\uff0c\u670d\u52a1\u7aef\u7684\u5b9a\u65f6\u6e05\u7406\u662f\u9ed8\u8ba4\u5f00\u542f\u7684\u3002\u5ba2\u6237\u7aef\u8d1f\u8d23\u5c06\u5f85\u5220\u9664\u6587\u4ef6\u632a\u5230.Trash\u76ee\u5f55\u4e0b\u3002 ## Hadoop FileSystem Shell \u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd ```bash hadoop fs -rm oss://bucket/a/b/c \u5ba2\u6237\u7aef\u7684 Hadoop Shell \u547d\u4ee4\u9ed8\u8ba4\u4e0d\u5f00\u542fTrash\u529f\u80fd\uff0c\u56e0\u6b64\u9700\u8981\u5728 core-site.xml \u91cc\u6dfb\u52a0\u4e00\u6761\u914d\u7f6e <property> <name>fs.trash.interval</name> <value>1440</value> </property> (\u8be5\u503c\u53ea\u9700\u5927\u4e8e0\u5373\u53ef) \u6b64\u65f6\u5728\u5ba2\u6237\u7aef\u4f1a\u81ea\u52a8\u5c06rm\u547d\u4ee4\u8f6c\u6362\u4e3a\u4e00\u6761 hadoop fs -mv oss://bucket/a/b/c /user/<username>/.Trash/Current/a/b/c \u547d\u4ee4\uff0c\u56e0\u6b64\uff0c\u60a8\u4e0d\u9700\u8981\u611f\u77e5\u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u670d\u52a1\u7aef\u4f1a\u8d1f\u8d23\u6e05\u7406\u3002 \u5982\u679c\u60a8\u60f3\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\uff0c\u91ca\u653e\u7a7a\u95f4\uff0c\u53ef\u4ee5\u6dfb\u52a0 -skipTrash \u53c2\u6570\uff0c\u6b64\u65f6\u5c06\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\u3002","title":"\u4ecb\u7ecd"},{"location":"jindofs/jindofs_trash_quickstart/#hadoop","text":"Hive/Spark/Flink \u7b49\u7ec4\u4ef6\u5e76\u4e0d\u611f\u77e5 OSS-HDFS \u670d\u52a1\u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u4f7f\u7528 FileSystem(HCFS) \u7684 delete \u63a5\u53e3\u610f\u5473\u7740\u7acb\u5373\u5220\u9664\u3002\u56e0\u6b64\u5982\u679c\u60f3\u8981\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd\uff0c\u9700\u8981\u663e\u5f0f\u5730\u8c03\u7528 FileSystem \u7684 rename \u63a5\u53e3\uff0c\u5c06\u76ee\u6807\u6587\u4ef6\u624b\u52a8\u632a\u5230 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\uff0c\u7531 OSS-HDFS \u670d\u52a1\u7aef\u8d1f\u8d23\u5b9a\u671f\u5220\u9664\u3002\u8fd9\u4e00\u70b9\u4e0a OSS-HDFS \u91c7\u53d6\u4e86\u8ddf\u5f00\u6e90 Hadoop \u76f8\u4f3c\u7684\u7b56\u7565\u3002","title":"Hadoop \u751f\u6001\u7ec4\u4ef6\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd"},{"location":"jindofs/jindotable_storagepolicy_restore/","text":"\u4f7f\u7528 JindoTable \u5de5\u5177\u5b9e\u73b0 OSS-HDFS \u7ed3\u6784\u5316\u6570\u636e\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8 \u00a7 \u6982\u8ff0 \u00a7 JindoTable \u662f\u57fa\u4e8e JindoSDK \u7684\u7ed3\u6784\u5316\u6570\u636e\u6cbb\u7406\u5de5\u5177\uff0c\u76ee\u6807\u662f\u5728 \u201c\u8868\u201d \u6216\u8005 \u201c\u5206\u533a\u201d \u5c42\u9762\u63d0\u4f9b\u4fbf\u6377\u7684\u6570\u636e\u8bbf\u95ee\u548c\u64cd\u4f5c\u3002\u4f7f\u7528 JindoTable \u5de5\u5177\u53ef\u4ee5\u65b9\u4fbf\u5730\u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u548c\u5206\u5c42\u5b58\u50a8\u3002 \u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u4f7f\u7528 JindoTable \u5de5\u5177\u5bf9\u5b58\u50a8\u5728 OSS-HDFS \u4e0a\u7684\u7ed3\u6784\u5316\u6570\u636e\u8fdb\u884c\u51b7\u70ed\u5206\u5c42\u5b58\u50a8\u3002 \u83b7\u53d6 \u00a7 \u53ef\u4ee5\u4ece JindoData \u4e0b\u8f7d\u9875\u9762 \u83b7\u53d6 JindoTable \u5de5\u5177\uff1a 1. \u4e0b\u8f7d jindosdk-<version>-<platform>.tar.gz \uff0c\u6839\u636e\u7cfb\u7edf\u9009\u62e9\u5408\u9002\u7684 <platform> \u3002 2. \u672c\u6587\u4ecb\u7ecd\u7684\u529f\u80fd\u6700\u4f4e\u7248\u672c\u8981\u6c42\u4e3a <version> 6.1.2\u3002 \u6ce8\uff1a - JindoTable \u5de5\u5177\u5bf9 OSS-HDFS \u7684\u4e34\u65f6\u89e3\u51bb\uff08 -once \uff09\u652f\u6301\u8bbe\u7f6e\u89e3\u51bb\u53c2\u6570\uff08 -d/-restoreDays <restore days> \uff09\uff0c\u7248\u672c\u8981\u6c42\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0\u3002\u82e5\u5bf9\u8be5\u529f\u80fd\u6709\u9700\u8981\uff0c\u8bf7\u4f7f\u7528\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0 \u7684\u7248\u672c\u3002 \u57fa\u672c\u7528\u6cd5 \u00a7 \u6309\u7167\u4e0a\u9762\u7684\u6b65\u9aa4\u83b7\u53d6\u538b\u7f29\u5305\u540e\uff0c\u89e3\u538b\u7f29\uff0c\u4fbf\u53ef\u5728 bin/ \u76ee\u5f55\u4e0b\u627e\u5230\u540d\u4e3a jindotable \u7684\u811a\u672c\u3002\u6267\u884c\u811a\u672c\uff0c\u5373\u53ef\u4f7f\u7528 JindoTable \u63d0\u4f9b\u7684\u5404\u79cd\u547d\u4ee4\u3002 \u6267\u884c jindotable -usage \u53ef\u4ee5\u663e\u793a\u5de5\u5177\u652f\u6301\u7684\u5404\u79cd\u547d\u4ee4\u3002 \u4e00\u4e9b\u6ce8\u610f\u4e8b\u9879\uff1a - \u6267\u884c\u65f6\uff0c\u987b\u4fdd\u6301\u89e3\u538b\u7f29\u540e\u7684\u76ee\u5f55\u7ed3\u6784\u3002 - \u9700\u8981\u96c6\u7fa4\u4e0a\u6709 Hadoop \u548c Hive \u73af\u5883\uff0c\u76f8\u5173\u7684\u73af\u5883\u53d8\u91cf\u5df2\u7ecf\u88ab\u6b63\u786e\u914d\u7f6e\u3002 \u5206\u5c42\u5b58\u50a8\u547d\u4ee4 \u00a7 \u652f\u6301 OSS-HDFS \u7ed3\u6784\u5316\u6570\u636e\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8\u529f\u80fd\uff0c\u547d\u4ee4\u4e3a -setStoragePolicy \uff0c\u6267\u884c jindotable -help setStoragePolicy \u53ef\u4ee5\u663e\u793a\u5e2e\u52a9\u3002 \u53c2\u6570\u8bf4\u660e \u00a7 jindotable -setStoragePolicy -t <dbName.tableName> -std/-ia/-archive/-coldArchive/-once [-c \"<condition>\" | -fullTable] [-d/-restoreDays <restore days>] [-b/-before <before days>] [-p/-parallel <parallelism>] [-mr/-mapReduce] [-e/-explain] [-w/-workingDir <working directory>] [-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u76ee\u6807\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a\u6570\u636e\u5e93\u540d.\u8868\u540d\u3002\u6570\u636e\u5e93\u548c\u8868\u540d\u4e4b\u95f4\u4ee5\u534a\u89d2\u53e5\u53f7\uff08.\uff09\u5206\u9694\u3002\u8868\u53ef\u4ee5\u662f\u5206\u533a\u8868\u6216\u975e\u5206\u533a\u8868\u3002 \u662f -std/-ia/-archive/-coldArchive/-once \u76ee\u6807\u5b58\u50a8\u65b9\u5f0f\u3002\u652f\u6301\u5982\u4e0b\u7c7b\u578b\uff1a-std\uff1a\u6807\u51c6\u5b58\u50a8\uff0c-ia\uff1a\u4f4e\u9891\u5b58\u50a8\uff0c-archive\uff1a\u5f52\u6863\u5b58\u50a8\uff0c-coldArchive\uff1a\u51b7\u5f52\u6863\u5b58\u50a8\uff0c-once\uff1a\u4e34\u65f6\u89e3\u51bb\u3002 \u662f -c \"<condition>\" / -fullTable -fullTable \u548c -c \"<condition>\" \u53ea\u9700\u63d0\u4f9b\u4e00\u4e2a\uff0c\u5373\u8981\u4e48\u6307\u5b9a -c \"<condition>\" \uff0c\u8981\u4e48\u6307\u5b9a -fullTable \u3002\u6307\u5b9a -fullTable \u65f6\uff0c\u5219\u4e3a\u79fb\u52a8\u6574\u8868\uff0c\u65e2\u53ef\u4ee5\u662f\u975e\u5206\u533a\u8868\u4e5f\u53ef\u4ee5\u662f\u5206\u533a\u8868\u3002\u6307\u5b9a -c \"<condition>\" \u65f6\uff0c\u5219\u63d0\u4f9b\u4e86\u4e00\u4e2a\u8fc7\u6ee4\u6761\u4ef6\uff0c\u7528\u6765\u9009\u62e9\u5e0c\u671b\u79fb\u52a8\u7684\u5206\u533a\uff0c\u652f\u6301\u5e38\u89c1\u8fd0\u7b97\u7b26\uff0c\u4f8b\u5982\u5927\u4e8e\u53f7\uff08>\uff09\u3002\u4f8b\u5982\uff0c\u6570\u636e\u7c7b\u578b\u4e3aString\u7684\u5206\u533ads\uff0c\u5e0c\u671b\u5206\u533a\u540d\u5927\u4e8e 'd'\uff0c\u5219\u4ee3\u7801\u4e3a-c \" ds > 'd' \"\u3002 \u662f -d/-restoreDays <restore days> \u6307\u5b9a\u4e34\u65f6\u89e3\u51bb\u7684\u5929\u6570\uff0c\u4ec5\u5bf9\u4e34\u65f6\u89e3\u51bb -once \u751f\u6548\u3002 \u5426 -b/before <before days> \u53ea\u6709\u521b\u5efa\u65f6\u95f4\u8ddd\u79bb\u73b0\u5728\u8d85\u8fc7\u4e00\u5b9a\u5929\u6570\u7684\u8868\u6216\u5206\u533a\u624d\u4f1a\u88ab\u6267\u884c\u3002 \u5426 -p/-parallel <parallelism> \u547d\u4ee4\u6267\u884c\u7684\u5e76\u884c\u5ea6\u3002 \u5426 -mr/-mapReduce \u4f7f\u7528Hadoop MapReduce\u800c\u975e\u672c\u5730\u591a\u7ebf\u7a0b\u6765\u6267\u884c\u547d\u4ee4\u3002 \u5426 -e/-explain \u5982\u679c\u51fa\u73b0\u8be5\u9009\u9879\uff0c\u5219\u4e3a\u89e3\u91ca\uff08explain\uff09\u6a21\u5f0f\uff0c\u53ea\u4f1a\u663e\u793a\u5f85\u64cd\u4f5c\u7684\u5206\u533a\u5217\u8868\uff0c\u800c\u4e0d\u4f1a\u771f\u6b63\u6267\u884c\u547d\u4ee4\u3002 \u5426 -w/-workingDir <working directory> \u53ea\u5728MapReduce\u4f5c\u4e1a\u65f6\u4f7f\u7528\uff0c\u4e3aMapReduce\u4f5c\u4e1a\u7684\u5de5\u4f5c\u76ee\u5f55\u3002\u5fc5\u987b\u6709\u8bfb\u5199\u6743\u9650\uff0c\u5de5\u4f5c\u76ee\u5f55\u53ef\u4ee5\u975e\u7a7a\uff08\u4f5c\u4e1a\u6267\u884c\u8fc7\u7a0b\u4e2d\u4f1a\u521b\u5efa\u4e34\u65f6\u6587\u4ef6\uff0c\u6267\u884c\u5b8c\u6bd5\u4f1a\u6e05\u7406\u4e34\u65f6\u6587\u4ef6\uff09\u3002 \u5426 -l/-logDir <log directory> \u6307\u5b9aLog\u6587\u4ef6\u7684\u76ee\u5f55\u3002 \u5426 \u6ce8\u610f\u4e8b\u9879 \u00a7 JindoTable \u63d0\u4f9b\u7684\u4e0a\u8ff0\u529f\u80fd\u57fa\u672c\u5bf9\u5e94\u4e8e OSS-HDFS \u7ebf\u4e0a\u670d\u52a1\u7684\u5206\u5c42\u5b58\u50a8\u80fd\u529b\uff0c\u6709\u4e0b\u5217\u4e8b\u9879\u9700\u8981\u6ce8\u610f\uff1a 1. \u4ec5\u652f\u6301 OSS-HDFS \u5b58\u50a8\u7684\u8868\u6216\u8005\u5206\u533a\u3002\u5bf9\u666e\u901a OSS \u6570\u636e\u7684\u5206\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoTable \u63d0\u4f9b\u7684 archiveTable \u4e0e unarchiveTable \u547d\u4ee4\u3002 2. \u6210\u529f\u6267\u884c -std/-ia/-archive/-coldArchive/-once \u6307\u5b9a\u7684\u8f6c\u6362\u5b58\u50a8\u7c7b\u578b\u547d\u4ee4\u540e\uff0c\u5fc5\u987b\u7b49\u5f85 2 \u5929\u624d\u53ef\u518d\u6b21\u8f6c\u6362\u5b58\u50a8\u7c7b\u578b\u3002 3. \u6807\u51c6\u548c\u4f4e\u9891\u7c7b\u578b\u65e0\u6cd5\u4e34\u65f6\u89e3\u51bb\uff08 -once \uff09\u3002\u5f52\u6863\u3001\u51b7\u5f52\u6863\uff0c\u6216\u8005\u5df2\u7ecf\u5904\u4e8e\u89e3\u51bb\u72b6\u6001\u65f6\uff0c\u53ef\u4ee5\u89e3\u51bb\uff0c\u4f46\u5982\u4e0a\u6240\u8ff0\uff0c\u95f4\u9694\u987b\u8d85\u8fc7 2 \u5929\u3002 4. \u4e34\u65f6\u89e3\u51bb\u547d\u4ee4\u8fd4\u56de\u540e\uff0c\u6570\u636e\u4ecd\u4e0d\u80fd\u7acb\u5373\u53ef\u8bfb\uff0c\u9700\u8981\u7b49\u5f85\u8fdc\u7aef\u670d\u52a1\u5b8c\u6210\u6570\u636e\u7684\u89e3\u51bb\u3002\u5bf9\u4e8e\u5f52\u6863\u6570\u636e\uff0c\u901a\u5e38\u9700\u8981\u6570\u5206\u949f\uff1b\u5bf9\u4e8e\u51b7\u5f52\u6863\u6570\u636e\uff0c\u901a\u5e38\u9700\u8981\u6570\u5c0f\u65f6\u3002 5. \u4e34\u65f6\u89e3\u51bb\u6709\u671f\u9650\u9650\u5236\uff0c\u901a\u8fc7 -d/restoreDays <restore days> \u6307\u5b9a\uff0c\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4 1 \u5929\uff0c\u5230\u671f\u540e\u6570\u636e\u5c06\u56de\u5230\u4e0d\u53ef\u8bfb\u72b6\u6001\u3002\u8be5\u529f\u80fd\u8981\u6c42\u7248\u672c\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0\u3002","title":"\u4f7f\u7528 JindoTable \u5de5\u5177\u5b9e\u73b0 OSS-HDFS \u7ed3\u6784\u5316\u6570\u636e\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8"},{"location":"jindofs/jindotable_storagepolicy_restore/#jindotable-oss-hdfs","text":"","title":"\u4f7f\u7528 JindoTable \u5de5\u5177\u5b9e\u73b0 OSS-HDFS \u7ed3\u6784\u5316\u6570\u636e\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8"},{"location":"jindofs/jindotable_storagepolicy_restore/#_1","text":"JindoTable \u662f\u57fa\u4e8e JindoSDK \u7684\u7ed3\u6784\u5316\u6570\u636e\u6cbb\u7406\u5de5\u5177\uff0c\u76ee\u6807\u662f\u5728 \u201c\u8868\u201d \u6216\u8005 \u201c\u5206\u533a\u201d \u5c42\u9762\u63d0\u4f9b\u4fbf\u6377\u7684\u6570\u636e\u8bbf\u95ee\u548c\u64cd\u4f5c\u3002\u4f7f\u7528 JindoTable \u5de5\u5177\u53ef\u4ee5\u65b9\u4fbf\u5730\u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u548c\u5206\u5c42\u5b58\u50a8\u3002 \u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u4f7f\u7528 JindoTable \u5de5\u5177\u5bf9\u5b58\u50a8\u5728 OSS-HDFS \u4e0a\u7684\u7ed3\u6784\u5316\u6570\u636e\u8fdb\u884c\u51b7\u70ed\u5206\u5c42\u5b58\u50a8\u3002","title":"\u6982\u8ff0"},{"location":"jindofs/jindotable_storagepolicy_restore/#_2","text":"\u53ef\u4ee5\u4ece JindoData \u4e0b\u8f7d\u9875\u9762 \u83b7\u53d6 JindoTable \u5de5\u5177\uff1a 1. \u4e0b\u8f7d jindosdk-<version>-<platform>.tar.gz \uff0c\u6839\u636e\u7cfb\u7edf\u9009\u62e9\u5408\u9002\u7684 <platform> \u3002 2. \u672c\u6587\u4ecb\u7ecd\u7684\u529f\u80fd\u6700\u4f4e\u7248\u672c\u8981\u6c42\u4e3a <version> 6.1.2\u3002 \u6ce8\uff1a - JindoTable \u5de5\u5177\u5bf9 OSS-HDFS \u7684\u4e34\u65f6\u89e3\u51bb\uff08 -once \uff09\u652f\u6301\u8bbe\u7f6e\u89e3\u51bb\u53c2\u6570\uff08 -d/-restoreDays <restore days> \uff09\uff0c\u7248\u672c\u8981\u6c42\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0\u3002\u82e5\u5bf9\u8be5\u529f\u80fd\u6709\u9700\u8981\uff0c\u8bf7\u4f7f\u7528\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0 \u7684\u7248\u672c\u3002","title":"\u83b7\u53d6"},{"location":"jindofs/jindotable_storagepolicy_restore/#_3","text":"\u6309\u7167\u4e0a\u9762\u7684\u6b65\u9aa4\u83b7\u53d6\u538b\u7f29\u5305\u540e\uff0c\u89e3\u538b\u7f29\uff0c\u4fbf\u53ef\u5728 bin/ \u76ee\u5f55\u4e0b\u627e\u5230\u540d\u4e3a jindotable \u7684\u811a\u672c\u3002\u6267\u884c\u811a\u672c\uff0c\u5373\u53ef\u4f7f\u7528 JindoTable \u63d0\u4f9b\u7684\u5404\u79cd\u547d\u4ee4\u3002 \u6267\u884c jindotable -usage \u53ef\u4ee5\u663e\u793a\u5de5\u5177\u652f\u6301\u7684\u5404\u79cd\u547d\u4ee4\u3002 \u4e00\u4e9b\u6ce8\u610f\u4e8b\u9879\uff1a - \u6267\u884c\u65f6\uff0c\u987b\u4fdd\u6301\u89e3\u538b\u7f29\u540e\u7684\u76ee\u5f55\u7ed3\u6784\u3002 - \u9700\u8981\u96c6\u7fa4\u4e0a\u6709 Hadoop \u548c Hive \u73af\u5883\uff0c\u76f8\u5173\u7684\u73af\u5883\u53d8\u91cf\u5df2\u7ecf\u88ab\u6b63\u786e\u914d\u7f6e\u3002","title":"\u57fa\u672c\u7528\u6cd5"},{"location":"jindofs/jindotable_storagepolicy_restore/#_4","text":"\u652f\u6301 OSS-HDFS \u7ed3\u6784\u5316\u6570\u636e\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8\u529f\u80fd\uff0c\u547d\u4ee4\u4e3a -setStoragePolicy \uff0c\u6267\u884c jindotable -help setStoragePolicy \u53ef\u4ee5\u663e\u793a\u5e2e\u52a9\u3002","title":"\u5206\u5c42\u5b58\u50a8\u547d\u4ee4"},{"location":"jindofs/jindotable_storagepolicy_restore/#_5","text":"jindotable -setStoragePolicy -t <dbName.tableName> -std/-ia/-archive/-coldArchive/-once [-c \"<condition>\" | -fullTable] [-d/-restoreDays <restore days>] [-b/-before <before days>] [-p/-parallel <parallelism>] [-mr/-mapReduce] [-e/-explain] [-w/-workingDir <working directory>] [-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u76ee\u6807\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a\u6570\u636e\u5e93\u540d.\u8868\u540d\u3002\u6570\u636e\u5e93\u548c\u8868\u540d\u4e4b\u95f4\u4ee5\u534a\u89d2\u53e5\u53f7\uff08.\uff09\u5206\u9694\u3002\u8868\u53ef\u4ee5\u662f\u5206\u533a\u8868\u6216\u975e\u5206\u533a\u8868\u3002 \u662f -std/-ia/-archive/-coldArchive/-once \u76ee\u6807\u5b58\u50a8\u65b9\u5f0f\u3002\u652f\u6301\u5982\u4e0b\u7c7b\u578b\uff1a-std\uff1a\u6807\u51c6\u5b58\u50a8\uff0c-ia\uff1a\u4f4e\u9891\u5b58\u50a8\uff0c-archive\uff1a\u5f52\u6863\u5b58\u50a8\uff0c-coldArchive\uff1a\u51b7\u5f52\u6863\u5b58\u50a8\uff0c-once\uff1a\u4e34\u65f6\u89e3\u51bb\u3002 \u662f -c \"<condition>\" / -fullTable -fullTable \u548c -c \"<condition>\" \u53ea\u9700\u63d0\u4f9b\u4e00\u4e2a\uff0c\u5373\u8981\u4e48\u6307\u5b9a -c \"<condition>\" \uff0c\u8981\u4e48\u6307\u5b9a -fullTable \u3002\u6307\u5b9a -fullTable \u65f6\uff0c\u5219\u4e3a\u79fb\u52a8\u6574\u8868\uff0c\u65e2\u53ef\u4ee5\u662f\u975e\u5206\u533a\u8868\u4e5f\u53ef\u4ee5\u662f\u5206\u533a\u8868\u3002\u6307\u5b9a -c \"<condition>\" \u65f6\uff0c\u5219\u63d0\u4f9b\u4e86\u4e00\u4e2a\u8fc7\u6ee4\u6761\u4ef6\uff0c\u7528\u6765\u9009\u62e9\u5e0c\u671b\u79fb\u52a8\u7684\u5206\u533a\uff0c\u652f\u6301\u5e38\u89c1\u8fd0\u7b97\u7b26\uff0c\u4f8b\u5982\u5927\u4e8e\u53f7\uff08>\uff09\u3002\u4f8b\u5982\uff0c\u6570\u636e\u7c7b\u578b\u4e3aString\u7684\u5206\u533ads\uff0c\u5e0c\u671b\u5206\u533a\u540d\u5927\u4e8e 'd'\uff0c\u5219\u4ee3\u7801\u4e3a-c \" ds > 'd' \"\u3002 \u662f -d/-restoreDays <restore days> \u6307\u5b9a\u4e34\u65f6\u89e3\u51bb\u7684\u5929\u6570\uff0c\u4ec5\u5bf9\u4e34\u65f6\u89e3\u51bb -once \u751f\u6548\u3002 \u5426 -b/before <before days> \u53ea\u6709\u521b\u5efa\u65f6\u95f4\u8ddd\u79bb\u73b0\u5728\u8d85\u8fc7\u4e00\u5b9a\u5929\u6570\u7684\u8868\u6216\u5206\u533a\u624d\u4f1a\u88ab\u6267\u884c\u3002 \u5426 -p/-parallel <parallelism> \u547d\u4ee4\u6267\u884c\u7684\u5e76\u884c\u5ea6\u3002 \u5426 -mr/-mapReduce \u4f7f\u7528Hadoop MapReduce\u800c\u975e\u672c\u5730\u591a\u7ebf\u7a0b\u6765\u6267\u884c\u547d\u4ee4\u3002 \u5426 -e/-explain \u5982\u679c\u51fa\u73b0\u8be5\u9009\u9879\uff0c\u5219\u4e3a\u89e3\u91ca\uff08explain\uff09\u6a21\u5f0f\uff0c\u53ea\u4f1a\u663e\u793a\u5f85\u64cd\u4f5c\u7684\u5206\u533a\u5217\u8868\uff0c\u800c\u4e0d\u4f1a\u771f\u6b63\u6267\u884c\u547d\u4ee4\u3002 \u5426 -w/-workingDir <working directory> \u53ea\u5728MapReduce\u4f5c\u4e1a\u65f6\u4f7f\u7528\uff0c\u4e3aMapReduce\u4f5c\u4e1a\u7684\u5de5\u4f5c\u76ee\u5f55\u3002\u5fc5\u987b\u6709\u8bfb\u5199\u6743\u9650\uff0c\u5de5\u4f5c\u76ee\u5f55\u53ef\u4ee5\u975e\u7a7a\uff08\u4f5c\u4e1a\u6267\u884c\u8fc7\u7a0b\u4e2d\u4f1a\u521b\u5efa\u4e34\u65f6\u6587\u4ef6\uff0c\u6267\u884c\u5b8c\u6bd5\u4f1a\u6e05\u7406\u4e34\u65f6\u6587\u4ef6\uff09\u3002 \u5426 -l/-logDir <log directory> \u6307\u5b9aLog\u6587\u4ef6\u7684\u76ee\u5f55\u3002 \u5426","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"jindofs/jindotable_storagepolicy_restore/#_6","text":"JindoTable \u63d0\u4f9b\u7684\u4e0a\u8ff0\u529f\u80fd\u57fa\u672c\u5bf9\u5e94\u4e8e OSS-HDFS \u7ebf\u4e0a\u670d\u52a1\u7684\u5206\u5c42\u5b58\u50a8\u80fd\u529b\uff0c\u6709\u4e0b\u5217\u4e8b\u9879\u9700\u8981\u6ce8\u610f\uff1a 1. \u4ec5\u652f\u6301 OSS-HDFS \u5b58\u50a8\u7684\u8868\u6216\u8005\u5206\u533a\u3002\u5bf9\u666e\u901a OSS \u6570\u636e\u7684\u5206\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoTable \u63d0\u4f9b\u7684 archiveTable \u4e0e unarchiveTable \u547d\u4ee4\u3002 2. \u6210\u529f\u6267\u884c -std/-ia/-archive/-coldArchive/-once \u6307\u5b9a\u7684\u8f6c\u6362\u5b58\u50a8\u7c7b\u578b\u547d\u4ee4\u540e\uff0c\u5fc5\u987b\u7b49\u5f85 2 \u5929\u624d\u53ef\u518d\u6b21\u8f6c\u6362\u5b58\u50a8\u7c7b\u578b\u3002 3. \u6807\u51c6\u548c\u4f4e\u9891\u7c7b\u578b\u65e0\u6cd5\u4e34\u65f6\u89e3\u51bb\uff08 -once \uff09\u3002\u5f52\u6863\u3001\u51b7\u5f52\u6863\uff0c\u6216\u8005\u5df2\u7ecf\u5904\u4e8e\u89e3\u51bb\u72b6\u6001\u65f6\uff0c\u53ef\u4ee5\u89e3\u51bb\uff0c\u4f46\u5982\u4e0a\u6240\u8ff0\uff0c\u95f4\u9694\u987b\u8d85\u8fc7 2 \u5929\u3002 4. \u4e34\u65f6\u89e3\u51bb\u547d\u4ee4\u8fd4\u56de\u540e\uff0c\u6570\u636e\u4ecd\u4e0d\u80fd\u7acb\u5373\u53ef\u8bfb\uff0c\u9700\u8981\u7b49\u5f85\u8fdc\u7aef\u670d\u52a1\u5b8c\u6210\u6570\u636e\u7684\u89e3\u51bb\u3002\u5bf9\u4e8e\u5f52\u6863\u6570\u636e\uff0c\u901a\u5e38\u9700\u8981\u6570\u5206\u949f\uff1b\u5bf9\u4e8e\u51b7\u5f52\u6863\u6570\u636e\uff0c\u901a\u5e38\u9700\u8981\u6570\u5c0f\u65f6\u3002 5. \u4e34\u65f6\u89e3\u51bb\u6709\u671f\u9650\u9650\u5236\uff0c\u901a\u8fc7 -d/restoreDays <restore days> \u6307\u5b9a\uff0c\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4 1 \u5929\uff0c\u5230\u671f\u540e\u6570\u636e\u5c06\u56de\u5230\u4e0d\u53ef\u8bfb\u72b6\u6001\u3002\u8be5\u529f\u80fd\u8981\u6c42\u7248\u672c\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0\u3002","title":"\u6ce8\u610f\u4e8b\u9879"},{"location":"jindofs/issues/dls_unarchive_issue/","text":"\u95ee\u9898\u63cf\u8ff0 \u00a7 \u7528\u6237\u8c03\u7528\u89e3\u5f52\u6863\u5c06\u6570\u636e\u4ece\u5f52\u6863/\u51b7\u5f52\u6863\u6062\u590d\u6807\u51c6\u64cd\u4f5c\u540e\uff0c\u6709\u65f6\u8fd8\u662f\u4f1a\u51fa\u73b0\u89e3\u5f52\u6863\u7684\u6587\u4ef6\u4e0d\u53ef\u8bfb\u53d6\u60c5\u51b5\uff0c\u51fa\u73b0\u5982\u4e0b\u62a5\u9519 Read from oss://<DLS_BUCKET>.<DLS_ENDPOINT>/<FILE> with error message: Caused by error 30005: IO error: IO error: [RequestId]: XXXXXXXXXXXXXX [HostId]: <OSS_ENDPOINT> [ErrorMessage]: [E1010]HTTP/1.1 403 Forbidden: <?xml version=\"1.0\" encoding=\"UTF-8\"?><Error> <Code>InvalidObjectState</Code> <Message>The operation is not valid for the object's state</Message> <RequestId>XXXXXXXXXXXXXXX</RequestId> <HostId><DLS_BUCKET>.<OSS_ENDPOINT></HostId> <ObjectName>.dlsdata/xxxxxxxxxxxx</ObjectName></Error> [ErrorCode]: 1010 [RequestId]: XXXXXXXXXXXXXXX \u95ee\u9898\u6839\u56e0 \u00a7 \u51fa\u73b0\u4ee5\u4e0a\u60c5\u51b5\u5927\u6982\u7387\u7531\u4e8e\u7528\u6237\u6570\u636e\u5757\u8fc7\u5927\uff0c\u5bfc\u81f4OSS\u6570\u636e\u5757\u5728\u8f6c\u6362\u5b58\u50a8\u7c7b\u578b\u65f6\u51fa\u73b0\u6570\u636e\u7684\u62f7\u8d1d\u8d85\u65f6\uff0c\u4efb\u52a1\u89e3\u5f52\u6863\u6700\u7ec8\u51fa\u73b0\u5931\u8d25\u3002\u6c38\u4e45\u89e3\u5f52\u6863\u5b58\u5728\u89e3\u51bb\u9650\u5236\uff0c\u63a8\u8350\u7528\u6237\u4f7f\u7528\u4e34\u65f6\u89e3\u51bb\u529f\u80fd\uff0c\u8be6\u7ec6\u7ec6\u8282\u53ef\u4ee5\u53c2\u8003 \u6587\u6863 \u3002 \u4fee\u590d\u6d41\u7a0b \u00a7 \u51fa\u73b0\u8fd9\u79cd\u60c5\u51b5\u7528\u6237\u5982\u4f55\u8fdb\u884c\u6570\u636e\u4fee\u590d\uff0c\u9700\u8981\u4f9d\u8d56JindoFsck\u547d\u4ee4\u6765\u4fee\u590d\uff0cJindoFsck \u547d\u4ee4 \u4e0b\u8f7d\u94fe\u63a5 \uff0c JindoFsck\u547d\u4ee4\u4f7f\u7528\u5982\u4e0b [root@master-1-1(192.168.0.182) ~]# ./jindofsck jindofsck [-help] [-version] [-restoreFileBlocks -dlsUri <path> [-restoreDays <restoreDays>]] [-restoreBlock -dlsUri <path> -block <block> [-restoreDays <restoreDays>]] [-restoreBlocks -dlsUri <path> [-days <days>] [-restoreDays <restoreDays>]] JindoFsck \u73b0\u5728\u652f\u6301\u4e09\u79cd\u7c7b\u578b\u6570\u636e\u7684\u4fee\u590d\uff0c\u5206\u522b\u5bf9\u5e94\u89e3\u51bb\u5355\u4e2a\u6570\u636e\u5757\uff0c\u89e3\u51bb\u5355\u4e2a\u6587\u4ef6\uff0c\u89e3\u51bb\u7279\u5b9a\u65f6\u95f4\u5185\u6c38\u4e45\u89e3\u51bb\u5931\u8d25\u7684\u6570\u636e\u5757\u3002\u7528\u6237\u53ef\u4ee5\u6839\u636e\u9700\u8981\u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\u53bb\u4e34\u65f6\u89e3\u51bb\u6570\u636e\u3002 \u89e3\u51bb\u5355\u4e2a\u6570\u636e\u5757 \u00a7 ./jindofsck -restoreBlock -dlsUri oss://<DLS_BUCKET>.<DLS_ENDPOINT>/ -block .dlsdata/hosts24 -restoreDays \u53c2\u6570\u8bf4\u660e | \u53c2\u6570 | \u63cf\u8ff0 | \u662f\u5426\u5fc5\u9009 | | :--- | :--- | :--- | | -dlsUri | OSS-HDFS Bucket\u5730\u5740\u3002 | \u662f| | -block | \u9700\u8981\u6062\u590d\u7684block\u7684\u5730\u5740\uff0c\u91c7\u7528\u76f8\u5bf9\u8def\u5f84\uff0c\u4e0d\u80fdoss:// \u6216\u8005 / \u5f00\u5934\uff0c \u683c\u5f0f .dlsdata/xxxxxx | \u662f| | -restoreDays | \u6307\u5b9a\u4e34\u65f6\u89e3\u51bb\u5929\u6570\uff0c\u5bf9\u4e8e\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 7 \u5929\uff0c\u51b7\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 365 \u5929\uff0c \u9ed8\u8ba4\u89e3\u51bb\u5f52\u68637\u5929\uff0c\u51b7\u5f52\u686330\u5929\u3002 | \u5426 | \u89e3\u51bb\u5355\u4e2a\u6587\u4ef6 \u00a7 ./jindofsck -restoreFileBlocks -dlsUri oss://<DLS_BUCKET>.<DLS_ENDPOINT>/<PATH> -restoreDays 2 \u53c2\u6570\u8bf4\u660e | \u53c2\u6570 | \u63cf\u8ff0 | \u662f\u5426\u5fc5\u9009 | | :--- | :--- | :--- | | -dlsUri | OSS-HDFS Bucket\u5730\u5740\u3002 | \u662f| | -restoreDays | \u6307\u5b9a\u4e34\u65f6\u89e3\u51bb\u5929\u6570\uff0c\u5bf9\u4e8e\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 7 \u5929\uff0c\u51b7\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 365 \u5929\uff0c \u9ed8\u8ba4\u89e3\u51bb\u5f52\u68637\u5929\uff0c\u51b7\u5f52\u686330\u5929\u3002 | \u5426 | \u89e3\u51bb\u6307\u5b9a\u65f6\u95f4\u6bb5\u5931\u8d25\u7684\u6570\u636e \u00a7 ./jindofsck -restoreBlocks -dlsUri oss://<DLS_BUCKET>.<DLS_ENDPOINT>/<PATH> -days 2 -restoreDays 2 \u53c2\u6570\u8bf4\u660e | \u53c2\u6570 | \u63cf\u8ff0 | \u662f\u5426\u5fc5\u9009 | | :--- | :--- | :--- | | -dlsUri | OSS-HDFS Bucket\u5730\u5740\u3002 | \u662f| | -days | \u8868\u793a\u6062\u590d\u51e0\u5929\u5185\u6c38\u4e45\u89e3\u5f52\u6863\u5931\u8d25\u7684\u6570\u636e\uff0c\u9ed8\u8ba4\u503c\u4e3a 2\uff0c \u8868\u793a\u6062\u590d\u4e24\u5929\u5185\u6c38\u4e45\u89e3\u5f52\u6863\u5931\u8d25\u7684\u6570\u636e\u3002 | \u5426| | -restoreDays | \u6307\u5b9a\u4e34\u65f6\u89e3\u51bb\u5929\u6570\uff0c\u5bf9\u4e8e\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 7 \u5929\uff0c\u51b7\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 365 \u5929\uff0c \u9ed8\u8ba4\u89e3\u51bb\u5f52\u68637\u5929\uff0c\u51b7\u5f52\u686330\u5929\u3002 | \u5426 |","title":"Dls unarchive issue"},{"location":"jindofs/issues/dls_unarchive_issue/#_1","text":"\u7528\u6237\u8c03\u7528\u89e3\u5f52\u6863\u5c06\u6570\u636e\u4ece\u5f52\u6863/\u51b7\u5f52\u6863\u6062\u590d\u6807\u51c6\u64cd\u4f5c\u540e\uff0c\u6709\u65f6\u8fd8\u662f\u4f1a\u51fa\u73b0\u89e3\u5f52\u6863\u7684\u6587\u4ef6\u4e0d\u53ef\u8bfb\u53d6\u60c5\u51b5\uff0c\u51fa\u73b0\u5982\u4e0b\u62a5\u9519 Read from oss://<DLS_BUCKET>.<DLS_ENDPOINT>/<FILE> with error message: Caused by error 30005: IO error: IO error: [RequestId]: XXXXXXXXXXXXXX [HostId]: <OSS_ENDPOINT> [ErrorMessage]: [E1010]HTTP/1.1 403 Forbidden: <?xml version=\"1.0\" encoding=\"UTF-8\"?><Error> <Code>InvalidObjectState</Code> <Message>The operation is not valid for the object's state</Message> <RequestId>XXXXXXXXXXXXXXX</RequestId> <HostId><DLS_BUCKET>.<OSS_ENDPOINT></HostId> <ObjectName>.dlsdata/xxxxxxxxxxxx</ObjectName></Error> [ErrorCode]: 1010 [RequestId]: XXXXXXXXXXXXXXX","title":"\u95ee\u9898\u63cf\u8ff0"},{"location":"jindofs/issues/dls_unarchive_issue/#_2","text":"\u51fa\u73b0\u4ee5\u4e0a\u60c5\u51b5\u5927\u6982\u7387\u7531\u4e8e\u7528\u6237\u6570\u636e\u5757\u8fc7\u5927\uff0c\u5bfc\u81f4OSS\u6570\u636e\u5757\u5728\u8f6c\u6362\u5b58\u50a8\u7c7b\u578b\u65f6\u51fa\u73b0\u6570\u636e\u7684\u62f7\u8d1d\u8d85\u65f6\uff0c\u4efb\u52a1\u89e3\u5f52\u6863\u6700\u7ec8\u51fa\u73b0\u5931\u8d25\u3002\u6c38\u4e45\u89e3\u5f52\u6863\u5b58\u5728\u89e3\u51bb\u9650\u5236\uff0c\u63a8\u8350\u7528\u6237\u4f7f\u7528\u4e34\u65f6\u89e3\u51bb\u529f\u80fd\uff0c\u8be6\u7ec6\u7ec6\u8282\u53ef\u4ee5\u53c2\u8003 \u6587\u6863 \u3002","title":"\u95ee\u9898\u6839\u56e0"},{"location":"jindofs/issues/dls_unarchive_issue/#_3","text":"\u51fa\u73b0\u8fd9\u79cd\u60c5\u51b5\u7528\u6237\u5982\u4f55\u8fdb\u884c\u6570\u636e\u4fee\u590d\uff0c\u9700\u8981\u4f9d\u8d56JindoFsck\u547d\u4ee4\u6765\u4fee\u590d\uff0cJindoFsck \u547d\u4ee4 \u4e0b\u8f7d\u94fe\u63a5 \uff0c JindoFsck\u547d\u4ee4\u4f7f\u7528\u5982\u4e0b [root@master-1-1(192.168.0.182) ~]# ./jindofsck jindofsck [-help] [-version] [-restoreFileBlocks -dlsUri <path> [-restoreDays <restoreDays>]] [-restoreBlock -dlsUri <path> -block <block> [-restoreDays <restoreDays>]] [-restoreBlocks -dlsUri <path> [-days <days>] [-restoreDays <restoreDays>]] JindoFsck \u73b0\u5728\u652f\u6301\u4e09\u79cd\u7c7b\u578b\u6570\u636e\u7684\u4fee\u590d\uff0c\u5206\u522b\u5bf9\u5e94\u89e3\u51bb\u5355\u4e2a\u6570\u636e\u5757\uff0c\u89e3\u51bb\u5355\u4e2a\u6587\u4ef6\uff0c\u89e3\u51bb\u7279\u5b9a\u65f6\u95f4\u5185\u6c38\u4e45\u89e3\u51bb\u5931\u8d25\u7684\u6570\u636e\u5757\u3002\u7528\u6237\u53ef\u4ee5\u6839\u636e\u9700\u8981\u6267\u884c\u4e0a\u8ff0\u547d\u4ee4\u53bb\u4e34\u65f6\u89e3\u51bb\u6570\u636e\u3002","title":"\u4fee\u590d\u6d41\u7a0b"},{"location":"jindofs/issues/dls_unarchive_issue/#_4","text":"./jindofsck -restoreBlock -dlsUri oss://<DLS_BUCKET>.<DLS_ENDPOINT>/ -block .dlsdata/hosts24 -restoreDays \u53c2\u6570\u8bf4\u660e | \u53c2\u6570 | \u63cf\u8ff0 | \u662f\u5426\u5fc5\u9009 | | :--- | :--- | :--- | | -dlsUri | OSS-HDFS Bucket\u5730\u5740\u3002 | \u662f| | -block | \u9700\u8981\u6062\u590d\u7684block\u7684\u5730\u5740\uff0c\u91c7\u7528\u76f8\u5bf9\u8def\u5f84\uff0c\u4e0d\u80fdoss:// \u6216\u8005 / \u5f00\u5934\uff0c \u683c\u5f0f .dlsdata/xxxxxx | \u662f| | -restoreDays | \u6307\u5b9a\u4e34\u65f6\u89e3\u51bb\u5929\u6570\uff0c\u5bf9\u4e8e\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 7 \u5929\uff0c\u51b7\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 365 \u5929\uff0c \u9ed8\u8ba4\u89e3\u51bb\u5f52\u68637\u5929\uff0c\u51b7\u5f52\u686330\u5929\u3002 | \u5426 |","title":"\u89e3\u51bb\u5355\u4e2a\u6570\u636e\u5757"},{"location":"jindofs/issues/dls_unarchive_issue/#_5","text":"./jindofsck -restoreFileBlocks -dlsUri oss://<DLS_BUCKET>.<DLS_ENDPOINT>/<PATH> -restoreDays 2 \u53c2\u6570\u8bf4\u660e | \u53c2\u6570 | \u63cf\u8ff0 | \u662f\u5426\u5fc5\u9009 | | :--- | :--- | :--- | | -dlsUri | OSS-HDFS Bucket\u5730\u5740\u3002 | \u662f| | -restoreDays | \u6307\u5b9a\u4e34\u65f6\u89e3\u51bb\u5929\u6570\uff0c\u5bf9\u4e8e\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 7 \u5929\uff0c\u51b7\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 365 \u5929\uff0c \u9ed8\u8ba4\u89e3\u51bb\u5f52\u68637\u5929\uff0c\u51b7\u5f52\u686330\u5929\u3002 | \u5426 |","title":"\u89e3\u51bb\u5355\u4e2a\u6587\u4ef6"},{"location":"jindofs/issues/dls_unarchive_issue/#_6","text":"./jindofsck -restoreBlocks -dlsUri oss://<DLS_BUCKET>.<DLS_ENDPOINT>/<PATH> -days 2 -restoreDays 2 \u53c2\u6570\u8bf4\u660e | \u53c2\u6570 | \u63cf\u8ff0 | \u662f\u5426\u5fc5\u9009 | | :--- | :--- | :--- | | -dlsUri | OSS-HDFS Bucket\u5730\u5740\u3002 | \u662f| | -days | \u8868\u793a\u6062\u590d\u51e0\u5929\u5185\u6c38\u4e45\u89e3\u5f52\u6863\u5931\u8d25\u7684\u6570\u636e\uff0c\u9ed8\u8ba4\u503c\u4e3a 2\uff0c \u8868\u793a\u6062\u590d\u4e24\u5929\u5185\u6c38\u4e45\u89e3\u5f52\u6863\u5931\u8d25\u7684\u6570\u636e\u3002 | \u5426| | -restoreDays | \u6307\u5b9a\u4e34\u65f6\u89e3\u51bb\u5929\u6570\uff0c\u5bf9\u4e8e\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 7 \u5929\uff0c\u51b7\u5f52\u6863\u7c7b\u578b\u6700\u591a\u89e3\u51bb 365 \u5929\uff0c \u9ed8\u8ba4\u89e3\u51bb\u5f52\u68637\u5929\uff0c\u51b7\u5f52\u686330\u5929\u3002 | \u5426 |","title":"\u89e3\u51bb\u6307\u5b9a\u65f6\u95f4\u6bb5\u5931\u8d25\u7684\u6570\u636e"},{"location":"jindofuse/jindofuse_quickstart/","text":"JindoFuse \u4f7f\u7528\u6307\u5357 \u00a7 \u6982\u8ff0 \u00a7 OSS/OSS-HDFS \u53ef\u4ee5\u901a\u8fc7 JindoFuse \u63d0\u4f9b POSIX \u652f\u6301\uff0c\u5c06 OSS/OSS-HDFS \u4e0a\u7684\u6587\u4ef6\u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c OSS/OSS-HDFS \u4e0a\u7684\u6587\u4ef6\u3002 \u90e8\u7f72\u5b89\u88c5 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.y.z.tar.gz\uff08x.y.z\u8868\u7248\u672c\u53f7\uff09\uff0c \u4e0b\u8f7d\u9875\u9762 \u3002 \u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u53c2\u89c1 \u6587\u6863\u94fe\u63a5 \u6302\u8f7d JindoFuse \u00a7 \u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002 \u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a mkdir -p <mountpoint> \u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a jindo-fuse <mount_point> -ouri=[<osspath>] \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 <oss_path> \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 <mount_point> \u3002 <mount_point> \u9700\u66ff\u6362\u4e3a\u4e00\u4e2a\u672c\u5730\u8def\u5f84\u3002 <oss_path> \u9700\u66ff\u6362\u4e3a\u5f85\u6620\u5c04\u7684 OSS/OSS-HDFS \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a Bucket \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002\u5982 oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/subdir/ \u6302\u8f7d OSS \u548c OSS-HDFS \u8def\u5f84\u7684\u65b9\u5f0f\u57fa\u672c\u76f8\u540c\uff0c\u4ec5\u8def\u5f84\u4e2d\u7684 endpoint \u7565\u6709\u4e0d\u540c\u3002\u6839\u8def\u5f84\u793a\u4f8b\u53ef\u53c2\u89c1\uff1a \u300aOSS/OSS-HDFS \u5feb\u901f\u5165\u95e8\u300b \u786e\u8ba4\u6302\u8f7d\u6210\u529f\uff0cjindo-fuse \u8fdb\u7a0b\u5b58\u5728\uff0c\u4e14\u542f\u52a8\u53c2\u6570\u4e0e\u9884\u671f\u4e00\u81f4 ps -ef | grep jindo-fuse \u8bbf\u95ee JindoFuse \u00a7 \u5982\u679c\u5c06 JindoFS \u670d\u52a1\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1 ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/ \u5378\u8f7d JindoFuse \u00a7 \u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point> \u81ea\u52a8\u5378\u8f7d JindoFuse \u00a7 \u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u7279\u6027\u652f\u6301 \u00a7 \u76ee\u524d JindoFuse \u5df2\u7ecf\u652f\u6301\u4ee5\u4e0b POSIX API\uff1a \u7279\u6027 \u8bf4\u660e OSS OSS-HDFS getattr() \u67e5\u8be2\u6587\u4ef6\u5c5e\u6027\uff0c\u7c7b\u4f3c ls \u2705 \u2705 mkdir() \u521b\u5efa\u76ee\u5f55\uff0c\u7c7b\u4f3c mkdir \u2705 \u2705 rmdir() \u5220\u9664\u76ee\u5f55\uff0c\u7c7b\u4f3c rm -rf \u2705 \u2705 unlink() \u5220\u9664\u6587\u4ef6\uff0c\u7c7b\u4f3c unlink \u2705 \u2705 rename() \u91cd\u547d\u540d\uff0c\u7c7b\u4f3c mv \u2705 \u2705 read() \u987a\u5e8f\u8bfb\u53d6 \u2705 \u2705 pread() \u968f\u673a\u8bfb \u2705 \u2705 write() \u987a\u5e8f\u5199 \u2705 \u2705 pwrite() \u968f\u673a\u5199 \u274c \u2705 flush() \u5237\u65b0\u5185\u5b58\u5230\u5185\u6838\u7f13\u51b2\u533a \u4ec5\u652f\u6301\u4ee5\u6700\u8ffd\u52a0\u5199\u65b9\u5f0f\u6253\u5f00\u7684\u6587\u4ef6 \u2705 fsync() \u5237\u65b0\u5185\u5b58\u5230\u78c1\u76d8 \u4ec5\u652f\u6301\u4ee5\u6700\u8ffd\u52a0\u5199\u65b9\u5f0f\u6253\u5f00\u7684\u6587\u4ef6 \u2705 release() \u5173\u95ed\u6587\u4ef6 \u2705 \u2705 readdir() \u8bfb\u53d6\u76ee\u5f55 \u2705 \u2705 create() \u521b\u5efa\u6587\u4ef6 \u2705 \u2705 open() O_APPEND \u901a\u8fc7\u8ffd\u52a0\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 \u652f\u6301\uff0c\u4f46\u6709 \u4f7f\u7528\u9650\u5236 \u2705 open() O_TRUNC \u901a\u8fc7\u8986\u76d6\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 \u2705 \u2705 ftruncate() \u5bf9\u6253\u5f00\u7684\u6587\u4ef6\u8fdb\u884c\u622a\u65ad \u274c \u2705 truncate() \u5bf9\u672a\u6253\u5f00\u7684\u6587\u4ef6\u8fdb\u884c\u622a\u65ad\uff0c\u7c7b\u4f3c truncate -s \u274c \u2705 lseek() \u6307\u5b9a\u6253\u5f00\u6587\u4ef6\u4e2d\u7684\u8bfb\u5199\u4f4d\u7f6e\u3002 \u274c \u2705 chmod() \u4fee\u6539\u6587\u4ef6\u6743\u9650\uff0c\u7c7b\u4f3c chmod \u274c \u2705 access() \u67e5\u8be2\u6587\u4ef6\u6743\u9650 \u2705 \u2705 utimes() \u4fee\u6539\u6587\u4ef6\u7684\u5b58\u53d6\u65f6\u95f4\u548c\u66f4\u6539\u65f6\u95f4 \u274c \u2705 setxattr() \u4fee\u6539\u6587\u4ef6 xattr \u5c5e\u6027 \u274c \u2705 getxattr() \u83b7\u53d6\u6587\u4ef6 xattr \u5c5e\u6027 \u274c \u2705 listxattr() \u5217\u4e3e\u6587\u4ef6 xattr \u5c5e\u6027 \u274c \u2705 removexattr() \u5220\u9664\u6587\u4ef6 xattr \u5c5e\u6027 \u274c \u2705 lock() \u652f\u6301 posix \u9501\uff0c\u7c7b\u4f3c fcntl \u274c \u2705 fallocate() \u4e3a\u6587\u4ef6\u9884\u5206\u914d\u7269\u7406\u7a7a\u95f4 \u274c \u2705 symlink() \u521b\u5efa\u8f6f\u8fde\u63a5 \u274c \u76ee\u524d\u4ec5\u652f\u6301 OSS-HDFS \u5185\u90e8\u4f7f\u7528\uff0c\u4e14\u4e0d\u652f\u6301\u7f13\u5b58\u52a0\u901f readlink() \u8bfb\u53d6\u8f6f\u8fde\u63a5 \u274c \u76ee\u524d\u4ec5\u652f\u6301 OSS-HDFS \u5185\u90e8\u4f7f\u7528\uff0c\u4e14\u4e0d\u652f\u6301\u7f13\u5b58\u52a0\u901f \u9ad8\u9636\u4f7f\u7528 \u00a7 \u6302\u8f7d\u9009\u9879 \u00a7 \u53c2\u6570\u540d\u79f0 \u5fc5\u9009 \u7248\u672c \u53c2\u6570\u8bf4\u660e \u4f7f\u7528\u8303\u4f8b uri \u2713 4.3.0+ \u914d\u7f6e\u9700\u8981\u6620\u5c04\u7684 oss \u8def\u5f84\u3002\u8def\u5f84\u53ef\u4ee5\u662f\u6839\u76ee\u5f55\uff0c\u4e5f\u53ef\u4ee5\u662f\u5b50\u76ee\u5f55\u3002\u4f8b\u5982\uff1aoss://examplebucket/ \u6216 oss://examplebucket/subdir\u3002 -ouri=oss://examplebucket/ f 4.3.0+ \u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u9ed8\u8ba4\u4f7f\u7528\u5b88\u62a4\u8fdb\u7a0b\u65b9\u5f0f\u540e\u53f0\u542f\u52a8\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -f d 4.3.0+ \u4f7f\u7528 Debug \u6a21\u5f0f\uff0c\u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -d auto_unmount 4.3.0+ fuse\u8fdb\u7a0b\u9000\u51fa\u540e\u81ea\u52a8umount\u6302\u8f7d\u8282\u70b9\u3002 -oauto_unmount ro 4.3.0+ \u53ea\u8bfb\u6302\u8f7d\uff0c\u542f\u7528\u53c2\u6570\u540e\u4e0d\u5141\u8bb8\u5199\u64cd\u4f5c\u3002 -oro direct_io 4.3.0+ \u5f00\u542f\u540e\uff0c\u8bfb\u5199\u6587\u4ef6\u53ef\u4ee5\u7ed5\u8fc7page cache\u3002 -odirect_io kernel_cache 4.3.0+ \u5f00\u542f\u540e\uff0c\u5229\u7528\u5185\u6838\u7f13\u5b58\u4f18\u5316\u8bfb\u6027\u80fd\u3002 -okernel_cache auto_cache 4.3.0+ \u9ed8\u8ba4\u5f00\u542f\uff0c\u4e0ekernel_cache \u4e8c\u9009\u4e00\uff0c\u4e0ekernel_cache\u4e0d\u540c\u7684\u662f\uff0c\u5982\u679c\u6587\u4ef6\u5927\u5c0f\u6216\u4fee\u6539\u65f6\u95f4\u53d1\u751f\u53d8\u5316\uff0c\u7f13\u5b58\u5c31\u4f1a\u5931\u6548\u3002 entry_timeout 4.3.0+ \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oentry_timeout=60 attr_timeout 4.3.0+ \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oattr_timeout=60 negative_timeout 4.3.0+ \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u5931\u8d25\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -onegative_timeout=0 max_idle_threads 4.3.0+ \u9ed8\u8ba4\u503c\uff0c10\u3002\u5904\u7406\u5185\u6838\u56de\u8c03\u7684\u7a7a\u95f2\u7ebf\u7a0b\u6c60\u3002 -omax_idle_threads=10 xengine 4.3.0+ \u6253\u5f00\u7f13\u5b58 -oxengine pread 4.5.1+ \u9ed8\u8ba4\u4f7f\u7528\u987a\u5e8f\u8bfb\u3002\u6253\u5f00\u540e\uff0c\u4f7f\u7528\u968f\u673a\u8bfb\u4ee3\u66ff\u987a\u5e8f\u8bfb\uff0c\u9002\u7528\u4e8e\u968f\u673a\u8bfb\u8fdc\u591a\u4e8e\u987a\u5e8f\u8bfb\u7684\u573a\u666f\u3002 -opread no_symlink 4.5.1+ \u914d\u7f6e\u540e\uff0c\u5173\u95edsymlink\u529f\u80fd\u3002 -ono_symlink no_writeback 4.5.1+ \u914d\u7f6e\u540e\uff0c\u5173\u95edwriteback\u529f\u80fd\u3002 -ono_writeback no_flock 4.5.1+ \u914d\u7f6e\u540e\uff0c\u5173\u95edflock\u529f\u80fd\u3002 -ono_flock no_xattr 4.5.1+ \u914d\u7f6e\u540e\uff0c\u5173\u95edxttar\u529f\u80fd\u3002 -ono_xattr \u914d\u7f6e\u9009\u9879 \u00a7 \u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u8bf4\u660e logger.dir /tmp/bigboot-log \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF logger.verbose 0 \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint \u8bbf\u95ee JindoFS \u670d\u52a1\u7684\u5730\u5740\uff0c\u5982oss-cn-xxx.aliyuncs.com fs.oss.accessKeyId \u8bbf\u95ee JindoFS \u670d\u52a1\u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret \u8bbf\u95ee JindoFS \u670d\u52a1\u9700\u8981\u7684 accessKeySecret \u66f4\u591a\u53c2\u6570\u53ef\u89c1 \u300a\u5ba2\u6237\u7aef\u5e38\u7528\u914d\u7f6e\u300b \u3002 \u652f\u6301\u5c06 jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<oss_path>] -ofs.oss.endpoint=[<your_endpoint>] -ofs.oss.accessKeyId=[<your_key_id>] -ofs.oss.accessKeySecret=[<your_key_secret>] \u5e38\u89c1\u95ee\u9898 \u00a7 Input/Output error \u00a7 \u4e0d\u50cf\u4f7f\u7528 JindoSDK \u8c03\u7528 API \u53ef\u4ee5\u83b7\u53d6\u66f4\u4e3a\u5177\u4f53\u7684 ErrorMsg\uff0cJindoFuse \u53ea\u80fd\u663e\u793a\u64cd\u4f5c\u7cfb\u7edf\u9884\u8bbe\u7684\u9519\u8bef\u4fe1\u606f\uff0c\u6bd4\u5982\u4ee5\u4e0b\u9519\u8bef\u5c31\u975e\u5e38\u5e38\u89c1\uff1a $ ls /mnt/oss/ ls: /mnt/oss/: Input/output error \u5982\u679c\u9700\u8981\u5b9a\u4f4d\u5177\u4f53\u7684\u9519\u8bef\u539f\u56e0\uff0c\u53ef\u4ee5\u6839\u636e JindoSDK \u914d\u7f6e\u4e2d\u7684 logger.dir\uff0c\u5728\u6307\u5b9a\u8def\u5f84\u4e0b\u7684 jindosdk.log \u6587\u4ef6\u4e2d\uff0c\u5bfb\u627e\u5177\u4f53\u7684\u9519\u8bef\u3002 \u4e0b\u9762\u5c55\u793a\u7684\u8fd9\u4e2a\u9519\u8bef\u5c31\u6765\u81ea jindosdk.log \uff0c\u6839\u636e\u62a5\u9519\u4fe1\u606f\u53ef\u89c1\uff0c\u8fd9\u4e2a\u9519\u8bef\u662f\u5e38\u89c1\u7684\u9274\u6743\u95ee\u9898\u3002 EMMDD HH:mm:ss jindofs_connectivity.cpp:13] Please check your Endpoint/Bucket/RoleArn. Failed test connectivity, operation: mkdir, errMsg: [RequestId]: 618B8183343EA53531C62B74 [HostId]: oss-cn-shanghai-internal.aliyuncs.com [ErrorMessage]: [E1010]HTTP/1.1 403 Forbidden ...","title":"JindoFuse \u5feb\u901f\u5165\u95e8"},{"location":"jindofuse/jindofuse_quickstart/#jindofuse","text":"","title":"JindoFuse \u4f7f\u7528\u6307\u5357"},{"location":"jindofuse/jindofuse_quickstart/#_1","text":"OSS/OSS-HDFS \u53ef\u4ee5\u901a\u8fc7 JindoFuse \u63d0\u4f9b POSIX \u652f\u6301\uff0c\u5c06 OSS/OSS-HDFS \u4e0a\u7684\u6587\u4ef6\u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u8ba9\u60a8\u80fd\u591f\u50cf\u64cd\u4f5c\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u4e00\u6837\u64cd\u4f5c OSS/OSS-HDFS \u4e0a\u7684\u6587\u4ef6\u3002","title":"\u6982\u8ff0"},{"location":"jindofuse/jindofuse_quickstart/#_2","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.y.z.tar.gz\uff08x.y.z\u8868\u7248\u672c\u53f7\uff09\uff0c \u4e0b\u8f7d\u9875\u9762 \u3002 \u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u53c2\u89c1 \u6587\u6863\u94fe\u63a5","title":"\u90e8\u7f72\u5b89\u88c5"},{"location":"jindofuse/jindofuse_quickstart/#jindofuse_1","text":"\u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002 \u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a mkdir -p <mountpoint> \u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a jindo-fuse <mount_point> -ouri=[<osspath>] \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 <oss_path> \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 <mount_point> \u3002 <mount_point> \u9700\u66ff\u6362\u4e3a\u4e00\u4e2a\u672c\u5730\u8def\u5f84\u3002 <oss_path> \u9700\u66ff\u6362\u4e3a\u5f85\u6620\u5c04\u7684 OSS/OSS-HDFS \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a Bucket \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002\u5982 oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/subdir/ \u6302\u8f7d OSS \u548c OSS-HDFS \u8def\u5f84\u7684\u65b9\u5f0f\u57fa\u672c\u76f8\u540c\uff0c\u4ec5\u8def\u5f84\u4e2d\u7684 endpoint \u7565\u6709\u4e0d\u540c\u3002\u6839\u8def\u5f84\u793a\u4f8b\u53ef\u53c2\u89c1\uff1a \u300aOSS/OSS-HDFS \u5feb\u901f\u5165\u95e8\u300b \u786e\u8ba4\u6302\u8f7d\u6210\u529f\uff0cjindo-fuse \u8fdb\u7a0b\u5b58\u5728\uff0c\u4e14\u542f\u52a8\u53c2\u6570\u4e0e\u9884\u671f\u4e00\u81f4 ps -ef | grep jindo-fuse","title":"\u6302\u8f7d JindoFuse"},{"location":"jindofuse/jindofuse_quickstart/#jindofuse_2","text":"\u5982\u679c\u5c06 JindoFS \u670d\u52a1\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1 ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/","title":"\u8bbf\u95ee JindoFuse"},{"location":"jindofuse/jindofuse_quickstart/#jindofuse_3","text":"\u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point>","title":"\u5378\u8f7d JindoFuse"},{"location":"jindofuse/jindofuse_quickstart/#jindofuse_4","text":"\u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002","title":"\u81ea\u52a8\u5378\u8f7d JindoFuse"},{"location":"jindofuse/jindofuse_quickstart/#_3","text":"\u76ee\u524d JindoFuse \u5df2\u7ecf\u652f\u6301\u4ee5\u4e0b POSIX API\uff1a \u7279\u6027 \u8bf4\u660e OSS OSS-HDFS getattr() \u67e5\u8be2\u6587\u4ef6\u5c5e\u6027\uff0c\u7c7b\u4f3c ls \u2705 \u2705 mkdir() \u521b\u5efa\u76ee\u5f55\uff0c\u7c7b\u4f3c mkdir \u2705 \u2705 rmdir() \u5220\u9664\u76ee\u5f55\uff0c\u7c7b\u4f3c rm -rf \u2705 \u2705 unlink() \u5220\u9664\u6587\u4ef6\uff0c\u7c7b\u4f3c unlink \u2705 \u2705 rename() \u91cd\u547d\u540d\uff0c\u7c7b\u4f3c mv \u2705 \u2705 read() \u987a\u5e8f\u8bfb\u53d6 \u2705 \u2705 pread() \u968f\u673a\u8bfb \u2705 \u2705 write() \u987a\u5e8f\u5199 \u2705 \u2705 pwrite() \u968f\u673a\u5199 \u274c \u2705 flush() \u5237\u65b0\u5185\u5b58\u5230\u5185\u6838\u7f13\u51b2\u533a \u4ec5\u652f\u6301\u4ee5\u6700\u8ffd\u52a0\u5199\u65b9\u5f0f\u6253\u5f00\u7684\u6587\u4ef6 \u2705 fsync() \u5237\u65b0\u5185\u5b58\u5230\u78c1\u76d8 \u4ec5\u652f\u6301\u4ee5\u6700\u8ffd\u52a0\u5199\u65b9\u5f0f\u6253\u5f00\u7684\u6587\u4ef6 \u2705 release() \u5173\u95ed\u6587\u4ef6 \u2705 \u2705 readdir() \u8bfb\u53d6\u76ee\u5f55 \u2705 \u2705 create() \u521b\u5efa\u6587\u4ef6 \u2705 \u2705 open() O_APPEND \u901a\u8fc7\u8ffd\u52a0\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 \u652f\u6301\uff0c\u4f46\u6709 \u4f7f\u7528\u9650\u5236 \u2705 open() O_TRUNC \u901a\u8fc7\u8986\u76d6\u5199\u7684\u65b9\u5f0f\u6253\u5f00\u6587\u4ef6 \u2705 \u2705 ftruncate() \u5bf9\u6253\u5f00\u7684\u6587\u4ef6\u8fdb\u884c\u622a\u65ad \u274c \u2705 truncate() \u5bf9\u672a\u6253\u5f00\u7684\u6587\u4ef6\u8fdb\u884c\u622a\u65ad\uff0c\u7c7b\u4f3c truncate -s \u274c \u2705 lseek() \u6307\u5b9a\u6253\u5f00\u6587\u4ef6\u4e2d\u7684\u8bfb\u5199\u4f4d\u7f6e\u3002 \u274c \u2705 chmod() \u4fee\u6539\u6587\u4ef6\u6743\u9650\uff0c\u7c7b\u4f3c chmod \u274c \u2705 access() \u67e5\u8be2\u6587\u4ef6\u6743\u9650 \u2705 \u2705 utimes() \u4fee\u6539\u6587\u4ef6\u7684\u5b58\u53d6\u65f6\u95f4\u548c\u66f4\u6539\u65f6\u95f4 \u274c \u2705 setxattr() \u4fee\u6539\u6587\u4ef6 xattr \u5c5e\u6027 \u274c \u2705 getxattr() \u83b7\u53d6\u6587\u4ef6 xattr \u5c5e\u6027 \u274c \u2705 listxattr() \u5217\u4e3e\u6587\u4ef6 xattr \u5c5e\u6027 \u274c \u2705 removexattr() \u5220\u9664\u6587\u4ef6 xattr \u5c5e\u6027 \u274c \u2705 lock() \u652f\u6301 posix \u9501\uff0c\u7c7b\u4f3c fcntl \u274c \u2705 fallocate() \u4e3a\u6587\u4ef6\u9884\u5206\u914d\u7269\u7406\u7a7a\u95f4 \u274c \u2705 symlink() \u521b\u5efa\u8f6f\u8fde\u63a5 \u274c \u76ee\u524d\u4ec5\u652f\u6301 OSS-HDFS \u5185\u90e8\u4f7f\u7528\uff0c\u4e14\u4e0d\u652f\u6301\u7f13\u5b58\u52a0\u901f readlink() \u8bfb\u53d6\u8f6f\u8fde\u63a5 \u274c \u76ee\u524d\u4ec5\u652f\u6301 OSS-HDFS \u5185\u90e8\u4f7f\u7528\uff0c\u4e14\u4e0d\u652f\u6301\u7f13\u5b58\u52a0\u901f","title":"\u7279\u6027\u652f\u6301"},{"location":"jindofuse/jindofuse_quickstart/#_4","text":"","title":"\u9ad8\u9636\u4f7f\u7528"},{"location":"jindofuse/jindofuse_quickstart/#_5","text":"\u53c2\u6570\u540d\u79f0 \u5fc5\u9009 \u7248\u672c \u53c2\u6570\u8bf4\u660e \u4f7f\u7528\u8303\u4f8b uri \u2713 4.3.0+ \u914d\u7f6e\u9700\u8981\u6620\u5c04\u7684 oss \u8def\u5f84\u3002\u8def\u5f84\u53ef\u4ee5\u662f\u6839\u76ee\u5f55\uff0c\u4e5f\u53ef\u4ee5\u662f\u5b50\u76ee\u5f55\u3002\u4f8b\u5982\uff1aoss://examplebucket/ \u6216 oss://examplebucket/subdir\u3002 -ouri=oss://examplebucket/ f 4.3.0+ \u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u9ed8\u8ba4\u4f7f\u7528\u5b88\u62a4\u8fdb\u7a0b\u65b9\u5f0f\u540e\u53f0\u542f\u52a8\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -f d 4.3.0+ \u4f7f\u7528 Debug \u6a21\u5f0f\uff0c\u5728\u524d\u53f0\u542f\u52a8\u8fdb\u7a0b\u3002\u4f7f\u7528\u8be5\u53c2\u6570\u65f6\uff0c\u63a8\u8350\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u3002 -d auto_unmount 4.3.0+ fuse\u8fdb\u7a0b\u9000\u51fa\u540e\u81ea\u52a8umount\u6302\u8f7d\u8282\u70b9\u3002 -oauto_unmount ro 4.3.0+ \u53ea\u8bfb\u6302\u8f7d\uff0c\u542f\u7528\u53c2\u6570\u540e\u4e0d\u5141\u8bb8\u5199\u64cd\u4f5c\u3002 -oro direct_io 4.3.0+ \u5f00\u542f\u540e\uff0c\u8bfb\u5199\u6587\u4ef6\u53ef\u4ee5\u7ed5\u8fc7page cache\u3002 -odirect_io kernel_cache 4.3.0+ \u5f00\u542f\u540e\uff0c\u5229\u7528\u5185\u6838\u7f13\u5b58\u4f18\u5316\u8bfb\u6027\u80fd\u3002 -okernel_cache auto_cache 4.3.0+ \u9ed8\u8ba4\u5f00\u542f\uff0c\u4e0ekernel_cache \u4e8c\u9009\u4e00\uff0c\u4e0ekernel_cache\u4e0d\u540c\u7684\u662f\uff0c\u5982\u679c\u6587\u4ef6\u5927\u5c0f\u6216\u4fee\u6539\u65f6\u95f4\u53d1\u751f\u53d8\u5316\uff0c\u7f13\u5b58\u5c31\u4f1a\u5931\u6548\u3002 entry_timeout 4.3.0+ \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oentry_timeout=60 attr_timeout 4.3.0+ \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u5c5e\u6027\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -oattr_timeout=60 negative_timeout 4.3.0+ \u9ed8\u8ba4\u503c\uff0c60\u3002\u6587\u4ef6\u540d\u8bfb\u53d6\u5931\u8d25\u7f13\u5b58\u4fdd\u7559\u65f6\u95f4\uff08\u79d2\uff09\uff0c\u7528\u4e8e\u4f18\u5316\u6027\u80fd\u30020\u8868\u793a\u4e0d\u7f13\u5b58\u3002 -onegative_timeout=0 max_idle_threads 4.3.0+ \u9ed8\u8ba4\u503c\uff0c10\u3002\u5904\u7406\u5185\u6838\u56de\u8c03\u7684\u7a7a\u95f2\u7ebf\u7a0b\u6c60\u3002 -omax_idle_threads=10 xengine 4.3.0+ \u6253\u5f00\u7f13\u5b58 -oxengine pread 4.5.1+ \u9ed8\u8ba4\u4f7f\u7528\u987a\u5e8f\u8bfb\u3002\u6253\u5f00\u540e\uff0c\u4f7f\u7528\u968f\u673a\u8bfb\u4ee3\u66ff\u987a\u5e8f\u8bfb\uff0c\u9002\u7528\u4e8e\u968f\u673a\u8bfb\u8fdc\u591a\u4e8e\u987a\u5e8f\u8bfb\u7684\u573a\u666f\u3002 -opread no_symlink 4.5.1+ \u914d\u7f6e\u540e\uff0c\u5173\u95edsymlink\u529f\u80fd\u3002 -ono_symlink no_writeback 4.5.1+ \u914d\u7f6e\u540e\uff0c\u5173\u95edwriteback\u529f\u80fd\u3002 -ono_writeback no_flock 4.5.1+ \u914d\u7f6e\u540e\uff0c\u5173\u95edflock\u529f\u80fd\u3002 -ono_flock no_xattr 4.5.1+ \u914d\u7f6e\u540e\uff0c\u5173\u95edxttar\u529f\u80fd\u3002 -ono_xattr","title":"\u6302\u8f7d\u9009\u9879"},{"location":"jindofuse/jindofuse_quickstart/#_6","text":"\u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u8bf4\u660e logger.dir /tmp/bigboot-log \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF logger.verbose 0 \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint \u8bbf\u95ee JindoFS \u670d\u52a1\u7684\u5730\u5740\uff0c\u5982oss-cn-xxx.aliyuncs.com fs.oss.accessKeyId \u8bbf\u95ee JindoFS \u670d\u52a1\u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret \u8bbf\u95ee JindoFS \u670d\u52a1\u9700\u8981\u7684 accessKeySecret \u66f4\u591a\u53c2\u6570\u53ef\u89c1 \u300a\u5ba2\u6237\u7aef\u5e38\u7528\u914d\u7f6e\u300b \u3002 \u652f\u6301\u5c06 jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u4e0e\u6302\u8f7d\u9009\u9879\u4e00\u540c\u5728\u6302\u8f7d\u65f6\u6307\u5b9a\uff08\u6302\u8f7d\u65f6\u6307\u5b9a\u53c2\u6570\u7684\u4f18\u5148\u7ea7\u9ad8\u4e8e\u914d\u7f6e\u6587\u4ef6\uff09\uff0c\u5982: jindo-fuse <mount_point> -ouri=[<oss_path>] -ofs.oss.endpoint=[<your_endpoint>] -ofs.oss.accessKeyId=[<your_key_id>] -ofs.oss.accessKeySecret=[<your_key_secret>]","title":"\u914d\u7f6e\u9009\u9879"},{"location":"jindofuse/jindofuse_quickstart/#_7","text":"","title":"\u5e38\u89c1\u95ee\u9898"},{"location":"jindofuse/jindofuse_quickstart/#inputoutput-error","text":"\u4e0d\u50cf\u4f7f\u7528 JindoSDK \u8c03\u7528 API \u53ef\u4ee5\u83b7\u53d6\u66f4\u4e3a\u5177\u4f53\u7684 ErrorMsg\uff0cJindoFuse \u53ea\u80fd\u663e\u793a\u64cd\u4f5c\u7cfb\u7edf\u9884\u8bbe\u7684\u9519\u8bef\u4fe1\u606f\uff0c\u6bd4\u5982\u4ee5\u4e0b\u9519\u8bef\u5c31\u975e\u5e38\u5e38\u89c1\uff1a $ ls /mnt/oss/ ls: /mnt/oss/: Input/output error \u5982\u679c\u9700\u8981\u5b9a\u4f4d\u5177\u4f53\u7684\u9519\u8bef\u539f\u56e0\uff0c\u53ef\u4ee5\u6839\u636e JindoSDK \u914d\u7f6e\u4e2d\u7684 logger.dir\uff0c\u5728\u6307\u5b9a\u8def\u5f84\u4e0b\u7684 jindosdk.log \u6587\u4ef6\u4e2d\uff0c\u5bfb\u627e\u5177\u4f53\u7684\u9519\u8bef\u3002 \u4e0b\u9762\u5c55\u793a\u7684\u8fd9\u4e2a\u9519\u8bef\u5c31\u6765\u81ea jindosdk.log \uff0c\u6839\u636e\u62a5\u9519\u4fe1\u606f\u53ef\u89c1\uff0c\u8fd9\u4e2a\u9519\u8bef\u662f\u5e38\u89c1\u7684\u9274\u6743\u95ee\u9898\u3002 EMMDD HH:mm:ss jindofs_connectivity.cpp:13] Please check your Endpoint/Bucket/RoleArn. Failed test connectivity, operation: mkdir, errMsg: [RequestId]: 618B8183343EA53531C62B74 [HostId]: oss-cn-shanghai-internal.aliyuncs.com [ErrorMessage]: [E1010]HTTP/1.1 403 Forbidden ...","title":"Input/Output error"},{"location":"jindosdk/jindosdk_cli_options/","text":"Jindo CLI \u4f7f\u7528\u6307\u5357 \u00a7 \u7528\u6cd5\u6c47\u603b \u00a7 Usage: jindo fs [-archive [-i] [-c] <path>] [-cat <src> ...] [-count [-h] <path> ...] [-cp [-f] <src> ... <dst>] [-checksum <src> ...] [-chgrp [-R] GROUP PATH...] [-chmod [-R] <MODE[,MODE]... | OCTALMODE> PATH...] [-chown [-R] [OWNER][:[GROUP]] PATH...] [-copyFromLocal [-f] <localsrc> ... <dst>] [-copyToLocal [-f] <src> ... <localdst>] [-createSnapshot <snapshotDir> [<snapshotName>]] [-checkStoragePolicy -path <path>] [-deleteSnapshot <snapshotDir> <snapshotName>] [-du [-s] <path> ...] [-decommission -hosts <hosts>] [-formatCache] [-get [-f] <src> ... <localdst>] [-getfacl [-R] <path>] [-getfattr [-R] {-n name | -d} <path>] [-getStoragePolicy -path <path>] [-ls [-R] <path>] [-listPolicies] [-load [-meta] [-data] [-s] [-m] [-R] [-replica value] <path>] [-mkdir <path>] [-mv <src> ... <dst>] [-moveFromLocal <localsrc> ... <dst>] [-moveToLocal <src> <localdst>] [-metaDiff <path>] [-put [-f] <localsrc> ... <dst>] [-rm [-f] [-r|-R] <src> ...] [-rmdir <src> ...] [-test -[defsz] <path>] [-renameSnapshot <snapshotDir> <oldName> <newName>] [-restore [-days value] <path>] [-stat [format] <path> ...] [-setStoragePolicy -path <path> -policy <policy>] [-setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>]] [-setfattr {-n name [-v value] | -x name} <path>] [-sync [-update] [-thread thread_num] <localsrc> <dst> [-touchz <path> ...] [-truncate [-w] <length> <path> ...] [-unarchive [-i] [-a] <path>] [-uncache <path>] [-unsetStoragePolicy -path <path>] FS \u5b50\u547d\u4ee4 \u00a7 archive \u00a7 jindo fs -archive oss://<bucket>/<dir> \u5c06 OSS \u4e0a\u6587\u4ef6\u8bbe\u7f6e\u4e3a\u5f52\u6863\u7c7b\u578b\uff0c\u53ef\u9009\u53c2\u6570\uff1a \u53c2\u6570 Description -i ia \u4f4e\u9891 -c coldArchive \u51b7\u5f52\u6863 jindo fs -archive -i oss://<bucket>/<dir> jindo fs -archive -c oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS cat \u00a7 jindo fs -cat oss://<bucket>/<dir> \u663e\u793a\u6587\u672c\u5185\u5bb9\uff0c\u65e0\u53ef\u9009\u53c2\u6570 \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS count \u00a7 \u663e\u793a\u6587\u4ef6\u5927\u5c0f\u4ee5\u53ca\u6587\u4ef6\u6570\u91cf\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -h | \u663e\u793a\u6587\u4ef6\u5927\u5c0f\u5355\u4f4d| jindo fs -count -h oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS cp \u00a7 \u590d\u5236\u6587\u4ef6\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u8986\u76d6| jindo fs -cp -f oss://<bucket>/<sourcedir> oss://<bucket>/<targetdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS checksum \u00a7 \u8ba1\u7b97\u6587\u4ef6\u7684CRC32 CheckSum\u5927\u5c0f jindo fs -checksum oss://<bucket>/file \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS chgrp \u00a7 \u6539\u53d8\u6587\u4ef6\u7684\u6240\u5c5e\u7fa4\u7ec4 jindo fs -chgrp <groupname> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS chmod \u00a7 \u6539\u53d8\u6587\u4ef6\u7684\u8bbf\u95ee\u6743\u9650 jindo fs -chmod <mode> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS chown \u00a7 \u6539\u53d8\u6587\u4ef6\u7684\u6240\u6709\u4eba jindo fs -chown <username> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS copyFromLocal \u00a7 \u4ece\u672c\u5730\u590d\u5236\u6587\u4ef6\u5230oss\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u8986\u76d6| jindo fs -copyFromLocal -f <localdir> oss://<bucket>/<targetdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS copyToLocal \u00a7 \u4eceoss\u590d\u5236\u6587\u4ef6\u5230\u672c\u5730\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u8986\u76d6| jindo fs -copyToLocal -f oss://<bucket>/<targetdir> <localdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS createSnapshot \u00a7 \u521b\u5efa\u5feb\u7167\uff0c\u4f7f\u7528\u524d\u5fc5\u987b\u5148\u5141\u8bb8\u5feb\u7167\u529f\u80fd jindo fs -createSnapShot <snapshotDir> [<snapshotName>] \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS checkStoragePolicy \u00a7 \u663e\u793a\u6587\u4ef6\u7684\u5b58\u50a8\u7b56\u7565 jindo fs -checkStoragePolicy -path oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS deleteSnapshot \u00a7 \u5220\u9664\u5feb\u7167 jindo fs -deleteSnapshot oss://<bucket>/<dir> <snapshotName> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS du \u00a7 \u663e\u793a\u76ee\u5f55\u4e2d\u6240\u6709\u6587\u4ef6\u7684\u5927\u5c0f \u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -s | \u6c42\u76ee\u6807\u6587\u4ef6\u5939\u7684\u603b\u548c| | -h | \u6807\u51c6\u5355\u4f4d\u663e\u793a| jindo fs -du oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS get \u00a7 \u4e0b\u8f7d\u6587\u4ef6\u5230\u672c\u5730\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u8986\u76d6| jindo fs -get -f oss://<bucket>/<dir> <localdst> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS getfacl \u00a7 \u663e\u793a\u6587\u4ef6\u6216\u76ee\u5f55\u7684\u8bbf\u95ee\u63a7\u5236\u5217\u8868\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -R | \u9012\u5f52\u663e\u793a| jindo fs -getfacl -R oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS getfattr \u00a7 \u663e\u793a\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u7684\u6269\u5c55\u5c5e\u6027\u540d\u79f0\u548c\u503c\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -R | \u9012\u5f52\u663e\u793a| | -n | \u6309\u540d\u79f0\u663e\u793a| | -d | \u663e\u793a\u6240\u6709| jindo fs -getfattr [-R] {-n user.myAttr | -d} oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS getStoragePolicy \u00a7 \u663e\u793a\u5b58\u50a8\u7b56\u7565 jindo fs -getStoragePolicy -path oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS ls \u00a7 \u5217\u51fa\u76ee\u5f55\u4e0b\u6587\u4ef6\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -R | \u9012\u5f52\u663e\u793a| jindo fs -ls [-R] oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS listPolicies \u00a7 \u663e\u793a\u652f\u6301\u7684\u5b58\u50a8\u7b56\u7565 jindo fs -listPolicies \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS load \u00a7 \u6570\u636e\u7f13\u5b58\u547d\u4ee4\u53ef\u4ee5\u5907\u4efd\u5bf9\u5e94\u8def\u5f84\u7684\u6570\u636e\u81f3\u672c\u96c6\u7fa4\u7684\u78c1\u76d8\uff0c\u4ee5\u4fbf\u4e8e\u540e\u7eed\u53ef\u4ee5\u8bfb\u53d6\u672c\u5730\u6570\u636e\uff0c\u65e0\u9700\u8bfb\u53d6OSS\u7b49\u540e\u7aef\u4e0a\u7684\u6570\u636e\uff0c\u53ef\u9009\u53c2\u6570\u6709\u3002 | \u53c2\u6570 | \u8bf4\u660e | | --- | --- | | -s | \u8868\u793a\u7f13\u5b58\u8fc7\u7a0b\u540c\u6b65\u6267\u884c\uff0c\u6253\u5370\u8fdb\u5ea6\u548c\u6267\u884c\u4fe1\u606f\uff0c\u63a8\u8350\u5f00\u542f | | -replica | \u7f13\u5b58\u526f\u672c\u6570\u91cf\uff0c\u9ed8\u8ba4\u7f13\u5b581\u4e2a\u526f\u672c | | -R | \u9012\u5f52\u7f13\u5b58\u6587\u4ef6\uff0c\u5f53 path \u662f\u6587\u4ef6\u5939\u65f6\u5f00\u542f | jindo fs -load [-meta] [-data] [-s] [-m] [-R] [-replica value] oss://<bucket>/<dir> mkdir \u00a7 \u521b\u5efa\u6587\u4ef6\u5939 jindo fs -mkdir oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS mv \u00a7 \u79fb\u52a8\u6587\u4ef6 jindo fs -mv oss://<bucket>/<srcdir> oss://<bucket>/<dstdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS moveFromLocal \u00a7 \u5c06\u6587\u4ef6\u4ece\u672c\u5730\u79fb\u52a8\u5230\u8fdc\u7a0b\u8def\u5f84\u4e0a\uff0c\uff0c\u79fb\u52a8\u5b8c\u6210\u540e\u4f1a\u5220\u9664\u672c\u5730\u6587\u4ef6 jindo fs -moveFromLocal <localsrc> oss://<bucket>/<dstdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS moveToLocal \u00a7 \u5c06\u6587\u4ef6\u4ece\u8fdc\u7a0b\u8def\u5f84\u4e0a\u79fb\u52a8\u5230\u672c\u5730\uff0c\u79fb\u52a8\u5b8c\u6210\u540e\u4f1aOSS\u4e0a\u6587\u4ef6 jindo fs -moveToLocal oss://<bucket>/<srcdir> <localdst> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS metaDiff \u00a7 \u4f7f\u7528\u7f13\u5b58\u52a0\u901f\u7cfb\u7edf\u540e\uff0c\u663e\u793a\u672c\u5730\u5143\u6570\u636e\u548c\u8fdc\u7aef\u7684\u540c\u6b65\u60c5\u51b5 jindo fs -metaDiff oss://<bucket>/<srcdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS put \u00a7 \u5c06\u672c\u5730\u6587\u4ef6\u590d\u5236\u5230\u8fdc\u7a0b\u76ee\u5f55\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u8986\u76d6| jindo fs -put -f <localsrc> oss://<bucket>/<targetdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS rm \u00a7 \u5220\u9664\u8fdc\u7a0b\u8def\u5f84\u4e0a\u7684\u6587\u4ef6\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u5220\u9664| | -r/-rmr/-R | \u9012\u5f52\u5220\u9664| jindo fs -rm -f oss://<bucket>/<dir> jindo fs -rm -R oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS rmdir \u00a7 \u5220\u9664\u6587\u4ef6\u5939 jindo fs -rmdir oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS test \u00a7 \u57fa\u672c\u529f\u80fd\u68c0\u6d4b\uff1a | \u53c2\u6570 | Description | | --- | --- | | -d | \u662f\u5426\u662f\u6587\u4ef6\u5939| | -e | \u662f\u5426\u5b58\u5728\u8def\u5f84| | -f | \u662f\u5426\u662f\u6587\u4ef6| | -s | \u662f\u5426\u4e3a\u7a7a\u6587\u4ef6\u5939| | -z | \u6587\u4ef6\u957f\u5ea6\u662f\u5426\u4e3a0| jindo fs -test -d oss://<bucket>/<dir>/file \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS renameSnapshot \u00a7 \u4e3a\u5feb\u7167\u91cd\u547d\u540d jindo fs -renameSnapshot oss://<bucket>/<dir> <oldName> <newName> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS restore \u00a7 \u5c06\u6587\u4ef6\u4ece\u5f52\u6863\u72b6\u6001\u6062\u590d\uff0c\u51b7\u5f52\u6863\u6587\u4ef6\u548c\u5f52\u6863\u6587\u4ef6\u5fc5\u987b\u6062\u590d\u540e\u624d\u80fdunarchive\uff0c\u5fc5\u586b\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -days | \u89e3\u51bb\u7684\u65f6\u957f| jindo fs -restore -days value oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS stat \u00a7 \u663e\u793a\u6587\u4ef6\u72b6\u6001 jindo fs -stat oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS setStoragePolicy \u00a7 \u8bbe\u7f6e\u5b58\u50a8\u7b56\u7565 jindo fs -setStoragePolicy -path oss://<bucket>/<dir> -policy <policy> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS setfacl \u00a7 \u8bbe\u7f6e\u6587\u4ef6\u8bbf\u95ee\u7b56\u7565\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -b | \u5220\u9664\u6240\u6709\u8bbf\u95ee\u7b56\u7565| | -k | \u5220\u9664\u9ed8\u8ba4\u8bbf\u95ee\u7b56\u7565| | -m | \u6309\u540d\u79f0\u4fee\u6539\u8bbf\u95ee\u7b56\u7565| | -x | \u6309\u540d\u79f0\u5220\u9664\u8bbf\u95ee\u7b56\u7565| | -set | \u8bbe\u5b9a\u8bbf\u95ee\u7b56\u7565| jindo fs -setfacl -b oss://<bucket>/<dir> jindo fs -setfacl -k oss://<bucket>/<dir> jindo fs -setfacl -m user:<username>:<acl> oss://<bucket>/<dir> jindo fs -setfacl -x user:<username> oss://<bucket>/<dir> jindo fs -setfacl -set user::<acl> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS setfattr \u00a7 \u8bbe\u7f6e\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u7684\u6269\u5c55\u5c5e\u6027\u540d\u79f0\u548c\u503c\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -n | \u6309\u540d\u79f0\u8bbe\u7f6e| | -v | \u8bbe\u7f6e\u5c5e\u6027\u503c| | -x | \u6309\u540d\u79f0\u5220\u9664| jindo fs -setfattr {-n name [-v value] | -x name} oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS sync \u00a7 \u5c06\u672c\u5730\u8def\u5f84\u3001hdfs\u8def\u5f84\u3001oss\u8def\u5f84\u6216oss-hdfs\u8def\u5f84\u5230\u8fdc\u7a0b\u8def\u5f84\uff0c\u4f7f\u7528update\u53c2\u6570\u5219\u4f1a\u8df3\u8fc7\u76ee\u7684\u8def\u5f84\u4e2d\u4fee\u6539\u65f6\u95f4\u4e0d\u665a\u4e8e\u6e90\u8def\u5f84\u4e2d\u540c\u540d\u6587\u4ef6\u7684\u6587\u4ef6\u3002\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -update | \u65ad\u70b9\u7eed\u4f20 | jindo fs -sync [-update] -thread thread_num <localsrc> oss://<bucket>/<dir> jindo fs -sync [-update] -thread thread_num oss://<bucket>/<dir> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS touchz \u00a7 \u751f\u6210\u4e00\u4e2a\u5927\u5c0f\u4e3a0\u7684\u6587\u4ef6 jindo fs -touchz oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS truncate \u00a7 \u5c06\u6587\u4ef6\u88c1\u526a\u5230\u6307\u5b9a\u5927\u5c0f jindo fs -truncate <length> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS unarchive \u00a7 \u5c06\u6587\u4ef6\u6062\u590d\u4e3a\u6807\u51c6\u6587\u4ef6\uff0cAR\u548cCOLD\u7c7b\u578b\u6587\u4ef6\u9700\u8981\u5148restore jindo fs -unarchive oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS uncache \u00a7 \u5220\u9664\u7f13\u5b58\u7cfb\u7edf\u7684\u7f13\u5b58\u6587\u4ef6 jindo fs -uncache oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS unsetStoragePolicy \u00a7 \u53d6\u6d88\u5b58\u50a8\u7b56\u7565 jindo fs -unsetStoragePolicy -path oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"JindoCli \u53c2\u6570"},{"location":"jindosdk/jindosdk_cli_options/#jindo-cli","text":"","title":"Jindo CLI \u4f7f\u7528\u6307\u5357"},{"location":"jindosdk/jindosdk_cli_options/#_1","text":"Usage: jindo fs [-archive [-i] [-c] <path>] [-cat <src> ...] [-count [-h] <path> ...] [-cp [-f] <src> ... <dst>] [-checksum <src> ...] [-chgrp [-R] GROUP PATH...] [-chmod [-R] <MODE[,MODE]... | OCTALMODE> PATH...] [-chown [-R] [OWNER][:[GROUP]] PATH...] [-copyFromLocal [-f] <localsrc> ... <dst>] [-copyToLocal [-f] <src> ... <localdst>] [-createSnapshot <snapshotDir> [<snapshotName>]] [-checkStoragePolicy -path <path>] [-deleteSnapshot <snapshotDir> <snapshotName>] [-du [-s] <path> ...] [-decommission -hosts <hosts>] [-formatCache] [-get [-f] <src> ... <localdst>] [-getfacl [-R] <path>] [-getfattr [-R] {-n name | -d} <path>] [-getStoragePolicy -path <path>] [-ls [-R] <path>] [-listPolicies] [-load [-meta] [-data] [-s] [-m] [-R] [-replica value] <path>] [-mkdir <path>] [-mv <src> ... <dst>] [-moveFromLocal <localsrc> ... <dst>] [-moveToLocal <src> <localdst>] [-metaDiff <path>] [-put [-f] <localsrc> ... <dst>] [-rm [-f] [-r|-R] <src> ...] [-rmdir <src> ...] [-test -[defsz] <path>] [-renameSnapshot <snapshotDir> <oldName> <newName>] [-restore [-days value] <path>] [-stat [format] <path> ...] [-setStoragePolicy -path <path> -policy <policy>] [-setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>]] [-setfattr {-n name [-v value] | -x name} <path>] [-sync [-update] [-thread thread_num] <localsrc> <dst> [-touchz <path> ...] [-truncate [-w] <length> <path> ...] [-unarchive [-i] [-a] <path>] [-uncache <path>] [-unsetStoragePolicy -path <path>]","title":"\u7528\u6cd5\u6c47\u603b"},{"location":"jindosdk/jindosdk_cli_options/#fs","text":"","title":"FS \u5b50\u547d\u4ee4"},{"location":"jindosdk/jindosdk_cli_options/#archive","text":"jindo fs -archive oss://<bucket>/<dir> \u5c06 OSS \u4e0a\u6587\u4ef6\u8bbe\u7f6e\u4e3a\u5f52\u6863\u7c7b\u578b\uff0c\u53ef\u9009\u53c2\u6570\uff1a \u53c2\u6570 Description -i ia \u4f4e\u9891 -c coldArchive \u51b7\u5f52\u6863 jindo fs -archive -i oss://<bucket>/<dir> jindo fs -archive -c oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS","title":"archive"},{"location":"jindosdk/jindosdk_cli_options/#cat","text":"jindo fs -cat oss://<bucket>/<dir> \u663e\u793a\u6587\u672c\u5185\u5bb9\uff0c\u65e0\u53ef\u9009\u53c2\u6570 \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"cat"},{"location":"jindosdk/jindosdk_cli_options/#count","text":"\u663e\u793a\u6587\u4ef6\u5927\u5c0f\u4ee5\u53ca\u6587\u4ef6\u6570\u91cf\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -h | \u663e\u793a\u6587\u4ef6\u5927\u5c0f\u5355\u4f4d| jindo fs -count -h oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"count"},{"location":"jindosdk/jindosdk_cli_options/#cp","text":"\u590d\u5236\u6587\u4ef6\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u8986\u76d6| jindo fs -cp -f oss://<bucket>/<sourcedir> oss://<bucket>/<targetdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"cp"},{"location":"jindosdk/jindosdk_cli_options/#checksum","text":"\u8ba1\u7b97\u6587\u4ef6\u7684CRC32 CheckSum\u5927\u5c0f jindo fs -checksum oss://<bucket>/file \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"checksum"},{"location":"jindosdk/jindosdk_cli_options/#chgrp","text":"\u6539\u53d8\u6587\u4ef6\u7684\u6240\u5c5e\u7fa4\u7ec4 jindo fs -chgrp <groupname> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"chgrp"},{"location":"jindosdk/jindosdk_cli_options/#chmod","text":"\u6539\u53d8\u6587\u4ef6\u7684\u8bbf\u95ee\u6743\u9650 jindo fs -chmod <mode> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"chmod"},{"location":"jindosdk/jindosdk_cli_options/#chown","text":"\u6539\u53d8\u6587\u4ef6\u7684\u6240\u6709\u4eba jindo fs -chown <username> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"chown"},{"location":"jindosdk/jindosdk_cli_options/#copyfromlocal","text":"\u4ece\u672c\u5730\u590d\u5236\u6587\u4ef6\u5230oss\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u8986\u76d6| jindo fs -copyFromLocal -f <localdir> oss://<bucket>/<targetdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"copyFromLocal"},{"location":"jindosdk/jindosdk_cli_options/#copytolocal","text":"\u4eceoss\u590d\u5236\u6587\u4ef6\u5230\u672c\u5730\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u8986\u76d6| jindo fs -copyToLocal -f oss://<bucket>/<targetdir> <localdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"copyToLocal"},{"location":"jindosdk/jindosdk_cli_options/#createsnapshot","text":"\u521b\u5efa\u5feb\u7167\uff0c\u4f7f\u7528\u524d\u5fc5\u987b\u5148\u5141\u8bb8\u5feb\u7167\u529f\u80fd jindo fs -createSnapShot <snapshotDir> [<snapshotName>] \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"createSnapshot"},{"location":"jindosdk/jindosdk_cli_options/#checkstoragepolicy","text":"\u663e\u793a\u6587\u4ef6\u7684\u5b58\u50a8\u7b56\u7565 jindo fs -checkStoragePolicy -path oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"checkStoragePolicy"},{"location":"jindosdk/jindosdk_cli_options/#deletesnapshot","text":"\u5220\u9664\u5feb\u7167 jindo fs -deleteSnapshot oss://<bucket>/<dir> <snapshotName> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"deleteSnapshot"},{"location":"jindosdk/jindosdk_cli_options/#du","text":"\u663e\u793a\u76ee\u5f55\u4e2d\u6240\u6709\u6587\u4ef6\u7684\u5927\u5c0f \u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -s | \u6c42\u76ee\u6807\u6587\u4ef6\u5939\u7684\u603b\u548c| | -h | \u6807\u51c6\u5355\u4f4d\u663e\u793a| jindo fs -du oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"du"},{"location":"jindosdk/jindosdk_cli_options/#get","text":"\u4e0b\u8f7d\u6587\u4ef6\u5230\u672c\u5730\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u8986\u76d6| jindo fs -get -f oss://<bucket>/<dir> <localdst> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"get"},{"location":"jindosdk/jindosdk_cli_options/#getfacl","text":"\u663e\u793a\u6587\u4ef6\u6216\u76ee\u5f55\u7684\u8bbf\u95ee\u63a7\u5236\u5217\u8868\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -R | \u9012\u5f52\u663e\u793a| jindo fs -getfacl -R oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"getfacl"},{"location":"jindosdk/jindosdk_cli_options/#getfattr","text":"\u663e\u793a\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u7684\u6269\u5c55\u5c5e\u6027\u540d\u79f0\u548c\u503c\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -R | \u9012\u5f52\u663e\u793a| | -n | \u6309\u540d\u79f0\u663e\u793a| | -d | \u663e\u793a\u6240\u6709| jindo fs -getfattr [-R] {-n user.myAttr | -d} oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"getfattr"},{"location":"jindosdk/jindosdk_cli_options/#getstoragepolicy","text":"\u663e\u793a\u5b58\u50a8\u7b56\u7565 jindo fs -getStoragePolicy -path oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"getStoragePolicy"},{"location":"jindosdk/jindosdk_cli_options/#ls","text":"\u5217\u51fa\u76ee\u5f55\u4e0b\u6587\u4ef6\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -R | \u9012\u5f52\u663e\u793a| jindo fs -ls [-R] oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"ls"},{"location":"jindosdk/jindosdk_cli_options/#listpolicies","text":"\u663e\u793a\u652f\u6301\u7684\u5b58\u50a8\u7b56\u7565 jindo fs -listPolicies \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"listPolicies"},{"location":"jindosdk/jindosdk_cli_options/#load","text":"\u6570\u636e\u7f13\u5b58\u547d\u4ee4\u53ef\u4ee5\u5907\u4efd\u5bf9\u5e94\u8def\u5f84\u7684\u6570\u636e\u81f3\u672c\u96c6\u7fa4\u7684\u78c1\u76d8\uff0c\u4ee5\u4fbf\u4e8e\u540e\u7eed\u53ef\u4ee5\u8bfb\u53d6\u672c\u5730\u6570\u636e\uff0c\u65e0\u9700\u8bfb\u53d6OSS\u7b49\u540e\u7aef\u4e0a\u7684\u6570\u636e\uff0c\u53ef\u9009\u53c2\u6570\u6709\u3002 | \u53c2\u6570 | \u8bf4\u660e | | --- | --- | | -s | \u8868\u793a\u7f13\u5b58\u8fc7\u7a0b\u540c\u6b65\u6267\u884c\uff0c\u6253\u5370\u8fdb\u5ea6\u548c\u6267\u884c\u4fe1\u606f\uff0c\u63a8\u8350\u5f00\u542f | | -replica | \u7f13\u5b58\u526f\u672c\u6570\u91cf\uff0c\u9ed8\u8ba4\u7f13\u5b581\u4e2a\u526f\u672c | | -R | \u9012\u5f52\u7f13\u5b58\u6587\u4ef6\uff0c\u5f53 path \u662f\u6587\u4ef6\u5939\u65f6\u5f00\u542f | jindo fs -load [-meta] [-data] [-s] [-m] [-R] [-replica value] oss://<bucket>/<dir>","title":"load"},{"location":"jindosdk/jindosdk_cli_options/#mkdir","text":"\u521b\u5efa\u6587\u4ef6\u5939 jindo fs -mkdir oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"mkdir"},{"location":"jindosdk/jindosdk_cli_options/#mv","text":"\u79fb\u52a8\u6587\u4ef6 jindo fs -mv oss://<bucket>/<srcdir> oss://<bucket>/<dstdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"mv"},{"location":"jindosdk/jindosdk_cli_options/#movefromlocal","text":"\u5c06\u6587\u4ef6\u4ece\u672c\u5730\u79fb\u52a8\u5230\u8fdc\u7a0b\u8def\u5f84\u4e0a\uff0c\uff0c\u79fb\u52a8\u5b8c\u6210\u540e\u4f1a\u5220\u9664\u672c\u5730\u6587\u4ef6 jindo fs -moveFromLocal <localsrc> oss://<bucket>/<dstdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"moveFromLocal"},{"location":"jindosdk/jindosdk_cli_options/#movetolocal","text":"\u5c06\u6587\u4ef6\u4ece\u8fdc\u7a0b\u8def\u5f84\u4e0a\u79fb\u52a8\u5230\u672c\u5730\uff0c\u79fb\u52a8\u5b8c\u6210\u540e\u4f1aOSS\u4e0a\u6587\u4ef6 jindo fs -moveToLocal oss://<bucket>/<srcdir> <localdst> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"moveToLocal"},{"location":"jindosdk/jindosdk_cli_options/#metadiff","text":"\u4f7f\u7528\u7f13\u5b58\u52a0\u901f\u7cfb\u7edf\u540e\uff0c\u663e\u793a\u672c\u5730\u5143\u6570\u636e\u548c\u8fdc\u7aef\u7684\u540c\u6b65\u60c5\u51b5 jindo fs -metaDiff oss://<bucket>/<srcdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"metaDiff"},{"location":"jindosdk/jindosdk_cli_options/#put","text":"\u5c06\u672c\u5730\u6587\u4ef6\u590d\u5236\u5230\u8fdc\u7a0b\u76ee\u5f55\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u8986\u76d6| jindo fs -put -f <localsrc> oss://<bucket>/<targetdir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"put"},{"location":"jindosdk/jindosdk_cli_options/#rm","text":"\u5220\u9664\u8fdc\u7a0b\u8def\u5f84\u4e0a\u7684\u6587\u4ef6\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -f | \u5f3a\u5236\u5220\u9664| | -r/-rmr/-R | \u9012\u5f52\u5220\u9664| jindo fs -rm -f oss://<bucket>/<dir> jindo fs -rm -R oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"rm"},{"location":"jindosdk/jindosdk_cli_options/#rmdir","text":"\u5220\u9664\u6587\u4ef6\u5939 jindo fs -rmdir oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"rmdir"},{"location":"jindosdk/jindosdk_cli_options/#test","text":"\u57fa\u672c\u529f\u80fd\u68c0\u6d4b\uff1a | \u53c2\u6570 | Description | | --- | --- | | -d | \u662f\u5426\u662f\u6587\u4ef6\u5939| | -e | \u662f\u5426\u5b58\u5728\u8def\u5f84| | -f | \u662f\u5426\u662f\u6587\u4ef6| | -s | \u662f\u5426\u4e3a\u7a7a\u6587\u4ef6\u5939| | -z | \u6587\u4ef6\u957f\u5ea6\u662f\u5426\u4e3a0| jindo fs -test -d oss://<bucket>/<dir>/file \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"test"},{"location":"jindosdk/jindosdk_cli_options/#renamesnapshot","text":"\u4e3a\u5feb\u7167\u91cd\u547d\u540d jindo fs -renameSnapshot oss://<bucket>/<dir> <oldName> <newName> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"renameSnapshot"},{"location":"jindosdk/jindosdk_cli_options/#restore","text":"\u5c06\u6587\u4ef6\u4ece\u5f52\u6863\u72b6\u6001\u6062\u590d\uff0c\u51b7\u5f52\u6863\u6587\u4ef6\u548c\u5f52\u6863\u6587\u4ef6\u5fc5\u987b\u6062\u590d\u540e\u624d\u80fdunarchive\uff0c\u5fc5\u586b\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -days | \u89e3\u51bb\u7684\u65f6\u957f| jindo fs -restore -days value oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"restore"},{"location":"jindosdk/jindosdk_cli_options/#stat","text":"\u663e\u793a\u6587\u4ef6\u72b6\u6001 jindo fs -stat oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"stat"},{"location":"jindosdk/jindosdk_cli_options/#setstoragepolicy","text":"\u8bbe\u7f6e\u5b58\u50a8\u7b56\u7565 jindo fs -setStoragePolicy -path oss://<bucket>/<dir> -policy <policy> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"setStoragePolicy"},{"location":"jindosdk/jindosdk_cli_options/#setfacl","text":"\u8bbe\u7f6e\u6587\u4ef6\u8bbf\u95ee\u7b56\u7565\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -b | \u5220\u9664\u6240\u6709\u8bbf\u95ee\u7b56\u7565| | -k | \u5220\u9664\u9ed8\u8ba4\u8bbf\u95ee\u7b56\u7565| | -m | \u6309\u540d\u79f0\u4fee\u6539\u8bbf\u95ee\u7b56\u7565| | -x | \u6309\u540d\u79f0\u5220\u9664\u8bbf\u95ee\u7b56\u7565| | -set | \u8bbe\u5b9a\u8bbf\u95ee\u7b56\u7565| jindo fs -setfacl -b oss://<bucket>/<dir> jindo fs -setfacl -k oss://<bucket>/<dir> jindo fs -setfacl -m user:<username>:<acl> oss://<bucket>/<dir> jindo fs -setfacl -x user:<username> oss://<bucket>/<dir> jindo fs -setfacl -set user::<acl> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"setfacl"},{"location":"jindosdk/jindosdk_cli_options/#setfattr","text":"\u8bbe\u7f6e\u6587\u4ef6\u6216\u8005\u76ee\u5f55\u7684\u6269\u5c55\u5c5e\u6027\u540d\u79f0\u548c\u503c\uff0c\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -n | \u6309\u540d\u79f0\u8bbe\u7f6e| | -v | \u8bbe\u7f6e\u5c5e\u6027\u503c| | -x | \u6309\u540d\u79f0\u5220\u9664| jindo fs -setfattr {-n name [-v value] | -x name} oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"setfattr"},{"location":"jindosdk/jindosdk_cli_options/#sync","text":"\u5c06\u672c\u5730\u8def\u5f84\u3001hdfs\u8def\u5f84\u3001oss\u8def\u5f84\u6216oss-hdfs\u8def\u5f84\u5230\u8fdc\u7a0b\u8def\u5f84\uff0c\u4f7f\u7528update\u53c2\u6570\u5219\u4f1a\u8df3\u8fc7\u76ee\u7684\u8def\u5f84\u4e2d\u4fee\u6539\u65f6\u95f4\u4e0d\u665a\u4e8e\u6e90\u8def\u5f84\u4e2d\u540c\u540d\u6587\u4ef6\u7684\u6587\u4ef6\u3002\u53ef\u9009\u53c2\u6570\uff1a | \u53c2\u6570 | Description | | --- | --- | | -update | \u65ad\u70b9\u7eed\u4f20 | jindo fs -sync [-update] -thread thread_num <localsrc> oss://<bucket>/<dir> jindo fs -sync [-update] -thread thread_num oss://<bucket>/<dir> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"sync"},{"location":"jindosdk/jindosdk_cli_options/#touchz","text":"\u751f\u6210\u4e00\u4e2a\u5927\u5c0f\u4e3a0\u7684\u6587\u4ef6 jindo fs -touchz oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"touchz"},{"location":"jindosdk/jindosdk_cli_options/#truncate","text":"\u5c06\u6587\u4ef6\u88c1\u526a\u5230\u6307\u5b9a\u5927\u5c0f jindo fs -truncate <length> oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"truncate"},{"location":"jindosdk/jindosdk_cli_options/#unarchive","text":"\u5c06\u6587\u4ef6\u6062\u590d\u4e3a\u6807\u51c6\u6587\u4ef6\uff0cAR\u548cCOLD\u7c7b\u578b\u6587\u4ef6\u9700\u8981\u5148restore jindo fs -unarchive oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"unarchive"},{"location":"jindosdk/jindosdk_cli_options/#uncache","text":"\u5220\u9664\u7f13\u5b58\u7cfb\u7edf\u7684\u7f13\u5b58\u6587\u4ef6 jindo fs -uncache oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1a\u6807\u51c6OSS\u3001OSS-HDFS","title":"uncache"},{"location":"jindosdk/jindosdk_cli_options/#unsetstoragepolicy","text":"\u53d6\u6d88\u5b58\u50a8\u7b56\u7565 jindo fs -unsetStoragePolicy -path oss://<bucket>/<dir> \u9002\u7528\u8303\u56f4\uff1aOSS-HDFS","title":"unsetStoragePolicy"},{"location":"jindosdk/jindosdk_configuration/","text":"\u5ba2\u6237\u7aef\u5e38\u7528\u914d\u7f6e \u00a7 OSS/OSS-HDFS \u00a7 \u65e5\u5fd7\u914d\u7f6e\u9879 \u00a7 \u914d\u7f6e\u9879 \u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e \u7248\u672c logger.dir \u5b57\u7b26\u4e32 /tmp/bigboot-log \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa\uff0clogger.consolelogger \u548c logger.jnilogger \u5747\u4e3a false \u65f6\u751f\u6548\u3002 4.3.0+ logger.sync \u5e03\u5c14\u503c false \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa 4.3.0+ logger.consolelogger \u5e03\u5c14\u503c false \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef 4.3.0+ logger.jnilogger \u5e03\u5c14\u503c false \u6253\u5370\u65e5\u5fd7\u5230log4j 4.3.0+ logger.level \u6574\u578b 2 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 4.3.0+ logger.verbose \u6574\u578b 0 \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa 4.3.0+ logger.cleaner.enable \u5e03\u5c14\u503c false \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 4.3.0+ Java SDK \u8bbe\u7f6e log4j.properties \u00a7 log4j.logger.com.aliyun.jindodata=INFO log4j.logger.com.aliyun.jindodata.common.FsStats=INFO I/O \u76f8\u5173\u914d\u7f6e\u9879 \u00a7 \u914d\u7f6e\u9879 \u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e \u7248\u672c fs.oss.tmp.data.dirs \u5b57\u7b26\u4e32 /tmp/ \u5ba2\u6237\u7aef\u5199\u5165\u65f6\u7684\u4e34\u65f6\u6587\u4ef6\u76ee\u5f55\uff0c\u53ef\u914d\u7f6e\u591a\u4e2a\uff08\u9017\u53f7\u9694\u5f00\uff09\uff0c\u4f1a\u8f6e\u6d41\u5199\u5165\uff0c\u591a\u7528\u6237\u73af\u5883\u9700\u914d\u7f6e\u53ef\u8bfb\u5199\u6743\u9650 4.3.0+ fs.oss.tmp.data.cleaner.enable \u5e03\u5c14\u503c true \u4e34\u65f6\u6587\u4ef6\u81ea\u6e05\u7406\u670d\u52a1 4.3.0+ fs.oss.retry.count \u6574\u578b 5 \u8bbf\u95ee OSS/OSS-HDFS \u5931\u8d25\u91cd\u8bd5\u6b21\u6570 4.3.0+ fs.oss.retry.interval.millisecond \u6574\u578b 500 \u8bbf\u95ee OSS/OSS-HDFS \u5931\u8d25\u91cd\u8bd5\u95f4\u9694\uff08\u6beb\u79d2\uff09 4.3.0+ fs.oss.timeout.millisecond \u6574\u578b 30000 \u8bf7\u6c42 OSS/OSS-HDFS \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 4.3.0+ fs.oss.connection.timeout.millisecond \u6574\u578b 3000 \u8fde\u63a5 OSS/OSS-HDFS \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 4.3.0+ fs.oss.max.connections.per.host \u6574\u578b 100 \u8fde\u63a5 OSS/OSS-HDFS \u7684\u8fde\u63a5\u6c60\u5bf9\u6bcf\u4e2ahost\u7684\u6700\u5927\u8fde\u63a5\u6570\uff08\u8d85\u8fc7\u9608\u503c\u5916\u7684\u8fde\u63a5\u4f1a\u4f7f\u7528\u77ed\u8fde\u63a5\uff09 4.3.0+ fs.oss.upload.thread.concurrency \u6574\u578b MAX(cpu\u6838\u6570,16) \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u5e76\u53d1\u4e0a\u4f20\u7ebf\u7a0b\u6570 4.3.0+(deprecated) fs.oss.upload.queue.size \u6574\u578b MAX(cpu\u6838\u6570,16) \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u961f\u5217\u5927\u5c0f 4.3.0+(deprecated) fs.oss.upload.max.pending.tasks.per.stream \u6574\u578b 10 \u5355\u4e2a\u6587\u4ef6 OSS/OSS-HDFS \u6700\u5927\u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u6570 4.3.0+(deprecated) fs.oss.download.thread.concurrency \u6574\u578b MAX(cpu\u6838\u6570,16) \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u6700\u5927\u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u6570 4.3.0+(deprecated) fs.oss.read.readahead.max.buffer.count \u6574\u578b 48 \u6700\u5927\u540c\u65f6\u9884\u8bfb OSS/OSS-HDFS \u7684 buffer \u4e2a\u6570 4.3.0+(deprecated) fs.oss.read.buffer.size \u6574\u578b 1048576 OSS/OSS-HDFS \u8bfb\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 4.3.0+ fs.oss.write.buffer.size \u6574\u578b 1048576 OSS/OSS-HDFS \u5199\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 4.3.0+ fs.oss.flush.interval.millisecond \u6574\u578b -1 OSS/OSS-HDFS \u5237\u65b0\u7f13\u51b2\u533a\u95f4\u9694\uff08\u6beb\u79d2\uff09\uff0c\u5c0f\u4e8e 0 \u65f6\u4e0d\u751f\u6548 4.3.0+ fs.oss.blocklet.size.mb \u6574\u578b 8 OSS \u5206\u5757\u4e0a\u4f20\u65f6\u7684\u5757\u5927\u5c0f\uff08MB\uff09\u7531\u4e8e\u5206\u5757\u6570\u91cf\u6700\u591a\u4e3a10000\u5757\uff0c\u56e0\u6b64\u9ed8\u8ba4\u5199\u5165\u6587\u4ef6\u4e0d\u80fd\u8d85\u8fc780GB\u3002\u5982\u679c\u6709\u4e2a\u522b\u6587\u4ef6\u8d85\u8fc780G\uff0c\u5efa\u8bae\u6839\u636e\u6587\u4ef6\u5927\u5c0f\u5355\u72ec\u8c03\u5927\u672c\u914d\u7f6e\uff0c\u5e76\u540c\u65f6\u8c03\u5927\u8bf7\u6c42 oss \u7684\u8d85\u65f6\u65f6\u95f4\u3002\u5982\u6587\u4ef6\u5927\u5c0f\u672a\u77e5\uff0c\u6216\u8005\u8fdc\u8fdc\u8d85\u8fc780G\uff08\u5982\u8d85\u8fc7160G\uff09\uff0c\u5efa\u8bae\u8003\u8651\u4f7f\u7528 OSS-HDFS\uff08\u65e0\u6587\u4ef6\u5927\u5c0f\u9650\u5236\uff09 4.5.2+ fs.oss.checksum.crc64.enable \u5e03\u5c14\u503c true \u6587\u4ef6\u7ea7\u522b crc64 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u76ee\u524d\u5bf9\u5199OSS-HDFS\u6027\u80fd\u6709\u8f83\u5927\u5f71\u54cd\uff0c\u6027\u80fd\u4f18\u5148\u573a\u666f\u53ef\u4ee5\u8003\u8651\u5173\u95ed\u3002 4.6.0+ fs.oss.checksum.md5.enable \u5e03\u5c14\u503c false \u8bf7\u6c42\u7ea7\u522b md5 \u5b8c\u6574\u6027\u6821\u9a8c 4.6.0+ fs.oss.read.readahead.prefetcher.version \u5b57\u7b26\u4e32 default \u53ef\u9009\u503c\uff1a legacy \u539f\u9884\u8bfb\u7b97\u6cd5, default \u65b0\u9884\u8bfb\u7b97\u6cd5\u3002\u65b0\u9884\u8bfb\u7b97\u6cd5\u53ef\u80fd\u4f7f\u7528\u66f4\u591a\u5185\u5b58\u3002\u82e5\u914d\u7f6e\u4e86\u65b0\u9884\u8bfb\u7b97\u6cd5\u540e\u53d1\u751f\u6027\u80fd\u4e0b\u964d\uff0c\u53ef\u80fd\u662f\u7531\u4e8e\u5185\u5b58\u6c60\u5bb9\u91cf\u4e0d\u8db3\u5bfc\u81f4\u9884\u8bfb\u7684\u5757\u5728\u88ab\u8bbf\u95ee\u5230\u4e4b\u524d\u5c31\u88ab\u63d0\u524d\u9010\u51fa\u3002\u4e3a\u4e86\u907f\u514d\u8be5\u60c5\u51b5\u53d1\u751f\uff0c\u53ef\u4ee5\u8003\u8651\u7f29\u51cf\u6700\u5927\u9884\u8bfb\u957f\u5ea6\uff0c\u6216\u5141\u8bb8\u9884\u8bfb\u4f7f\u7528\u66f4\u591a\u5185\u5b58\u3002 6.2.0+ fs.oss.read.readahead.prefetch.size.max \u6574\u578b 268435456 \u9884\u8bfb\u6700\u5927\u957f\u5ea6\uff08\u5355\u4f4d\uff1abyte\uff09 6.2.0+ fs.oss.signer.version \u6574\u578b 0 \u7b7e\u540d\u7b97\u6cd5\u7248\u672c\uff0c\u53ef\u9009\u503c[0\uff0c1\uff0c4]\u3002\u63a8\u8350\u4f7f\u75280\uff0c\u8868\u793a\u9ed8\u8ba4\u7b97\u6cd5\u7248\u672c\uff0c6.3.0\u7248\u672c\u4e4b\u540e\u9ed8\u8ba4\u4f7f\u7528 V4 \u7b7e\u540d\u7b97\u6cd5\u63d0\u9ad8\u8bbf\u95ee OSS/OSS-HDFS\uff0c\u82e5\u7279\u6b8a\u573a\u666f\u5e0c\u671b\u6307\u5b9a\u7b7e\u540d\u7b97\u6cd5\u7248\u672c\uff0c\u8bf7\u6307\u5b9a 1 \u6216 4\u3002 6.3.0+ fs.oss.io.timeout.millisecond \u6574\u578b 90000 \u8bf7\u6c42 OSS/OSS-HDFS \u8bfb\u5199\u64cd\u4f5c\u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 6.6.0+ nextarch fs.oss.async.executor.number \u6574\u578b cpu\u6838\u6570 \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u5f02\u6b65io\u7ebf\u7a0b\u6570 6.6.0+ nextarch fs.oss.upload.async.concurrency \u6574\u578b MAX(cpu\u6838\u6570,16) \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u5f02\u6b65\u4e0a\u4f20\u5e76\u53d1\u6570 6.6.0+ nextarch fs.oss.download.async.concurrency \u6574\u578b MAX(cpu\u6838\u6570,16) \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u5f02\u6b65\u4e0b\u8f7d\u5e76\u53d1\u6570 6.6.0+ nextarch fs.oss.array.block.enable \u5e03\u5c14\u503c false \u9ed8\u8ba4\u5173\u95ed\u3002\u6253\u5f00\u540e\u3002\u5ba2\u6237\u7aef\u5199\u5165\u4f18\u5148\u4f7f\u7528\u5185\u5b58 buffer\uff0c\u5355\u4e2abuffer\u5927\u5c0f\u4e0e fs.oss.blocklet.size.mb \u5bf9\u9f50\uff0c\u82e5\u5185\u5b58\u4e0d\u8db3\uff0c\u5219\u5199\u5165\u78c1\u76d8\u3002\u4e0d\u5efa\u8bae\u5728\u5199\u5927\u6587\u4ef6\uff08>8M\uff09\u573a\u666f\u4f7f\u7528\uff0c\u53ef\u80fd\u4f1a\u9020\u6210\u5185\u5b58\u4e0d\u8db3\u3002 6.7.0+ nextarch \u5185\u5b58\u76f8\u5173\u914d\u7f6e\u9879 \u00a7 \u914d\u7f6e\u9879 \u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e \u7248\u672c fs.oss.memory.buffer.size.max.mb \u6574\u578b 6124 \u5185\u5b58\u6c60\u603b\u5bb9\u91cf\uff08\u5355\u4f4d\uff1aMB\uff09 4.3.0+(deprecated) fs.oss.memory.buffer.size.watermark \u6d6e\u70b9\u578b 0.3 \u5185\u5b58\u6c60\u7528\u4e8e\u9884\u8bfb\u7684\u5bb9\u91cf\u6bd4\u4f8b 4.3.0+(deprecated) fs.jdo.memory.pool.size.max.mb \u6574\u578b 6124 \u5185\u5b58\u6c60\u603b\u5bb9\u91cf\uff08\u5355\u4f4d\uff1aMB\uff09 6.7.0+ nextarch fs.jdo.memory.io.buffer.size.max.ratio \u6d6e\u70b9\u578b 0.8 \u5185\u5b58\u6c60\u6700\u5927\u7528\u4e8eIO buffer\u7684\u5bb9\u91cf\u6bd4\u4f8b 6.7.0+ nextarch Metrics \u76f8\u5173\u914d\u7f6e\u9879 \u00a7 \u914d\u7f6e\u9879 \u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e \u7248\u672c fs.jdo.metrics.level \u6574\u578b 1 mertrics \u6536\u96c6\u7b49\u7ea7\uff0c0\u662f\u4e0d\u6536\u96c6\uff0c1\u662f\u9759\u6001metrics\uff0c2\u662f\u5305\u542b bucket \u4fe1\u606f\u7684\u52a8\u6001 metrics\uff0c3\u5305\u542b\u6240\u6709 metrics 6.6.0+ nextarch fs.jdo.metrics.file.enable \u5e03\u5c14\u503c false \u63a7\u5236\u662f\u5426\u8f93\u51fa metrics \u5230\u6587\u4ef6 6.6.0+ nextarch fs.jdo.metrics.file.dir \u5b57\u7b26\u4e32 /tmp/metrics \u8f93\u51fa metrics \u5230\u6587\u4ef6\u7684\u6240\u5728\u76ee\u5f55 6.6.0+ nextarch fs.jdo.metrics.file.pid.append.enable \u5e03\u5c14\u503c false \u63a7\u5236\u8f93\u51fa metrics \u5230\u6587\u4ef6\u7684\u65b9\u5f0f\uff0ctrue \u4ee3\u8868\u4ee5 pid \u4f5c\u4e3a\u5b50\u76ee\u5f55\uff0c\u76ee\u5f55\u5c42\u7ea7\u4e3a(file_dir)/pid/jindosdk_(timestamp).metrics 6.7.0+ nextarch fs.jdo.metrics.file.number \u6574\u578b 16 \u63a7\u5236\u5355\u4e2a\u76ee\u5f55\u4e0b\u6700\u5927\u6587\u4ef6\u7684\u4e2a\u6570 6.6.0+ nextarch fs.jdo.metrics.interval.sec \u6574\u578b 15 \u8f93\u51fa metrics \u5230\u6587\u4ef6\u7684\u95f4\u9694\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 6.6.0+ nextarch fs.jdo.metrics.file.clean.enable \u5e03\u5c14\u503c false \u63a7\u5236\u662f\u5426\u81ea\u52a8\u6e05\u7406\u8f93\u51fa metrics \u7684\u6587\u4ef6, true \u4ee3\u8868\u6253\u5f00\u6e05\u7406\uff0c\u9ed8\u8ba424h\u6e05\u7406\u4e00\u6b21 6.6.0+ nextarch Credential Provider \u76f8\u5173\u914d\u7f6e\uff0c\u8be6\u89c1 \u300a\u914d\u7f6e OSS/OSS-HDFS Credential Provider\u300b","title":"\u914d\u7f6e"},{"location":"jindosdk/jindosdk_configuration/#_1","text":"","title":"\u5ba2\u6237\u7aef\u5e38\u7528\u914d\u7f6e"},{"location":"jindosdk/jindosdk_configuration/#ossoss-hdfs","text":"","title":"OSS/OSS-HDFS"},{"location":"jindosdk/jindosdk_configuration/#_2","text":"\u914d\u7f6e\u9879 \u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e \u7248\u672c logger.dir \u5b57\u7b26\u4e32 /tmp/bigboot-log \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa\uff0clogger.consolelogger \u548c logger.jnilogger \u5747\u4e3a false \u65f6\u751f\u6548\u3002 4.3.0+ logger.sync \u5e03\u5c14\u503c false \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa 4.3.0+ logger.consolelogger \u5e03\u5c14\u503c false \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef 4.3.0+ logger.jnilogger \u5e03\u5c14\u503c false \u6253\u5370\u65e5\u5fd7\u5230log4j 4.3.0+ logger.level \u6574\u578b 2 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002\u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 4.3.0+ logger.verbose \u6574\u578b 0 \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa 4.3.0+ logger.cleaner.enable \u5e03\u5c14\u503c false \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 4.3.0+","title":"\u65e5\u5fd7\u914d\u7f6e\u9879"},{"location":"jindosdk/jindosdk_configuration/#java-sdk-log4jproperties","text":"log4j.logger.com.aliyun.jindodata=INFO log4j.logger.com.aliyun.jindodata.common.FsStats=INFO","title":"Java SDK \u8bbe\u7f6e log4j.properties"},{"location":"jindosdk/jindosdk_configuration/#io","text":"\u914d\u7f6e\u9879 \u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e \u7248\u672c fs.oss.tmp.data.dirs \u5b57\u7b26\u4e32 /tmp/ \u5ba2\u6237\u7aef\u5199\u5165\u65f6\u7684\u4e34\u65f6\u6587\u4ef6\u76ee\u5f55\uff0c\u53ef\u914d\u7f6e\u591a\u4e2a\uff08\u9017\u53f7\u9694\u5f00\uff09\uff0c\u4f1a\u8f6e\u6d41\u5199\u5165\uff0c\u591a\u7528\u6237\u73af\u5883\u9700\u914d\u7f6e\u53ef\u8bfb\u5199\u6743\u9650 4.3.0+ fs.oss.tmp.data.cleaner.enable \u5e03\u5c14\u503c true \u4e34\u65f6\u6587\u4ef6\u81ea\u6e05\u7406\u670d\u52a1 4.3.0+ fs.oss.retry.count \u6574\u578b 5 \u8bbf\u95ee OSS/OSS-HDFS \u5931\u8d25\u91cd\u8bd5\u6b21\u6570 4.3.0+ fs.oss.retry.interval.millisecond \u6574\u578b 500 \u8bbf\u95ee OSS/OSS-HDFS \u5931\u8d25\u91cd\u8bd5\u95f4\u9694\uff08\u6beb\u79d2\uff09 4.3.0+ fs.oss.timeout.millisecond \u6574\u578b 30000 \u8bf7\u6c42 OSS/OSS-HDFS \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 4.3.0+ fs.oss.connection.timeout.millisecond \u6574\u578b 3000 \u8fde\u63a5 OSS/OSS-HDFS \u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 4.3.0+ fs.oss.max.connections.per.host \u6574\u578b 100 \u8fde\u63a5 OSS/OSS-HDFS \u7684\u8fde\u63a5\u6c60\u5bf9\u6bcf\u4e2ahost\u7684\u6700\u5927\u8fde\u63a5\u6570\uff08\u8d85\u8fc7\u9608\u503c\u5916\u7684\u8fde\u63a5\u4f1a\u4f7f\u7528\u77ed\u8fde\u63a5\uff09 4.3.0+ fs.oss.upload.thread.concurrency \u6574\u578b MAX(cpu\u6838\u6570,16) \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u5e76\u53d1\u4e0a\u4f20\u7ebf\u7a0b\u6570 4.3.0+(deprecated) fs.oss.upload.queue.size \u6574\u578b MAX(cpu\u6838\u6570,16) \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u961f\u5217\u5927\u5c0f 4.3.0+(deprecated) fs.oss.upload.max.pending.tasks.per.stream \u6574\u578b 10 \u5355\u4e2a\u6587\u4ef6 OSS/OSS-HDFS \u6700\u5927\u5e76\u53d1\u4e0a\u4f20\u4efb\u52a1\u6570 4.3.0+(deprecated) fs.oss.download.thread.concurrency \u6574\u578b MAX(cpu\u6838\u6570,16) \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u6700\u5927\u5e76\u53d1\u4e0b\u8f7d\u4efb\u52a1\u6570 4.3.0+(deprecated) fs.oss.read.readahead.max.buffer.count \u6574\u578b 48 \u6700\u5927\u540c\u65f6\u9884\u8bfb OSS/OSS-HDFS \u7684 buffer \u4e2a\u6570 4.3.0+(deprecated) fs.oss.read.buffer.size \u6574\u578b 1048576 OSS/OSS-HDFS \u8bfb\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 4.3.0+ fs.oss.write.buffer.size \u6574\u578b 1048576 OSS/OSS-HDFS \u5199\u7f13\u51b2\u533a\u5927\u5c0f\uff08\u5b57\u8282\uff09 4.3.0+ fs.oss.flush.interval.millisecond \u6574\u578b -1 OSS/OSS-HDFS \u5237\u65b0\u7f13\u51b2\u533a\u95f4\u9694\uff08\u6beb\u79d2\uff09\uff0c\u5c0f\u4e8e 0 \u65f6\u4e0d\u751f\u6548 4.3.0+ fs.oss.blocklet.size.mb \u6574\u578b 8 OSS \u5206\u5757\u4e0a\u4f20\u65f6\u7684\u5757\u5927\u5c0f\uff08MB\uff09\u7531\u4e8e\u5206\u5757\u6570\u91cf\u6700\u591a\u4e3a10000\u5757\uff0c\u56e0\u6b64\u9ed8\u8ba4\u5199\u5165\u6587\u4ef6\u4e0d\u80fd\u8d85\u8fc780GB\u3002\u5982\u679c\u6709\u4e2a\u522b\u6587\u4ef6\u8d85\u8fc780G\uff0c\u5efa\u8bae\u6839\u636e\u6587\u4ef6\u5927\u5c0f\u5355\u72ec\u8c03\u5927\u672c\u914d\u7f6e\uff0c\u5e76\u540c\u65f6\u8c03\u5927\u8bf7\u6c42 oss \u7684\u8d85\u65f6\u65f6\u95f4\u3002\u5982\u6587\u4ef6\u5927\u5c0f\u672a\u77e5\uff0c\u6216\u8005\u8fdc\u8fdc\u8d85\u8fc780G\uff08\u5982\u8d85\u8fc7160G\uff09\uff0c\u5efa\u8bae\u8003\u8651\u4f7f\u7528 OSS-HDFS\uff08\u65e0\u6587\u4ef6\u5927\u5c0f\u9650\u5236\uff09 4.5.2+ fs.oss.checksum.crc64.enable \u5e03\u5c14\u503c true \u6587\u4ef6\u7ea7\u522b crc64 \u5b8c\u6574\u6027\u6821\u9a8c\uff0c\u76ee\u524d\u5bf9\u5199OSS-HDFS\u6027\u80fd\u6709\u8f83\u5927\u5f71\u54cd\uff0c\u6027\u80fd\u4f18\u5148\u573a\u666f\u53ef\u4ee5\u8003\u8651\u5173\u95ed\u3002 4.6.0+ fs.oss.checksum.md5.enable \u5e03\u5c14\u503c false \u8bf7\u6c42\u7ea7\u522b md5 \u5b8c\u6574\u6027\u6821\u9a8c 4.6.0+ fs.oss.read.readahead.prefetcher.version \u5b57\u7b26\u4e32 default \u53ef\u9009\u503c\uff1a legacy \u539f\u9884\u8bfb\u7b97\u6cd5, default \u65b0\u9884\u8bfb\u7b97\u6cd5\u3002\u65b0\u9884\u8bfb\u7b97\u6cd5\u53ef\u80fd\u4f7f\u7528\u66f4\u591a\u5185\u5b58\u3002\u82e5\u914d\u7f6e\u4e86\u65b0\u9884\u8bfb\u7b97\u6cd5\u540e\u53d1\u751f\u6027\u80fd\u4e0b\u964d\uff0c\u53ef\u80fd\u662f\u7531\u4e8e\u5185\u5b58\u6c60\u5bb9\u91cf\u4e0d\u8db3\u5bfc\u81f4\u9884\u8bfb\u7684\u5757\u5728\u88ab\u8bbf\u95ee\u5230\u4e4b\u524d\u5c31\u88ab\u63d0\u524d\u9010\u51fa\u3002\u4e3a\u4e86\u907f\u514d\u8be5\u60c5\u51b5\u53d1\u751f\uff0c\u53ef\u4ee5\u8003\u8651\u7f29\u51cf\u6700\u5927\u9884\u8bfb\u957f\u5ea6\uff0c\u6216\u5141\u8bb8\u9884\u8bfb\u4f7f\u7528\u66f4\u591a\u5185\u5b58\u3002 6.2.0+ fs.oss.read.readahead.prefetch.size.max \u6574\u578b 268435456 \u9884\u8bfb\u6700\u5927\u957f\u5ea6\uff08\u5355\u4f4d\uff1abyte\uff09 6.2.0+ fs.oss.signer.version \u6574\u578b 0 \u7b7e\u540d\u7b97\u6cd5\u7248\u672c\uff0c\u53ef\u9009\u503c[0\uff0c1\uff0c4]\u3002\u63a8\u8350\u4f7f\u75280\uff0c\u8868\u793a\u9ed8\u8ba4\u7b97\u6cd5\u7248\u672c\uff0c6.3.0\u7248\u672c\u4e4b\u540e\u9ed8\u8ba4\u4f7f\u7528 V4 \u7b7e\u540d\u7b97\u6cd5\u63d0\u9ad8\u8bbf\u95ee OSS/OSS-HDFS\uff0c\u82e5\u7279\u6b8a\u573a\u666f\u5e0c\u671b\u6307\u5b9a\u7b7e\u540d\u7b97\u6cd5\u7248\u672c\uff0c\u8bf7\u6307\u5b9a 1 \u6216 4\u3002 6.3.0+ fs.oss.io.timeout.millisecond \u6574\u578b 90000 \u8bf7\u6c42 OSS/OSS-HDFS \u8bfb\u5199\u64cd\u4f5c\u8d85\u65f6\u65f6\u95f4\uff08\u6beb\u79d2\uff09 6.6.0+ nextarch fs.oss.async.executor.number \u6574\u578b cpu\u6838\u6570 \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u5f02\u6b65io\u7ebf\u7a0b\u6570 6.6.0+ nextarch fs.oss.upload.async.concurrency \u6574\u578b MAX(cpu\u6838\u6570,16) \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u5f02\u6b65\u4e0a\u4f20\u5e76\u53d1\u6570 6.6.0+ nextarch fs.oss.download.async.concurrency \u6574\u578b MAX(cpu\u6838\u6570,16) \u8fdb\u7a0b\u5185 OSS/OSS-HDFS \u5f02\u6b65\u4e0b\u8f7d\u5e76\u53d1\u6570 6.6.0+ nextarch fs.oss.array.block.enable \u5e03\u5c14\u503c false \u9ed8\u8ba4\u5173\u95ed\u3002\u6253\u5f00\u540e\u3002\u5ba2\u6237\u7aef\u5199\u5165\u4f18\u5148\u4f7f\u7528\u5185\u5b58 buffer\uff0c\u5355\u4e2abuffer\u5927\u5c0f\u4e0e fs.oss.blocklet.size.mb \u5bf9\u9f50\uff0c\u82e5\u5185\u5b58\u4e0d\u8db3\uff0c\u5219\u5199\u5165\u78c1\u76d8\u3002\u4e0d\u5efa\u8bae\u5728\u5199\u5927\u6587\u4ef6\uff08>8M\uff09\u573a\u666f\u4f7f\u7528\uff0c\u53ef\u80fd\u4f1a\u9020\u6210\u5185\u5b58\u4e0d\u8db3\u3002 6.7.0+ nextarch","title":"I/O \u76f8\u5173\u914d\u7f6e\u9879"},{"location":"jindosdk/jindosdk_configuration/#_3","text":"\u914d\u7f6e\u9879 \u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e \u7248\u672c fs.oss.memory.buffer.size.max.mb \u6574\u578b 6124 \u5185\u5b58\u6c60\u603b\u5bb9\u91cf\uff08\u5355\u4f4d\uff1aMB\uff09 4.3.0+(deprecated) fs.oss.memory.buffer.size.watermark \u6d6e\u70b9\u578b 0.3 \u5185\u5b58\u6c60\u7528\u4e8e\u9884\u8bfb\u7684\u5bb9\u91cf\u6bd4\u4f8b 4.3.0+(deprecated) fs.jdo.memory.pool.size.max.mb \u6574\u578b 6124 \u5185\u5b58\u6c60\u603b\u5bb9\u91cf\uff08\u5355\u4f4d\uff1aMB\uff09 6.7.0+ nextarch fs.jdo.memory.io.buffer.size.max.ratio \u6d6e\u70b9\u578b 0.8 \u5185\u5b58\u6c60\u6700\u5927\u7528\u4e8eIO buffer\u7684\u5bb9\u91cf\u6bd4\u4f8b 6.7.0+ nextarch","title":"\u5185\u5b58\u76f8\u5173\u914d\u7f6e\u9879"},{"location":"jindosdk/jindosdk_configuration/#metrics","text":"\u914d\u7f6e\u9879 \u7c7b\u578b \u9ed8\u8ba4\u503c \u8bf4\u660e \u7248\u672c fs.jdo.metrics.level \u6574\u578b 1 mertrics \u6536\u96c6\u7b49\u7ea7\uff0c0\u662f\u4e0d\u6536\u96c6\uff0c1\u662f\u9759\u6001metrics\uff0c2\u662f\u5305\u542b bucket \u4fe1\u606f\u7684\u52a8\u6001 metrics\uff0c3\u5305\u542b\u6240\u6709 metrics 6.6.0+ nextarch fs.jdo.metrics.file.enable \u5e03\u5c14\u503c false \u63a7\u5236\u662f\u5426\u8f93\u51fa metrics \u5230\u6587\u4ef6 6.6.0+ nextarch fs.jdo.metrics.file.dir \u5b57\u7b26\u4e32 /tmp/metrics \u8f93\u51fa metrics \u5230\u6587\u4ef6\u7684\u6240\u5728\u76ee\u5f55 6.6.0+ nextarch fs.jdo.metrics.file.pid.append.enable \u5e03\u5c14\u503c false \u63a7\u5236\u8f93\u51fa metrics \u5230\u6587\u4ef6\u7684\u65b9\u5f0f\uff0ctrue \u4ee3\u8868\u4ee5 pid \u4f5c\u4e3a\u5b50\u76ee\u5f55\uff0c\u76ee\u5f55\u5c42\u7ea7\u4e3a(file_dir)/pid/jindosdk_(timestamp).metrics 6.7.0+ nextarch fs.jdo.metrics.file.number \u6574\u578b 16 \u63a7\u5236\u5355\u4e2a\u76ee\u5f55\u4e0b\u6700\u5927\u6587\u4ef6\u7684\u4e2a\u6570 6.6.0+ nextarch fs.jdo.metrics.interval.sec \u6574\u578b 15 \u8f93\u51fa metrics \u5230\u6587\u4ef6\u7684\u95f4\u9694\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 6.6.0+ nextarch fs.jdo.metrics.file.clean.enable \u5e03\u5c14\u503c false \u63a7\u5236\u662f\u5426\u81ea\u52a8\u6e05\u7406\u8f93\u51fa metrics \u7684\u6587\u4ef6, true \u4ee3\u8868\u6253\u5f00\u6e05\u7406\uff0c\u9ed8\u8ba424h\u6e05\u7406\u4e00\u6b21 6.6.0+ nextarch Credential Provider \u76f8\u5173\u914d\u7f6e\uff0c\u8be6\u89c1 \u300a\u914d\u7f6e OSS/OSS-HDFS Credential Provider\u300b","title":"Metrics \u76f8\u5173\u914d\u7f6e\u9879"},{"location":"jindosdk/jindosdk_credential_provider/","text":"\u914d\u7f6e OSS/OSS-HDFS Credential Provider \u00a7 \u57fa\u672c\u914d\u7f6e\u65b9\u5f0f \u00a7 \u60a8\u53ef\u4ee5\u5c06 OSS/OSS-HDFS \u7684 Access Key ID \u3001 Access Secret \u3001 Endpoint \u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u3002EMR \u96c6\u7fa4\u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \u3002 \u914d\u7f6e\u9879\u5982\u4e0b\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> \u4f7f\u7528 Hadoop Credential Providers \u5b58\u50a8 AccessKey \u4fe1\u606f \u00a7 \u4e0a\u9762\u7684 fs.oss.accessKeyId \u548c fs.oss.accessKeySecret \u5c06\u4f1a\u660e\u6587\u914d\u7f6e\u5728 core-site.xml \u4e2d\uff0c\u53ef\u4ee5\u5c06\u5176\u4ee5\u52a0\u5bc6\u5bf9\u65b9\u5f0f\u5b58\u50a8\u81f3 Hadoop Credential Providers\u6587\u4ef6\u4e2d\u3002 \u4f7f\u7528Hadoop\u63d0\u4f9b\u7684\u547d\u4ee4\uff0c\u5b58\u50a8AccessKey\u548cSecurityToken\u4fe1\u606f\u81f3Credential\u6587\u4ef6\u4e2d\u3002\u547d\u4ee4\u683c\u5f0f\u5982\u4e0b\u3002 hadoop credential <subcommand> [options] \u4f8b\u5982\uff0c\u5b58\u50a8AccessKey\u548cToken\u4fe1\u606f\u81f3JECKS\u6587\u4ef6\u4e2d\uff0c\u9664\u4e86\u4f7f\u7528\u6587\u4ef6\u6743\u9650\u4fdd\u62a4\u8be5\u6587\u4ef6\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u6307\u5b9a\u5bc6\u7801\u52a0\u5bc6\u5b58\u50a8\u4fe1\u606f\uff0c\u5982\u679c\u4e0d\u6307\u5b9a\u5bc6\u7801\u5219\u4f7f\u7528\u9ed8\u8ba4\u5b57\u7b26\u4e32\u52a0\u5bc6\u3002 hadoop credential create fs.oss.accessKeyId -value AAA -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.accessKeySecret -value BBB -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.securityToken -value CCC -provider jceks://file/root/oss.jceks \u751f\u6210Credential\u6587\u4ef6\u540e\uff0c\u60a8\u9700\u8981\u914d\u7f6e\u4e0b\u9762\u7684\u53c2\u6570\u6765\u6307\u5b9aProvider\u7684\u7c7b\u578b\u548c\u4f4d\u7f6e\u3002 <configuration> <property> <name>fs.oss.security.credential.provider.path</name> <value>jceks://file/root/oss.jceks</value> <description>\u914d\u7f6e\u5b58\u50a8AK\u7684Credential\u6587\u4ef6\u3002\u4f8b\u5982\uff0cjceks://file/${user.home}/oss.jceks\u4e3aHOME\u4e0b\u7684oss.jceks\u6587\u4ef6</description> </property> </configuration> \u4f7f\u7528 JindoSDK OSS/OSS-HDFS Credential Provider \u00a7 \u9ed8\u8ba4\u4f1a\u914d\u7f6e SimpleCredentialsProvider\uff0cEnvironmentVariableCredentialsProvider\uff0cCommonCredentialsProvider \u8fd9\u4e09\u4e2aCredential Provider, \u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002 <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.JindoCredentialsProvider\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u60a8\u53ef\u4ee5\u6839\u636e\u60c5\u51b5\uff0c\u9009\u62e9\u4e0d\u540c\u7684Credential Provider\uff0c\u652f\u6301\u5982\u4e0bProvider\uff1a 1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS/OSS-HDFS \u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS/OSS-HDFS\u7684AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS/OSS-HDFS\u7684AccessKey Secret</value> </property> <property> <name>fs.oss.securityToken</name> <value>OSS/OSS-HDFS\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration> 2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS/OSS-HDFS \u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS/OSS-HDFS \u7684AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS/OSS-HDFS \u7684AccessKey Secret</value> </property> </configuration> 3. EnvironmentVariableCredentialsProvider \u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6 AK\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS/OSS-HDFS \u7684 AccessKey Id OSS_ACCESS_KEY_SECRET OSS/OSS-HDFS \u7684 AccessKey Secret OSS_SECURITY_TOKEN OSS/OSS-HDFS \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002 4. CommonCredentialsProvider \u4e3a\u901a\u7528\u914d\u7f6e\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS/OSS-HDFS \u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS/OSS-HDFS \u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS/OSS-HDFS \u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration> 5. CustomCredentialsProvider \u5bf9\u63a5\u5b9a\u5236\u7684\u514d\u5bc6\u670d\u52a1\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CustomCredentialsProvider</value> </property> </configuration> \u914d\u7f6e\u514d\u5bc6\u670d\u52a1\u5730\u5740 <configuration> <property> <name>aliyun.oss.provider.url</name> <value>\u514d\u5bc6\u670d\u52a1\u5730\u5740</value> </property> </configuration> aliyun.oss.provider.url \u652f\u6301 http(s) \u534f\u8bae \u548c secrets \u534f\u8bae\uff1a a. http(s) \u534f\u8bae http(s) \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a http://localhost:1234/sts , http \u514d\u5bc6\u534f\u8bae\u8981\u6c42\u8fd4\u56de\u7ed3\u679c\u4e3a Json \u683c\u5f0f\uff0c\u5982\u679c\u60a8\u9700\u8981\u5bf9\u63a5\u60a8\u7684 http \u514d\u5bc6\u670d\u52a1\uff0c\u514d\u5bc6\u670d\u52a1\u7684\u8bbe\u8ba1\u53c2\u8003 \u300a\u901a\u8fc7API\u4f7f\u7528\u5b9e\u4f8bRAM\u89d2\u8272#\u4e34\u65f6\u6388\u6743Token\u300b { \"AccessKeyId\" : \"XXXXXXXXX\", \"AccessKeySecret\" : \"XXXXXXXXX\", \"Expiration\" : \"2020-11-01T05:20:01Z\", \"SecurityToken\" : \"XXXXXXXXX\", \"LastUpdated\" : \"2020-10-31T23:20:01Z\", \"Code\" : \"Success\" } b. Secrets \u534f\u8bae Secrets \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a secrets:///local_path_prefix \uff0c\u5e38\u89c1\u4f7f\u7528\u4e8e k8s \u573a\u666f \uff0c \u5176\u4e2d local_path_prefix \u4e3a\u8def\u5f84\u524d\u7f00\uff0c\u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOssAccessKeyId /secret/JindoOssAccessKeySecret /secret/JindoOssSecurityToken \u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss/ \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOss/AccessKeyId /secret/JindoOss/AccessKeySecret /secret/JindoOss/SecurityToken JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 OSS/OSS-HDFS bucket \u914d\u7f6e\u4e0d\u540c\u7684 Credential Provider \u00a7 \u8be6\u60c5\u53c2\u8003 \u6309 bucket \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u3002","title":"CredentialProvider \u5feb\u901f\u5165\u95e8"},{"location":"jindosdk/jindosdk_credential_provider/#ossoss-hdfs-credential-provider","text":"","title":"\u914d\u7f6e OSS/OSS-HDFS Credential Provider"},{"location":"jindosdk/jindosdk_credential_provider/#_1","text":"\u60a8\u53ef\u4ee5\u5c06 OSS/OSS-HDFS \u7684 Access Key ID \u3001 Access Secret \u3001 Endpoint \u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u3002EMR \u96c6\u7fa4\u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \u3002 \u914d\u7f6e\u9879\u5982\u4e0b\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration>","title":"\u57fa\u672c\u914d\u7f6e\u65b9\u5f0f"},{"location":"jindosdk/jindosdk_credential_provider/#hadoop-credential-providers-accesskey","text":"\u4e0a\u9762\u7684 fs.oss.accessKeyId \u548c fs.oss.accessKeySecret \u5c06\u4f1a\u660e\u6587\u914d\u7f6e\u5728 core-site.xml \u4e2d\uff0c\u53ef\u4ee5\u5c06\u5176\u4ee5\u52a0\u5bc6\u5bf9\u65b9\u5f0f\u5b58\u50a8\u81f3 Hadoop Credential Providers\u6587\u4ef6\u4e2d\u3002 \u4f7f\u7528Hadoop\u63d0\u4f9b\u7684\u547d\u4ee4\uff0c\u5b58\u50a8AccessKey\u548cSecurityToken\u4fe1\u606f\u81f3Credential\u6587\u4ef6\u4e2d\u3002\u547d\u4ee4\u683c\u5f0f\u5982\u4e0b\u3002 hadoop credential <subcommand> [options] \u4f8b\u5982\uff0c\u5b58\u50a8AccessKey\u548cToken\u4fe1\u606f\u81f3JECKS\u6587\u4ef6\u4e2d\uff0c\u9664\u4e86\u4f7f\u7528\u6587\u4ef6\u6743\u9650\u4fdd\u62a4\u8be5\u6587\u4ef6\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u6307\u5b9a\u5bc6\u7801\u52a0\u5bc6\u5b58\u50a8\u4fe1\u606f\uff0c\u5982\u679c\u4e0d\u6307\u5b9a\u5bc6\u7801\u5219\u4f7f\u7528\u9ed8\u8ba4\u5b57\u7b26\u4e32\u52a0\u5bc6\u3002 hadoop credential create fs.oss.accessKeyId -value AAA -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.accessKeySecret -value BBB -provider jceks://file/root/oss.jceks hadoop credential create fs.oss.securityToken -value CCC -provider jceks://file/root/oss.jceks \u751f\u6210Credential\u6587\u4ef6\u540e\uff0c\u60a8\u9700\u8981\u914d\u7f6e\u4e0b\u9762\u7684\u53c2\u6570\u6765\u6307\u5b9aProvider\u7684\u7c7b\u578b\u548c\u4f4d\u7f6e\u3002 <configuration> <property> <name>fs.oss.security.credential.provider.path</name> <value>jceks://file/root/oss.jceks</value> <description>\u914d\u7f6e\u5b58\u50a8AK\u7684Credential\u6587\u4ef6\u3002\u4f8b\u5982\uff0cjceks://file/${user.home}/oss.jceks\u4e3aHOME\u4e0b\u7684oss.jceks\u6587\u4ef6</description> </property> </configuration>","title":"\u4f7f\u7528 Hadoop Credential Providers \u5b58\u50a8 AccessKey \u4fe1\u606f"},{"location":"jindosdk/jindosdk_credential_provider/#jindosdk-ossoss-hdfs-credential-provider","text":"\u9ed8\u8ba4\u4f1a\u914d\u7f6e SimpleCredentialsProvider\uff0cEnvironmentVariableCredentialsProvider\uff0cCommonCredentialsProvider \u8fd9\u4e09\u4e2aCredential Provider, \u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002 <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.JindoCredentialsProvider\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u60a8\u53ef\u4ee5\u6839\u636e\u60c5\u51b5\uff0c\u9009\u62e9\u4e0d\u540c\u7684Credential Provider\uff0c\u652f\u6301\u5982\u4e0bProvider\uff1a","title":"\u4f7f\u7528 JindoSDK OSS/OSS-HDFS Credential Provider"},{"location":"jindosdk/jindosdk_credential_provider/#1-temporarycredentialsprovider-accesskey-securitytoken-ossoss-hdfs","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS/OSS-HDFS\u7684AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS/OSS-HDFS\u7684AccessKey Secret</value> </property> <property> <name>fs.oss.securityToken</name> <value>OSS/OSS-HDFS\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration>","title":"1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS/OSS-HDFS \u7684\u60c5\u51b5\u3002"},{"location":"jindosdk/jindosdk_credential_provider/#2-simplecredentialsprovider-accesskey-ossoss-hdfs","text":"\u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>fs.oss.accessKeyId</name> <value>OSS/OSS-HDFS \u7684AccessKey Id</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>OSS/OSS-HDFS \u7684AccessKey Secret</value> </property> </configuration>","title":"2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS/OSS-HDFS \u7684\u60c5\u51b5\u3002"},{"location":"jindosdk/jindosdk_credential_provider/#3-environmentvariablecredentialsprovider-ak","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS/OSS-HDFS \u7684 AccessKey Id OSS_ACCESS_KEY_SECRET OSS/OSS-HDFS \u7684 AccessKey Secret OSS_SECURITY_TOKEN OSS/OSS-HDFS \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002","title":"3. EnvironmentVariableCredentialsProvider \u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6 AK\u3002"},{"location":"jindosdk/jindosdk_credential_provider/#4-commoncredentialsprovider","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS/OSS-HDFS \u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS/OSS-HDFS \u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS/OSS-HDFS \u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration>","title":"4. CommonCredentialsProvider \u4e3a\u901a\u7528\u914d\u7f6e\u3002"},{"location":"jindosdk/jindosdk_credential_provider/#5-customcredentialsprovider","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CustomCredentialsProvider</value> </property> </configuration> \u914d\u7f6e\u514d\u5bc6\u670d\u52a1\u5730\u5740 <configuration> <property> <name>aliyun.oss.provider.url</name> <value>\u514d\u5bc6\u670d\u52a1\u5730\u5740</value> </property> </configuration> aliyun.oss.provider.url \u652f\u6301 http(s) \u534f\u8bae \u548c secrets \u534f\u8bae\uff1a a. http(s) \u534f\u8bae http(s) \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a http://localhost:1234/sts , http \u514d\u5bc6\u534f\u8bae\u8981\u6c42\u8fd4\u56de\u7ed3\u679c\u4e3a Json \u683c\u5f0f\uff0c\u5982\u679c\u60a8\u9700\u8981\u5bf9\u63a5\u60a8\u7684 http \u514d\u5bc6\u670d\u52a1\uff0c\u514d\u5bc6\u670d\u52a1\u7684\u8bbe\u8ba1\u53c2\u8003 \u300a\u901a\u8fc7API\u4f7f\u7528\u5b9e\u4f8bRAM\u89d2\u8272#\u4e34\u65f6\u6388\u6743Token\u300b { \"AccessKeyId\" : \"XXXXXXXXX\", \"AccessKeySecret\" : \"XXXXXXXXX\", \"Expiration\" : \"2020-11-01T05:20:01Z\", \"SecurityToken\" : \"XXXXXXXXX\", \"LastUpdated\" : \"2020-10-31T23:20:01Z\", \"Code\" : \"Success\" } b. Secrets \u534f\u8bae Secrets \u534f\u8bae\u514d\u5bc6\u670d\u52a1\u5730\u5740\u683c\u5f0f\u4e3a secrets:///local_path_prefix \uff0c\u5e38\u89c1\u4f7f\u7528\u4e8e k8s \u573a\u666f \uff0c \u5176\u4e2d local_path_prefix \u4e3a\u8def\u5f84\u524d\u7f00\uff0c\u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOssAccessKeyId /secret/JindoOssAccessKeySecret /secret/JindoOssSecurityToken \u5982\u679c local_path_prefix \u4e3a secrets:///secret/JindoOss/ \uff0c\u5219\u4f1a\u5728\u8282\u70b9\u4e0a\u67e5\u627e\u5982\u4e0b AccessKey \u7b49\u6587\u4ef6 /secret/JindoOss/AccessKeyId /secret/JindoOss/AccessKeySecret /secret/JindoOss/SecurityToken","title":"5. CustomCredentialsProvider \u5bf9\u63a5\u5b9a\u5236\u7684\u514d\u5bc6\u670d\u52a1\u3002"},{"location":"jindosdk/jindosdk_credential_provider/#jindosdk-ossoss-hdfs-bucket-credential-provider","text":"\u8be6\u60c5\u53c2\u8003 \u6309 bucket \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u3002","title":"JindoSDK \u8fd8\u652f\u6301\u4e0d\u540c\u7684 OSS/OSS-HDFS bucket \u914d\u7f6e\u4e0d\u540c\u7684 Credential Provider"},{"location":"jindosdk/jindosdk_credential_provider_bucket/","text":"\u6309 bucket \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u00a7 \u6309\u7167 bucket \u914d\u7f6e Credential Provider \u00a7 <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleAliyunCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.AliyunCredentialsProvider\u7684\u5b9e\u73b0\u7c7b\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u8bf4\u660e XXX \u4e3a OSS/OSS-HDFS bucket \u540d\u79f0\u3002 Credential Provider \u7c7b\u578b \u00a7 1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS/OSS-HDFS \u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS/OSS-HDFS bucket\u7684AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS/OSS-HDFS bucket\u7684AccessKey Secret</value> </property> <property> <name>fs.oss.bucket.XXX.securityToken</name> <value>OSS/OSS-HDFS bucket\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration> 2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS/OSS-HDFS \u7684\u60c5\u51b5\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS/OSS-HDFS bucket\u7684AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS/OSS-HDFS bucket\u7684AccessKey Secret</value> </property> </configuration> 3. EnvironmentVariableCredentialsProvider \u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6 AccessKey\u3002 \u00a7 \u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS/OSS-HDFS bucket\u7684AccessKey Id OSS_ACCESS_KEY_SECRET OSS/OSS-HDFS bucket\u7684AccessKey Secret OSS_SECURITY_TOKEN OSS/OSS-HDFS bucket\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548Token\u65f6\u9700\u8981\u3002 4. CommonCredentialsProvider\u4e3a\u901a\u7528\u914d\u7f6e\u3002 \u00a7 \u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS/OSS-HDFS bucket \u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS/OSS-HDFS bucket \u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS/OSS-HDFS bucket \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration>","title":"Bucket \u7ea7\u522b\u914d\u7f6e CredentialProvider"},{"location":"jindosdk/jindosdk_credential_provider_bucket/#bucket-ossoss-hdfs-credential-provider","text":"","title":"\u6309 bucket \u914d\u7f6e OSS/OSS-HDFS Credential Provider"},{"location":"jindosdk/jindosdk_credential_provider_bucket/#bucket-credential-provider","text":"<configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleAliyunCredentialsProvider,com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider,com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> <description>\u914d\u7f6ecom.aliyun.jindodata.oss.auth.AliyunCredentialsProvider\u7684\u5b9e\u73b0\u7c7b\uff0c\u591a\u4e2a\u7c7b\u65f6\u4f7f\u7528\u82f1\u6587\u9017\u53f7\uff08, \uff09\u9694\u5f00\uff0c\u6309\u7167\u5148\u540e\u987a\u5e8f\u8bfb\u53d6Credential\u76f4\u81f3\u8bfb\u5230\u6709\u6548\u7684Credential\u3002Provider\u8be6\u60c5\u8bf7\u53c2\u89c1Credential Provider\u7c7b\u578b\u3002</description> </property> </configuration> \u8bf4\u660e XXX \u4e3a OSS/OSS-HDFS bucket \u540d\u79f0\u3002","title":"\u6309\u7167 bucket \u914d\u7f6e Credential Provider"},{"location":"jindosdk/jindosdk_credential_provider_bucket/#credential-provider","text":"","title":"Credential Provider \u7c7b\u578b"},{"location":"jindosdk/jindosdk_credential_provider_bucket/#1-temporarycredentialsprovider-accesskey-securitytoken-ossoss-hdfs","text":"\u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.TemporaryCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS/OSS-HDFS bucket\u7684AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS/OSS-HDFS bucket\u7684AccessKey Secret</value> </property> <property> <name>fs.oss.bucket.XXX.securityToken</name> <value>OSS/OSS-HDFS bucket\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)</value> </property> </configuration>","title":"1. TemporaryCredentialsProvider \u9002\u5408\u4f7f\u7528\u6709\u65f6\u6548\u6027\u7684 AccessKey \u548c SecurityToken \u8bbf\u95ee OSS/OSS-HDFS \u7684\u60c5\u51b5\u3002"},{"location":"jindosdk/jindosdk_credential_provider_bucket/#2-simplecredentialsprovider-accesskey-ossoss-hdfs","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.accessKeyId</name> <value>OSS/OSS-HDFS bucket\u7684AccessKey Id</value> </property> <property> <name>fs.oss.bucket.XXX.accessKeySecret</name> <value>OSS/OSS-HDFS bucket\u7684AccessKey Secret</value> </property> </configuration>","title":"2. SimpleCredentialsProvider \u9002\u5408\u4f7f\u7528\u957f\u671f\u6709\u6548\u7684 AccessKey \u8bbf\u95ee OSS/OSS-HDFS \u7684\u60c5\u51b5\u3002"},{"location":"jindosdk/jindosdk_credential_provider_bucket/#3-environmentvariablecredentialsprovider-accesskey","text":"\u914d\u7f6eProvider\u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.EnvironmentVariableCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff0c\u9700\u8981\u5728\u73af\u5883\u53d8\u91cf\u4e2d\u914d\u7f6e\u4ee5\u4e0b\u53c2\u6570\uff1a \u53c2\u6570 \u53c2\u6570\u8bf4\u660e OSS_ACCESS_KEY_ID OSS/OSS-HDFS bucket\u7684AccessKey Id OSS_ACCESS_KEY_SECRET OSS/OSS-HDFS bucket\u7684AccessKey Secret OSS_SECURITY_TOKEN OSS/OSS-HDFS bucket\u7684SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c\uff09\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548Token\u65f6\u9700\u8981\u3002","title":"3. EnvironmentVariableCredentialsProvider \u5728\u73af\u5883\u53d8\u91cf\u4e2d\u83b7\u53d6 AccessKey\u3002"},{"location":"jindosdk/jindosdk_credential_provider_bucket/#4-commoncredentialsprovider","text":"\u914d\u7f6e Provider \u7c7b\u578b\uff1a <configuration> <property> <name>fs.oss.bucket.XXX.credentials.provider</name> <value>com.aliyun.jindodata.oss.auth.CommonCredentialsProvider</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS AK\uff1a <configuration> <property> <name>jindo.common.accessKeyId</name> <value>OSS/OSS-HDFS bucket \u7684 AccessKey Id</value> </property> <property> <name>jindo.common.accessKeySecret</name> <value>OSS/OSS-HDFS bucket \u7684 AccessKey Secret</value> </property> <property> <name>jindo.common.securityToken</name> <value>OSS/OSS-HDFS bucket \u7684 SecurityToken\uff08\u4e34\u65f6\u5b89\u5168\u4ee4\u724c)\u3002\u8bf4\u660e \u4ec5\u914d\u7f6e\u6709\u65f6\u6548 Token \u65f6\u9700\u8981\u3002</value> </property> </configuration>","title":"4. CommonCredentialsProvider\u4e3a\u901a\u7528\u914d\u7f6e\u3002"},{"location":"jindosdk/jindosdk_credential_provider_faq/","text":"\u8bbf\u95ee OSS/OSS-HDFS \u65f6 AK \u76f8\u5173\u5e38\u89c1\u95ee\u9898 \u00a7 1. EMR \u96c6\u7fa4\u5185\u5982\u4f55\u514d\u5bc6\u8bbf\u95ee OSS/OSS-HDFS \u00a7 JindoSDK \u4f1a\u4f7f\u7528\u521b\u5efa\u96c6\u7fa4\u65f6\u7ed1\u5b9a\u7684 ECS \u5e94\u7528\u89d2\u8272\uff08\u9ed8\u8ba4\u4e3a AliyunECSInstanceForEMRRole\uff09\u83b7\u53d6 Security Token \u8bbf\u95ee OSS/OSS-HDFS\u3002 \u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \u4e2d\u66f4\u65b0\u4e0b\u5217\u53c2\u6570\uff1a \u53c2\u6570 \u503c fs.oss.credentials.provider com.aliyun.jindodata.oss.auth.EcsStsCredentialsProvider 2. \u5982\u4f55\u68c0\u67e5 EMR \u96c6\u7fa4\u5185\u514d\u5bc6\u670d\u52a1\u662f\u5426\u53ef\u7528\uff1f \u00a7 \u65b9\u6cd5\u4e00\uff1a \u00a7 \u6267\u884c\u4e0b\u9762\u547d\u4ee4\uff1a curl http://100.100.100.200/latest/meta-data/Ram/Security-credentials/AliyunECSInstanceForEMRRole \u5982\u679c\u8fd4\u56de\u4e0b\u9762\u683c\u5f0f\u7684\u5185\u5bb9\uff0c\u5219\u6b63\u5e38\u3002 \"AccessKeyId\" : \"STS.NUreXXXXXX\", \"AccessKeySecret\" : \"BsmbnDoXXXXXXXX\", \"Expiration\" : \"2022-11-22T11:27:39Z\", \"SecurityToken\" : \"CAISlwJ1q6FXXXXXXX\", \"LastUpdated\" : \"2022-11-22T05:27:39Z\", \"Code\" : \"Success\" \u65b9\u6cd5\u4e8c\uff1a \u00a7 \u914d\u7f6e\u4e3a ECS \u514d\u5bc6\uff1a \u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \u4e2d\u66f4\u65b0\u4e0b\u5217\u53c2\u6570\uff1a \u53c2\u6570 \u503c fs.oss.credentials.provider com.aliyun.jindodata.oss.auth.EcsStsCredentialsProvider \u4f7f\u7528 HDFS shell \u8bbf\u95ee OSS/OSS-HDFS bucket, \u82e5\u80fd\u6b63\u5e38\u8bbf\u95ee\uff0c\u5219\u786e\u8ba4\u514d\u5bc6\u6b63\u5e38\u3002 3. \u5e38\u9a7b\u670d\u52a1\u6bcf 5-6 \u4e2a\u5c0f\u65f6\u8bbf\u95ee OSS/OSS-HDFS \u5931\u8d25 \u00a7 \u67d0\u4e9b\u7248\u672c\u6709\u5df2\u77e5\u514d\u5bc6\u95ee\u9898\uff0c\u8bf7\u67e5\u770b \u5df2\u77e5\u95ee\u9898 \u65b9\u6cd5\u4e00\uff1a\u4f7f\u7528\u56fa\u5b9a AK \u00a7 \u914d\u7f6e\u4f7f\u7528 SimpleCredentialsProvider , \u8bf7\u53c2\u8003 \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u65b9\u6cd5\u4e8c\uff1a\u5347\u7ea7 JindoSDK \u4e3a\u6700\u65b0\u7248\u672c \u00a7 \u65e7\u7248\u63a7\u5236\u53f0\u5347\u7ea7\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b \u65b0\u7248\u63a7\u5236\u53f0\u5347\u7ea7\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b E-MapReduce EMR-5.5.0/EMR-3.39.0 \u4ee5\u524d\u7248\u672c\u7684\u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b 4. \u8bbf\u95ee OSS/OSS-HDFS \u8def\u5f84\u4e2d\u5305\u542b AK \u4fe1\u606f\u62a5\u9519 \u00a7 \u62a5\u9519\u4fe1\u606f\u5982\u4e0b\uff1a The Filesystem URI contains login details. This authentication mechanism is no longer supported. \u4e3a\u4e86\u9632\u6b62 AK \u6cc4\u9732\u7b49\u5b89\u5168\u95ee\u9898\uff0cJindoSDK 4.0.0 \u5f00\u59cb\u4e0d\u652f\u6301\u8def\u5f84\u4e2d\u643a\u5e26 AK(AccessKeyId,AccessKeySecret\u7b49\uff09\u4fe1\u606f\u3002 \u65b9\u6cd5\u4e00\uff1a \u00a7 \u8bbf\u95ee\u8def\u5f84\u4e2d\u53bb\u9664 AK \u4fe1\u606f\uff1b \u65b9\u6cd5\u4e8c\uff1a \u00a7 \u5982\u679c\u4e00\u5b9a\u8981\u8fd9\u6837\u4f7f\u7528\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u53c2\u6570\u5f00\u542f\u3002\u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \u4e2d\u65b0\u589e\u4e0b\u5217\u53c2\u6570\uff1a \u53c2\u6570 \u503c fs.oss.uri-with-secrets.enable true","title":"\u8bbf\u95ee OSS/OSS-HDFS \u65f6 AK \u76f8\u5173\u5e38\u89c1\u95ee\u9898"},{"location":"jindosdk/jindosdk_credential_provider_faq/#ossoss-hdfs-ak","text":"","title":"\u8bbf\u95ee OSS/OSS-HDFS \u65f6 AK \u76f8\u5173\u5e38\u89c1\u95ee\u9898"},{"location":"jindosdk/jindosdk_credential_provider_faq/#1-emr-ossoss-hdfs","text":"JindoSDK \u4f1a\u4f7f\u7528\u521b\u5efa\u96c6\u7fa4\u65f6\u7ed1\u5b9a\u7684 ECS \u5e94\u7528\u89d2\u8272\uff08\u9ed8\u8ba4\u4e3a AliyunECSInstanceForEMRRole\uff09\u83b7\u53d6 Security Token \u8bbf\u95ee OSS/OSS-HDFS\u3002 \u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \u4e2d\u66f4\u65b0\u4e0b\u5217\u53c2\u6570\uff1a \u53c2\u6570 \u503c fs.oss.credentials.provider com.aliyun.jindodata.oss.auth.EcsStsCredentialsProvider","title":"1. EMR \u96c6\u7fa4\u5185\u5982\u4f55\u514d\u5bc6\u8bbf\u95ee OSS/OSS-HDFS"},{"location":"jindosdk/jindosdk_credential_provider_faq/#2-emr","text":"","title":"2. \u5982\u4f55\u68c0\u67e5 EMR \u96c6\u7fa4\u5185\u514d\u5bc6\u670d\u52a1\u662f\u5426\u53ef\u7528\uff1f"},{"location":"jindosdk/jindosdk_credential_provider_faq/#_1","text":"\u6267\u884c\u4e0b\u9762\u547d\u4ee4\uff1a curl http://100.100.100.200/latest/meta-data/Ram/Security-credentials/AliyunECSInstanceForEMRRole \u5982\u679c\u8fd4\u56de\u4e0b\u9762\u683c\u5f0f\u7684\u5185\u5bb9\uff0c\u5219\u6b63\u5e38\u3002 \"AccessKeyId\" : \"STS.NUreXXXXXX\", \"AccessKeySecret\" : \"BsmbnDoXXXXXXXX\", \"Expiration\" : \"2022-11-22T11:27:39Z\", \"SecurityToken\" : \"CAISlwJ1q6FXXXXXXX\", \"LastUpdated\" : \"2022-11-22T05:27:39Z\", \"Code\" : \"Success\"","title":"\u65b9\u6cd5\u4e00\uff1a"},{"location":"jindosdk/jindosdk_credential_provider_faq/#_2","text":"\u914d\u7f6e\u4e3a ECS \u514d\u5bc6\uff1a \u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \u4e2d\u66f4\u65b0\u4e0b\u5217\u53c2\u6570\uff1a \u53c2\u6570 \u503c fs.oss.credentials.provider com.aliyun.jindodata.oss.auth.EcsStsCredentialsProvider \u4f7f\u7528 HDFS shell \u8bbf\u95ee OSS/OSS-HDFS bucket, \u82e5\u80fd\u6b63\u5e38\u8bbf\u95ee\uff0c\u5219\u786e\u8ba4\u514d\u5bc6\u6b63\u5e38\u3002","title":"\u65b9\u6cd5\u4e8c\uff1a"},{"location":"jindosdk/jindosdk_credential_provider_faq/#3-5-6-ossoss-hdfs","text":"\u67d0\u4e9b\u7248\u672c\u6709\u5df2\u77e5\u514d\u5bc6\u95ee\u9898\uff0c\u8bf7\u67e5\u770b \u5df2\u77e5\u95ee\u9898","title":"3. \u5e38\u9a7b\u670d\u52a1\u6bcf 5-6 \u4e2a\u5c0f\u65f6\u8bbf\u95ee OSS/OSS-HDFS \u5931\u8d25"},{"location":"jindosdk/jindosdk_credential_provider_faq/#ak","text":"\u914d\u7f6e\u4f7f\u7528 SimpleCredentialsProvider , \u8bf7\u53c2\u8003 \u914d\u7f6e OSS/OSS-HDFS Credential Provider","title":"\u65b9\u6cd5\u4e00\uff1a\u4f7f\u7528\u56fa\u5b9a AK"},{"location":"jindosdk/jindosdk_credential_provider_faq/#jindosdk","text":"\u65e7\u7248\u63a7\u5236\u53f0\u5347\u7ea7\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b \u65b0\u7248\u63a7\u5236\u53f0\u5347\u7ea7\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b E-MapReduce EMR-5.5.0/EMR-3.39.0 \u4ee5\u524d\u7248\u672c\u7684\u96c6\u7fa4\uff0c\u8bf7\u53c2\u8003 EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b","title":"\u65b9\u6cd5\u4e8c\uff1a\u5347\u7ea7 JindoSDK \u4e3a\u6700\u65b0\u7248\u672c"},{"location":"jindosdk/jindosdk_credential_provider_faq/#4-ossoss-hdfs-ak","text":"\u62a5\u9519\u4fe1\u606f\u5982\u4e0b\uff1a The Filesystem URI contains login details. This authentication mechanism is no longer supported. \u4e3a\u4e86\u9632\u6b62 AK \u6cc4\u9732\u7b49\u5b89\u5168\u95ee\u9898\uff0cJindoSDK 4.0.0 \u5f00\u59cb\u4e0d\u652f\u6301\u8def\u5f84\u4e2d\u643a\u5e26 AK(AccessKeyId,AccessKeySecret\u7b49\uff09\u4fe1\u606f\u3002","title":"4. \u8bbf\u95ee OSS/OSS-HDFS \u8def\u5f84\u4e2d\u5305\u542b AK \u4fe1\u606f\u62a5\u9519"},{"location":"jindosdk/jindosdk_credential_provider_faq/#_3","text":"\u8bbf\u95ee\u8def\u5f84\u4e2d\u53bb\u9664 AK \u4fe1\u606f\uff1b","title":"\u65b9\u6cd5\u4e00\uff1a"},{"location":"jindosdk/jindosdk_credential_provider_faq/#_4","text":"\u5982\u679c\u4e00\u5b9a\u8981\u8fd9\u6837\u4f7f\u7528\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u53c2\u6570\u5f00\u542f\u3002\u5728 Hadoop-Common \u9875\u9762\u9009\u62e9 \u914d\u7f6e > core-site.xml \u4e2d\u65b0\u589e\u4e0b\u5217\u53c2\u6570\uff1a \u53c2\u6570 \u503c fs.oss.uri-with-secrets.enable true","title":"\u65b9\u6cd5\u4e8c\uff1a"},{"location":"jindosdk/jindosdk_deployment_ai/","text":"AI \u573a\u666f\u90e8\u7f72 JindoSDK \u00a7 \u5728\u4f7f\u7528 JindoFuse\uff0cJindo CLI\uff0cPyJindo \u7b49\u975e Hadoop \u751f\u6001\u7ec4\u4ef6\u65f6\uff0c\u4f1a\u8bbf\u95ee\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6240\u5728\u7684\u76ee\u5f55\u8bfb\u53d6\u914d\u7f6e\u6587\u4ef6\u3002 \u90e8\u7f72 JindoSDK \u00a7 \u4e0b\u8f7d\u5e76\u89e3\u538b JindoSDK TAR \u5305\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u4e0b\u8f7d6.7.3\u7248\u672cJindoSDK TAR\u5305\u3002\u4ee5\u5927\u591a\u6570linux x86\u73af\u5883\u4e3a\u4f8b\u3002\u5176\u4ed6\u5e73\u53f0\u90e8\u7f72\u53c2\u89c1 \u300a\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK\u300b wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u89e3\u538b JindoSDK TAR \u5305\u3002 tar zxvf jindosdk-6.7.3-linux.tar.gz \u914d\u7f6e\u73af\u5883\u53d8\u91cf\u3002 \u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u89e3\u538b\u5728 /usr/lib/jindosdk-6.7.3-linux \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_HOME=/usr/lib/jindosdk-6.7.3-linux export JINDOSDK_CONF_DIR=${JINDOSDK_HOME}/conf export PATH=${PATH}:${JINDOSDK_HOME}/bin export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${JINDOSDK_HOME}/lib/native \u91cd\u8981 \u8bf7\u5c06\u5b89\u88c5\u76ee\u5f55\u548c\u73af\u5883\u53d8\u91cf\u90e8\u7f72\u5230\u6240\u6709\u6240\u9700\u8282\u70b9\u4e0a\u3002 \u4fee\u6539\u914d\u7f6e \u00a7 \u914d\u7f6e\u6587\u4ef6\u4f4d\u4e8e JINDOSDK_CONF_DIR \u6240\u5728\u76ee\u5f55\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\u3002 cfg \u914d\u7f6e\u6587\u4ef6 \u00a7 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/jindosdk-log [jindosdk] # \u5df2\u521b\u5efa OSS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3a oss-cn-hangzhou.aliyuncs.com\u3002 # \u5df2\u521b\u5efa OSS-HDFS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3a cn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95eeOSS\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> \u514d\u5bc6\u8bbf\u95ee \u00a7 \u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7 RAM\u89d2\u8272\u6388\u6743 \u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/jindosdk-log [jindosdk] # \u5df2\u521b\u5efa OSS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3a oss-cn-hangzhou.aliyuncs.com\u3002 # \u5df2\u521b\u5efa OSS-HDFS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3a cn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE fs.oss.provider.format = JSON","title":"AI \u573a\u666f\u90e8\u7f72"},{"location":"jindosdk/jindosdk_deployment_ai/#ai-jindosdk","text":"\u5728\u4f7f\u7528 JindoFuse\uff0cJindo CLI\uff0cPyJindo \u7b49\u975e Hadoop \u751f\u6001\u7ec4\u4ef6\u65f6\uff0c\u4f1a\u8bbf\u95ee\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6240\u5728\u7684\u76ee\u5f55\u8bfb\u53d6\u914d\u7f6e\u6587\u4ef6\u3002","title":"AI \u573a\u666f\u90e8\u7f72 JindoSDK"},{"location":"jindosdk/jindosdk_deployment_ai/#jindosdk","text":"\u4e0b\u8f7d\u5e76\u89e3\u538b JindoSDK TAR \u5305\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u4e0b\u8f7d6.7.3\u7248\u672cJindoSDK TAR\u5305\u3002\u4ee5\u5927\u591a\u6570linux x86\u73af\u5883\u4e3a\u4f8b\u3002\u5176\u4ed6\u5e73\u53f0\u90e8\u7f72\u53c2\u89c1 \u300a\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK\u300b wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u89e3\u538b JindoSDK TAR \u5305\u3002 tar zxvf jindosdk-6.7.3-linux.tar.gz \u914d\u7f6e\u73af\u5883\u53d8\u91cf\u3002 \u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u89e3\u538b\u5728 /usr/lib/jindosdk-6.7.3-linux \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_HOME=/usr/lib/jindosdk-6.7.3-linux export JINDOSDK_CONF_DIR=${JINDOSDK_HOME}/conf export PATH=${PATH}:${JINDOSDK_HOME}/bin export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${JINDOSDK_HOME}/lib/native \u91cd\u8981 \u8bf7\u5c06\u5b89\u88c5\u76ee\u5f55\u548c\u73af\u5883\u53d8\u91cf\u90e8\u7f72\u5230\u6240\u6709\u6240\u9700\u8282\u70b9\u4e0a\u3002","title":"\u90e8\u7f72 JindoSDK"},{"location":"jindosdk/jindosdk_deployment_ai/#_1","text":"\u914d\u7f6e\u6587\u4ef6\u4f4d\u4e8e JINDOSDK_CONF_DIR \u6240\u5728\u76ee\u5f55\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \uff0c\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\u3002","title":"\u4fee\u6539\u914d\u7f6e"},{"location":"jindosdk/jindosdk_deployment_ai/#cfg","text":"\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/jindosdk-log [jindosdk] # \u5df2\u521b\u5efa OSS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3a oss-cn-hangzhou.aliyuncs.com\u3002 # \u5df2\u521b\u5efa OSS-HDFS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3a cn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95eeOSS\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret>","title":"cfg \u914d\u7f6e\u6587\u4ef6"},{"location":"jindosdk/jindosdk_deployment_ai/#_2","text":"\u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7 RAM\u89d2\u8272\u6388\u6743 \u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/jindosdk-log [jindosdk] # \u5df2\u521b\u5efa OSS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3a oss-cn-hangzhou.aliyuncs.com\u3002 # \u5df2\u521b\u5efa OSS-HDFS Bucket \u5bf9\u5e94\u7684 Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3a cn-hangzhou.oss-dls.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE fs.oss.provider.format = JSON","title":"\u514d\u5bc6\u8bbf\u95ee"},{"location":"jindosdk/jindosdk_deployment_hadoop/","text":"\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK \u00a7 \u90e8\u7f72 JindoSDK \u00a7 \u4e0b\u8f7d\u5e76\u89e3\u538b JindoSDK TAR \u5305\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u4e0b\u8f7d6.7.3\u7248\u672cJindoSDK TAR\u5305\u3002\u4ee5\u5927\u591a\u6570linux x86\u73af\u5883\u4e3a\u4f8b\u3002\u5176\u4ed6\u5e73\u53f0\u90e8\u7f72\u53c2\u89c1 \u300a\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK\u300b wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u89e3\u538b JindoSDK TAR \u5305\u3002 tar zxvf jindosdk-6.7.3-linux.tar.gz \u914d\u7f6e\u73af\u5883\u53d8\u91cf\u3002 \u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u89e3\u538b\u5728 /usr/lib/jindosdk-6.7.3-linux \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_HOME=/usr/lib/jindosdk-6.7.3-linux export JINDOSDK_CONF_DIR=${JINDOSDK_HOME}/conf export PATH=${PATH}:${JINDOSDK_HOME}/bin export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${JINDOSDK_HOME}/lib/native export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:${JINDOSDK_HOME}/lib/* \u91cd\u8981 \u8bf7\u5c06\u5b89\u88c5\u76ee\u5f55\u548c\u73af\u5883\u53d8\u91cf\u90e8\u7f72\u5230\u6240\u6709\u6240\u9700\u8282\u70b9\u4e0a\u3002 Hadoop \u914d\u7f6e\u6587\u4ef6 \u00a7 \u914d\u7f6e OSS/OSS-HDFS \u5b9e\u73b0\u7c7b\u53caAccessKey\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u8fdb\u5165Hadoop\u7684core-site.xml\u914d\u7f6e\u6587\u4ef6\u3002 vim <HADOOP_HOME>/etc/hadoop/core-site.xml \u5c06 OSS/OSS-HDFS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230Hadoop\u7684core-site.xml\u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06 OSS/OSS-HDFS \u7684Bucket\u5bf9\u5e94\u7684accessKeyId\u3001accessKeySecret\u9884\u5148\u914d\u7f6e\u5728Hadoop\u7684core-site.xml\u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS \u670d\u52a1 Endpoint\u3002 \u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u66f4\u591a\u914d\u7f6e\u65b9\u5f0f\u3002 \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u6309 bucket \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u8bbf\u95ee OSS/OSS-HDFS \u65f6 AK \u76f8\u5173\u5e38\u89c1\u95ee\u9898","title":"Hadoop \u73af\u5883\u5b8c\u6574\u90e8\u7f72"},{"location":"jindosdk/jindosdk_deployment_hadoop/#hadoop-jindosdk","text":"","title":"\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK"},{"location":"jindosdk/jindosdk_deployment_hadoop/#jindosdk","text":"\u4e0b\u8f7d\u5e76\u89e3\u538b JindoSDK TAR \u5305\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u4e0b\u8f7d6.7.3\u7248\u672cJindoSDK TAR\u5305\u3002\u4ee5\u5927\u591a\u6570linux x86\u73af\u5883\u4e3a\u4f8b\u3002\u5176\u4ed6\u5e73\u53f0\u90e8\u7f72\u53c2\u89c1 \u300a\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK\u300b wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u89e3\u538b JindoSDK TAR \u5305\u3002 tar zxvf jindosdk-6.7.3-linux.tar.gz \u914d\u7f6e\u73af\u5883\u53d8\u91cf\u3002 \u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u89e3\u538b\u5728 /usr/lib/jindosdk-6.7.3-linux \u76ee\u5f55\u4e3a\u4f8b\uff1a export JINDOSDK_HOME=/usr/lib/jindosdk-6.7.3-linux export JINDOSDK_CONF_DIR=${JINDOSDK_HOME}/conf export PATH=${PATH}:${JINDOSDK_HOME}/bin export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${JINDOSDK_HOME}/lib/native export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:${JINDOSDK_HOME}/lib/* \u91cd\u8981 \u8bf7\u5c06\u5b89\u88c5\u76ee\u5f55\u548c\u73af\u5883\u53d8\u91cf\u90e8\u7f72\u5230\u6240\u6709\u6240\u9700\u8282\u70b9\u4e0a\u3002","title":"\u90e8\u7f72 JindoSDK"},{"location":"jindosdk/jindosdk_deployment_hadoop/#hadoop","text":"\u914d\u7f6e OSS/OSS-HDFS \u5b9e\u73b0\u7c7b\u53caAccessKey\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u8fdb\u5165Hadoop\u7684core-site.xml\u914d\u7f6e\u6587\u4ef6\u3002 vim <HADOOP_HOME>/etc/hadoop/core-site.xml \u5c06 OSS/OSS-HDFS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230Hadoop\u7684core-site.xml\u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06 OSS/OSS-HDFS \u7684Bucket\u5bf9\u5e94\u7684accessKeyId\u3001accessKeySecret\u9884\u5148\u914d\u7f6e\u5728Hadoop\u7684core-site.xml\u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS \u670d\u52a1 Endpoint\u3002 \u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u66f4\u591a\u914d\u7f6e\u65b9\u5f0f\u3002 \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u6309 bucket \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u8bbf\u95ee OSS/OSS-HDFS \u65f6 AK \u76f8\u5173\u5e38\u89c1\u95ee\u9898","title":"Hadoop \u914d\u7f6e\u6587\u4ef6"},{"location":"jindosdk/jindosdk_deployment_lite_hadoop/","text":"\u5728 Hadoop \u73af\u5883\u6700\u7b80\u5b89\u88c5\u90e8\u7f72 JindoSDK \u00a7 \u90e8\u7f72 JindoSDK \u00a7 \u4e0b\u8f7d JindoSDK JAR \u5305\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u4e0b\u8f7d6.7.3\u7248\u672cJindoSDK JAR\u5305\u3002\u4ee5\u5927\u591a\u6570linux x86\u73af\u5883\u4e3a\u4f8b\u3002\u5176\u4ed6\u5e73\u53f0\u90e8\u7f72\u53c2\u89c1 \u300a\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK\u300b wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-sdk/6.7.3/jindo-sdk-6.7.3.jar wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-core/6.7.3/jindo-core-6.7.3.jar \u5b89\u88c5 JindoSDK JAR \u5305 \u5c06\u4e0b\u8f7d\u7684 jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a jindo-core-x.x.x.jar jindo-sdk-x.x.x.jar \u4ee5 jindosdk-6.7.3 \u4e3a\u4f8b: cp jindo-core-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ \u91cd\u8981 \u8bf7\u5c06\u5b89\u88c5\u76ee\u5f55\u548c\u73af\u5883\u53d8\u91cf\u90e8\u7f72\u5230\u6240\u6709\u6240\u9700\u8282\u70b9\u4e0a\u3002 Hadoop \u914d\u7f6e\u6587\u4ef6 \u00a7 \u914d\u7f6e OSS/OSS-HDFS \u5b9e\u73b0\u7c7b\u53caAccessKey\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u8fdb\u5165Hadoop\u7684core-site.xml\u914d\u7f6e\u6587\u4ef6\u3002 vim <HADOOP_HOME>/etc/hadoop/core-site.xml \u5c06 OSS/OSS-HDFS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230Hadoop\u7684core-site.xml\u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06 OSS/OSS-HDFS \u7684Bucket\u5bf9\u5e94\u7684accessKeyId\u3001accessKeySecret\u9884\u5148\u914d\u7f6e\u5728Hadoop\u7684core-site.xml\u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS \u670d\u52a1 Endpoint\u3002 \u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u66f4\u591a\u914d\u7f6e\u65b9\u5f0f\u3002 \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u6309 bucket \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u8bbf\u95ee OSS/OSS-HDFS \u65f6 AK \u76f8\u5173\u5e38\u89c1\u95ee\u9898","title":"Hadoop \u73af\u5883\u4ec5\u90e8\u7f72 Jar \u5305"},{"location":"jindosdk/jindosdk_deployment_lite_hadoop/#hadoop-jindosdk","text":"","title":"\u5728 Hadoop \u73af\u5883\u6700\u7b80\u5b89\u88c5\u90e8\u7f72 JindoSDK"},{"location":"jindosdk/jindosdk_deployment_lite_hadoop/#jindosdk","text":"\u4e0b\u8f7d JindoSDK JAR \u5305\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u4e0b\u8f7d6.7.3\u7248\u672cJindoSDK JAR\u5305\u3002\u4ee5\u5927\u591a\u6570linux x86\u73af\u5883\u4e3a\u4f8b\u3002\u5176\u4ed6\u5e73\u53f0\u90e8\u7f72\u53c2\u89c1 \u300a\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK\u300b wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-sdk/6.7.3/jindo-sdk-6.7.3.jar wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-core/6.7.3/jindo-core-6.7.3.jar \u5b89\u88c5 JindoSDK JAR \u5305 \u5c06\u4e0b\u8f7d\u7684 jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a jindo-core-x.x.x.jar jindo-sdk-x.x.x.jar \u4ee5 jindosdk-6.7.3 \u4e3a\u4f8b: cp jindo-core-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ \u91cd\u8981 \u8bf7\u5c06\u5b89\u88c5\u76ee\u5f55\u548c\u73af\u5883\u53d8\u91cf\u90e8\u7f72\u5230\u6240\u6709\u6240\u9700\u8282\u70b9\u4e0a\u3002","title":"\u90e8\u7f72 JindoSDK"},{"location":"jindosdk/jindosdk_deployment_lite_hadoop/#hadoop","text":"\u914d\u7f6e OSS/OSS-HDFS \u5b9e\u73b0\u7c7b\u53caAccessKey\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u8fdb\u5165Hadoop\u7684core-site.xml\u914d\u7f6e\u6587\u4ef6\u3002 vim <HADOOP_HOME>/etc/hadoop/core-site.xml \u5c06 OSS/OSS-HDFS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230Hadoop\u7684core-site.xml\u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06 OSS/OSS-HDFS \u7684Bucket\u5bf9\u5e94\u7684accessKeyId\u3001accessKeySecret\u9884\u5148\u914d\u7f6e\u5728Hadoop\u7684core-site.xml\u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> \u914d\u7f6e OSS/OSS-HDFS \u670d\u52a1 Endpoint\u3002 \u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u66f4\u591a\u914d\u7f6e\u65b9\u5f0f\u3002 \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u6309 bucket \u914d\u7f6e OSS/OSS-HDFS Credential Provider \u8bbf\u95ee OSS/OSS-HDFS \u65f6 AK \u76f8\u5173\u5e38\u89c1\u95ee\u9898","title":"Hadoop \u914d\u7f6e\u6587\u4ef6"},{"location":"jindosdk/jindosdk_deployment_multi_platform/","text":"\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK \u00a7 JindoSDK \u591a\u5e73\u53f0\u652f\u6301\u73b0\u72b6 \u00a7 JindoSDK \u76ee\u524d\u652f\u6301\u5927\u90e8\u5206\u5e38\u89c1\u7684 Linux \u53d1\u884c\u7248\uff0c\u5e76\u652f\u6301 x86/aarch64 \u5e73\u53f0\u3002 \u652f\u6301 MacOS \u64cd\u4f5c\u7cfb\u7edf\uff0c\u6240\u6709\u7248\u672c\u5747\u652f\u6301 x86 \u5e73\u53f0\uff0c\u90e8\u5206\u7248\u672c\u652f\u6301 M1/M2 \u82af\u7247\u7684 arm \u5e73\u53f0\u3002 \u6682\u4e0d\u652f\u6301 Windows \u64cd\u4f5c\u7cfb\u7edf\u3002 \u6682\u4e0d\u652f\u6301\u672a\u517c\u5bb9 glibc \u7684 linux \u53d1\u884c\u7248\uff0c\u5982\u9ed8\u8ba4\u4f7f\u7528 musl \u7684\u53d1\u884c\u7248\u3002 \u4ea7\u51fa\u7269 \u00a7 \u4ea7\u51fa\u7269\u5305\u542b\u4e24\u7c7b\uff1a\u5b8c\u6574\u4ea7\u51fa\u7269\u3001lite\u4ea7\u51fa\u7269\u3002\u4ee5\u4e0b x.y.z \u8868\u793a\u7248\u672c\u53f7\u3002 \u5b8c\u6574\u4ea7\u51fa\u7269\u8bf4\u660e \u00a7 \u5728 Aliyun EMR \u4e2d\u9ed8\u8ba4\u5b89\u88c5\u4e86\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u5b89\u88c5\u8def\u5f84\u4e3a/opt/apps/JINDOSDK/jindosdk-current/\u3002 \u5b8c\u6574\u4ea7\u51fa\u7269\u662f\u4e00\u4e2atar\u5305\uff0c\u540d\u79f0\u4e3a jindosdk-x.y.z-<\u5e73\u53f0\u540d\u79f0>.tar.gz \u3002 \u5b8c\u6574\u4ea7\u51fa\u7269\u5305\u542b hadoop sdk\uff0cjindo-fuse, jindo-distcp, jindotable\uff0cjindo flink connector, jindocli \u7b49\u5de5\u5177\uff0c\u53ef\u4ee5\u90e8\u7f72\u4f7f\u7528\u3002\u4e5f\u5305\u542b\u4e8c\u6b21\u5f00\u53d1\u6240\u9700\u7684so\u7b49\u4f9d\u8d56\u3002 \u5982\u679c\u5e0c\u671b\u5b89\u88c5\u5b8c\u6574\u7684 JindoSDK\uff0c\u53ef\u4ee5\u90e8\u7f72\u5b89\u88c5\u8fd9\u7c7b\u4ea7\u51fa\u7269\uff0c\u4f46\u8be5\u4ea7\u51fa\u7269\u4e00\u822c\u8f83\u5927\u3002 lite \u4ea7\u51fa\u7269 \u00a7 \u5982\u679c\u5e0c\u671b\u5728 Hadoop \u73af\u5883\u4e2d\u4ec5\u4f7f\u7528 JindoSDK \u4e2d\u7684 hadoop sdk\uff0c\u53ef\u4ee5\u4f7f\u7528\u6b64\u7c7b\u4ea7\u51fa\u7269\u3002 \u4f7f\u7528Java SDK\u65f6\uff0cLinux X86\u5e73\u53f0\u9700\u8981\u4f9d\u8d56 jindo-sdk-x.y.z.jar \u548c jindo-core-x.y.z.jar \uff0c\u5176\u4ed6\u5e73\u53f0\u9700\u8981\u4f9d\u8d56 jindo-sdk-x.y.z.jar \u3001 jindo-core-x.y.z.jar \u548c jindo-core-<\u5e73\u53f0\u540d\u79f0>-x.y.z .jar\u3002 \u5b89\u88c5\u65b9\u5f0f\u7684\u8bdd\uff0c\u5373\u53ef\u4ee5\u901a\u8fc7maven\u5728pom.xml\u4e2d\u914d\u7f6e\u4f9d\u8d56\uff0c\u5982 6.7.3-oss-maven \u3002\u4e5f\u53ef\u4ee5\u901a\u8fc7oss\u5730\u5740\u76f4\u63a5\u4e0b\u8f7d\u90e8\u7f72\uff0c\u5982 6.7.3-download \u3002 \u4ea7\u7269\u7269\u5e73\u53f0\u8bf4\u660e\u8868 \u00a7 \u4ea7\u51fa\u7269\u540d\u79f0 \u8bf4\u660e jindosdk-x.y.z-linux.tar.gz \u652f\u6301\u5927\u591a\u6570 linux x86 \u53d1\u884c\u7248\u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.17+ \u7248\u672c jindosdk-x.y.z-linux-el7-aarch64.tar.gz \u652f\u6301\u5927\u591a\u6570 linux aarch64 \u53d1\u884c\u7248\u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.17+ \u7248\u672c jindosdk-x.y.z-linux-el6-x86_64.tar.gz \u652f\u6301 centos6 x86 \u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.12+ \u7248\u672c jindosdk-x.y.z-linux-ubuntu22-x86_64.tar.gz \u652f\u6301 ubuntu22 x86 \u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.34+ \u7248\u672c jindosdk-x.y.z-macos-11_0-x86_64.tar.gz \u652f\u6301 macos x86 \u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 macos 11.0 \u4ee5\u4e0a\u7248\u672c jindosdk-x.y.z-macos-11_0-aarch64.tar.gz \u652f\u6301 macos m1/m2 \u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 macos 11.0 \u4ee5\u4e0a\u7248\u672c jindo-sdk-x.y.z.jar \u652f\u6301\u6240\u6709\u5e73\u53f0\u7684 lite \u4ea7\u51fa\u7269\uff0c\u7eaf java \u5b9e\u73b0\uff0c\u4e0d\u5305\u542b native \u5b9e\u73b0 jindo-core-x.y.z.jar \u652f\u6301\u5927\u591a\u6570 linux x86 \u53d1\u884c\u7248\u7684 lite \u4ea7\u51fa\u7269\uff0c\u5305\u542b native \u5b9e\u73b0\uff0c\u517c\u5bb9 glibc 2.17+ \u7248\u672c jindo-core-linux-el7-aarch64-x.y.z.jar \u652f\u6301\u5927\u591a\u6570 linux aarch64 \u53d1\u884c\u7248\u7684 lite \u4ea7\u51fa\u7269\uff0c\u5305\u542b native \u5b9e\u73b0\uff0c\u517c\u5bb9 glibc 2.17+ \u7248\u672c jindo-core-linux-el6-x86_64-x.y.z.jar \u652f\u6301 centos6 x86 \u7684 lite \u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.12+ \u7248\u672c jindo-core-linux-ubuntu22-x86_64-x.y.z.jar \u652f\u6301 ubuntu22 x86 \u7684 lite \u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.34+ \u7248\u672c jindo-core-macos-11_0-x86_64-x.y.z.jar \u652f\u6301 macos x86 \u7684 lite \u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 macos 11.0 \u4ee5\u4e0a\u7248\u672c jindo-core-macos-11_0-aarch64-x.y.z.jar \u652f\u6301 macos m1/m2 \u7684 lite \u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 macos 11.0 \u4ee5\u4e0a\u7248\u672c ps: 6.5.0 \u7248\u672c\u5f00\u59cb tar \u5305\u4e2d\u7684 lite jar \u9ed8\u8ba4\u4f7f\u7528 nextarch classifer\uff0c\u5373 jindo-core-x.y.z-nextarch.jar\u3001jindo-core-linux-el7-aarch64-x.y.z-nextarch.jar \u3002 JindoSDK \u90e8\u7f72\u793a\u4f8b \u00a7 \u4ee5\u4e0b 6.7.3 \u7248\u672c\u4e3a\u4f8b\u3002 Linux x86 \u73af\u5883\u793a\u4f8b \u00a7 \u5b8c\u6574\u5b89\u88c5\uff0c\u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u89e3\u538b\u5728 /usr/lib/jindosdk-6.7.3-linux \u76ee\u5f55\u4e3a\u4f8b\uff1a wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz tar zxvf jindosdk-6.7.3-linux.tar.gz -C /usr/lib/ export JINDOSDK_HOME=/usr/lib/jindosdk-6.7.3-linux export JINDOSDK_CONF_DIR=${JINDOSDK_HOME}/conf export PATH=${PATH}:${JINDOSDK_HOME}/bin export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${JINDOSDK_HOME}/lib/native export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:${JINDOSDK_HOME}/lib/* lite\u5b89\u88c5\uff0c\u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u4e0b\u8f7d\u5230 /share/hadoop/hdfs/lib/ \u76ee\u5f55\u4e3a\u4f8b\uff1a wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-sdk/6.7.3/jindo-sdk-6.7.3.jar wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-core/6.7.3/jindo-core-6.7.3.jar cp jindo-core-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ Linux aarch64 \u73af\u5883\u793a\u4f8b \u00a7 \u5b8c\u6574\u5b89\u88c5\uff0c\u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u89e3\u538b\u5728 /usr/lib/jindosdk-6.7.3-linux-el7-aarch64 \u76ee\u5f55\u4e3a\u4f8b\uff1a wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/x.y.z/jindosdk-6.7.3-linux-el7-aarch64.tar.gz tar zxvf jindosdk-6.7.3-linux-el7-aarch64.tar.gz -C /usr/lib/ export JINDOSDK_HOME=/usr/lib/jindosdk-6.7.3-linux-el7-aarch64 export JINDOSDK_CONF_DIR=${JINDOSDK_HOME}/conf export PATH=${PATH}:${JINDOSDK_HOME}/bin export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${JINDOSDK_HOME}/lib/native export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:${JINDOSDK_HOME}/lib/* lite\u5b89\u88c5\uff0c\u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u4e0b\u8f7d\u5230 /share/hadoop/hdfs/lib/ \u76ee\u5f55\u4e3a\u4f8b\uff1a wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-sdk/6.7.3/jindo-sdk-6.7.3.jar wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-core-linux-el7-aarch64/6.7.3/jindo-core-linux-el7-aarch64-6.7.3.jar cp jindo-core-linux-el7-aarch64-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindo-core-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ \u9644\u5f55 \u00a7 \u5b8c\u6574\u4ea7\u51fa\u7269\u793a\u4f8b \u00a7 \u4ee5\u4e0b x.y.z \u8868\u793a\u7248\u672c\u53f7\u3002 . \u251c\u2500\u2500 bin \u2502 \u251c\u2500\u2500 fusermount3 \u2502 \u251c\u2500\u2500 jindo \u2502 \u251c\u2500\u2500 jindo-fuse \u2502 \u251c\u2500\u2500 jindobench \u2502 \u251c\u2500\u2500 jindodiag \u2502 \u251c\u2500\u2500 jindosync \u2502 \u2514\u2500\u2500 jindotable \u251c\u2500\u2500 conf \u2502 \u251c\u2500\u2500 core-site.xml.template \u2502 \u251c\u2500\u2500 jindosdk.cfg.template \u2502 \u2514\u2500\u2500 log4j.properties.template \u251c\u2500\u2500 include \u2502 \u251c\u2500\u2500 jdo_api.h \u2502 \u251c\u2500\u2500 jdo_common.h \u2502 \u251c\u2500\u2500 jdo_content_summary.h \u2502 \u251c\u2500\u2500 jdo_data_types.h \u2502 \u251c\u2500\u2500 jdo_defines.h \u2502 \u251c\u2500\u2500 jdo_error.h \u2502 \u251c\u2500\u2500 jdo_file_status.h \u2502 \u251c\u2500\u2500 jdo_list_dir_result.h \u2502 \u251c\u2500\u2500 jdo_option_keys.h \u2502 \u2514\u2500\u2500 jdo_options.h \u251c\u2500\u2500 lib \u2502 \u251c\u2500\u2500 jindo-core-x.y.z-nextarch.jar \u2502 \u251c\u2500\u2500 jindo-core-linux-el7-aarch64-x.y.z-nextarch.jar \u2502 \u251c\u2500\u2500 jindo-sdk-x.y.z-nextarch.jar \u2502 \u251c\u2500\u2500 native \u2502 \u2502 \u251c\u2500\u2500 libfuse3.so.3 \u2502 \u2502 \u251c\u2500\u2500 libjemalloc.so \u2502 \u2502 \u251c\u2500\u2500 libjindo-csdk.so \u2502 \u2502 \u251c\u2500\u2500 libjindo-tensorflow1.15.so \u2502 \u2502 \u251c\u2500\u2500 libjindo-tensorflow2.8.so \u2502 \u2502 \u251c\u2500\u2500 libjindosdk.so \u2502 \u2502 \u251c\u2500\u2500 libjindosdk_c.so -> libjindosdk_c.so.x \u2502 \u2502 \u251c\u2500\u2500 libjindosdk_c.so.x -> libjindosdk_c.so.x.y.z \u2502 \u2502 \u2514\u2500\u2500 libjindosdk_c.so.x.y.z \u2502 \u2514\u2500\u2500 site-packages \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp310-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp311-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp312-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp36-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp37-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp38-abi3-linux_x86_64.whl \u2502 \u2514\u2500\u2500 pyjindo-x.y.z-cp39-abi3-linux_x86_64.whl \u251c\u2500\u2500 plugins \u2502 \u2514\u2500\u2500 libjindo-auth-client-plugin-nextarch.so \u2502 \u2514\u2500\u2500 flink \u2502 \u251c\u2500\u2500 jindo-flink-x.y.z-full.jar \u2502 \u2514\u2500\u2500 jindo-flink-x.y.z.jar \u2502 \u251c\u2500\u2500 spark2 \u2502 \u2502 \u2514\u2500\u2500 jindo-spark2-x.y.z.jar \u2502 \u2514\u2500\u2500 spark3 \u2502 \u2514\u2500\u2500 jindo-spark3-x.y.z.jar \u251c\u2500\u2500 tools \u2502 \u251c\u2500\u2500 jindo-dependence-shaded-x.y.z.jar \u2502 \u251c\u2500\u2500 jindo-distcp-tool-x.y.z.jar \u2502 \u251c\u2500\u2500 jindo-distjob-tool-x.y.z.jar \u2502 \u251c\u2500\u2500 jindotable-hive-tool-x.y.z.jar \u2502 \u2514\u2500\u2500 jindotable-shell-x.y.z.jar \u2514\u2500\u2500 versions \u251c\u2500\u2500 JINDOSDK_VERSION \u251c\u2500\u2500 JINDOTABLE_VERSION \u2514\u2500\u2500 JINDO_CONNECTORS_VERSION 9 directories, 29 files","title":"\u591a\u5e73\u53f0\u90e8\u7f72"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#jindosdk","text":"","title":"\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#jindosdk_1","text":"JindoSDK \u76ee\u524d\u652f\u6301\u5927\u90e8\u5206\u5e38\u89c1\u7684 Linux \u53d1\u884c\u7248\uff0c\u5e76\u652f\u6301 x86/aarch64 \u5e73\u53f0\u3002 \u652f\u6301 MacOS \u64cd\u4f5c\u7cfb\u7edf\uff0c\u6240\u6709\u7248\u672c\u5747\u652f\u6301 x86 \u5e73\u53f0\uff0c\u90e8\u5206\u7248\u672c\u652f\u6301 M1/M2 \u82af\u7247\u7684 arm \u5e73\u53f0\u3002 \u6682\u4e0d\u652f\u6301 Windows \u64cd\u4f5c\u7cfb\u7edf\u3002 \u6682\u4e0d\u652f\u6301\u672a\u517c\u5bb9 glibc \u7684 linux \u53d1\u884c\u7248\uff0c\u5982\u9ed8\u8ba4\u4f7f\u7528 musl \u7684\u53d1\u884c\u7248\u3002","title":"JindoSDK \u591a\u5e73\u53f0\u652f\u6301\u73b0\u72b6"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#_1","text":"\u4ea7\u51fa\u7269\u5305\u542b\u4e24\u7c7b\uff1a\u5b8c\u6574\u4ea7\u51fa\u7269\u3001lite\u4ea7\u51fa\u7269\u3002\u4ee5\u4e0b x.y.z \u8868\u793a\u7248\u672c\u53f7\u3002","title":"\u4ea7\u51fa\u7269"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#_2","text":"\u5728 Aliyun EMR \u4e2d\u9ed8\u8ba4\u5b89\u88c5\u4e86\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u5b89\u88c5\u8def\u5f84\u4e3a/opt/apps/JINDOSDK/jindosdk-current/\u3002 \u5b8c\u6574\u4ea7\u51fa\u7269\u662f\u4e00\u4e2atar\u5305\uff0c\u540d\u79f0\u4e3a jindosdk-x.y.z-<\u5e73\u53f0\u540d\u79f0>.tar.gz \u3002 \u5b8c\u6574\u4ea7\u51fa\u7269\u5305\u542b hadoop sdk\uff0cjindo-fuse, jindo-distcp, jindotable\uff0cjindo flink connector, jindocli \u7b49\u5de5\u5177\uff0c\u53ef\u4ee5\u90e8\u7f72\u4f7f\u7528\u3002\u4e5f\u5305\u542b\u4e8c\u6b21\u5f00\u53d1\u6240\u9700\u7684so\u7b49\u4f9d\u8d56\u3002 \u5982\u679c\u5e0c\u671b\u5b89\u88c5\u5b8c\u6574\u7684 JindoSDK\uff0c\u53ef\u4ee5\u90e8\u7f72\u5b89\u88c5\u8fd9\u7c7b\u4ea7\u51fa\u7269\uff0c\u4f46\u8be5\u4ea7\u51fa\u7269\u4e00\u822c\u8f83\u5927\u3002","title":"\u5b8c\u6574\u4ea7\u51fa\u7269\u8bf4\u660e"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#lite","text":"\u5982\u679c\u5e0c\u671b\u5728 Hadoop \u73af\u5883\u4e2d\u4ec5\u4f7f\u7528 JindoSDK \u4e2d\u7684 hadoop sdk\uff0c\u53ef\u4ee5\u4f7f\u7528\u6b64\u7c7b\u4ea7\u51fa\u7269\u3002 \u4f7f\u7528Java SDK\u65f6\uff0cLinux X86\u5e73\u53f0\u9700\u8981\u4f9d\u8d56 jindo-sdk-x.y.z.jar \u548c jindo-core-x.y.z.jar \uff0c\u5176\u4ed6\u5e73\u53f0\u9700\u8981\u4f9d\u8d56 jindo-sdk-x.y.z.jar \u3001 jindo-core-x.y.z.jar \u548c jindo-core-<\u5e73\u53f0\u540d\u79f0>-x.y.z .jar\u3002 \u5b89\u88c5\u65b9\u5f0f\u7684\u8bdd\uff0c\u5373\u53ef\u4ee5\u901a\u8fc7maven\u5728pom.xml\u4e2d\u914d\u7f6e\u4f9d\u8d56\uff0c\u5982 6.7.3-oss-maven \u3002\u4e5f\u53ef\u4ee5\u901a\u8fc7oss\u5730\u5740\u76f4\u63a5\u4e0b\u8f7d\u90e8\u7f72\uff0c\u5982 6.7.3-download \u3002","title":"lite \u4ea7\u51fa\u7269"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#_3","text":"\u4ea7\u51fa\u7269\u540d\u79f0 \u8bf4\u660e jindosdk-x.y.z-linux.tar.gz \u652f\u6301\u5927\u591a\u6570 linux x86 \u53d1\u884c\u7248\u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.17+ \u7248\u672c jindosdk-x.y.z-linux-el7-aarch64.tar.gz \u652f\u6301\u5927\u591a\u6570 linux aarch64 \u53d1\u884c\u7248\u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.17+ \u7248\u672c jindosdk-x.y.z-linux-el6-x86_64.tar.gz \u652f\u6301 centos6 x86 \u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.12+ \u7248\u672c jindosdk-x.y.z-linux-ubuntu22-x86_64.tar.gz \u652f\u6301 ubuntu22 x86 \u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.34+ \u7248\u672c jindosdk-x.y.z-macos-11_0-x86_64.tar.gz \u652f\u6301 macos x86 \u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 macos 11.0 \u4ee5\u4e0a\u7248\u672c jindosdk-x.y.z-macos-11_0-aarch64.tar.gz \u652f\u6301 macos m1/m2 \u7684\u5b8c\u6574\u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 macos 11.0 \u4ee5\u4e0a\u7248\u672c jindo-sdk-x.y.z.jar \u652f\u6301\u6240\u6709\u5e73\u53f0\u7684 lite \u4ea7\u51fa\u7269\uff0c\u7eaf java \u5b9e\u73b0\uff0c\u4e0d\u5305\u542b native \u5b9e\u73b0 jindo-core-x.y.z.jar \u652f\u6301\u5927\u591a\u6570 linux x86 \u53d1\u884c\u7248\u7684 lite \u4ea7\u51fa\u7269\uff0c\u5305\u542b native \u5b9e\u73b0\uff0c\u517c\u5bb9 glibc 2.17+ \u7248\u672c jindo-core-linux-el7-aarch64-x.y.z.jar \u652f\u6301\u5927\u591a\u6570 linux aarch64 \u53d1\u884c\u7248\u7684 lite \u4ea7\u51fa\u7269\uff0c\u5305\u542b native \u5b9e\u73b0\uff0c\u517c\u5bb9 glibc 2.17+ \u7248\u672c jindo-core-linux-el6-x86_64-x.y.z.jar \u652f\u6301 centos6 x86 \u7684 lite \u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.12+ \u7248\u672c jindo-core-linux-ubuntu22-x86_64-x.y.z.jar \u652f\u6301 ubuntu22 x86 \u7684 lite \u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 glibc 2.34+ \u7248\u672c jindo-core-macos-11_0-x86_64-x.y.z.jar \u652f\u6301 macos x86 \u7684 lite \u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 macos 11.0 \u4ee5\u4e0a\u7248\u672c jindo-core-macos-11_0-aarch64-x.y.z.jar \u652f\u6301 macos m1/m2 \u7684 lite \u4ea7\u51fa\u7269\uff0c\u517c\u5bb9 macos 11.0 \u4ee5\u4e0a\u7248\u672c ps: 6.5.0 \u7248\u672c\u5f00\u59cb tar \u5305\u4e2d\u7684 lite jar \u9ed8\u8ba4\u4f7f\u7528 nextarch classifer\uff0c\u5373 jindo-core-x.y.z-nextarch.jar\u3001jindo-core-linux-el7-aarch64-x.y.z-nextarch.jar \u3002","title":"\u4ea7\u7269\u7269\u5e73\u53f0\u8bf4\u660e\u8868"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#jindosdk_2","text":"\u4ee5\u4e0b 6.7.3 \u7248\u672c\u4e3a\u4f8b\u3002","title":"JindoSDK \u90e8\u7f72\u793a\u4f8b"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#linux-x86","text":"\u5b8c\u6574\u5b89\u88c5\uff0c\u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u89e3\u538b\u5728 /usr/lib/jindosdk-6.7.3-linux \u76ee\u5f55\u4e3a\u4f8b\uff1a wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz tar zxvf jindosdk-6.7.3-linux.tar.gz -C /usr/lib/ export JINDOSDK_HOME=/usr/lib/jindosdk-6.7.3-linux export JINDOSDK_CONF_DIR=${JINDOSDK_HOME}/conf export PATH=${PATH}:${JINDOSDK_HOME}/bin export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${JINDOSDK_HOME}/lib/native export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:${JINDOSDK_HOME}/lib/* lite\u5b89\u88c5\uff0c\u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u4e0b\u8f7d\u5230 /share/hadoop/hdfs/lib/ \u76ee\u5f55\u4e3a\u4f8b\uff1a wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-sdk/6.7.3/jindo-sdk-6.7.3.jar wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-core/6.7.3/jindo-core-6.7.3.jar cp jindo-core-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"Linux x86 \u73af\u5883\u793a\u4f8b"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#linux-aarch64","text":"\u5b8c\u6574\u5b89\u88c5\uff0c\u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u89e3\u538b\u5728 /usr/lib/jindosdk-6.7.3-linux-el7-aarch64 \u76ee\u5f55\u4e3a\u4f8b\uff1a wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/x.y.z/jindosdk-6.7.3-linux-el7-aarch64.tar.gz tar zxvf jindosdk-6.7.3-linux-el7-aarch64.tar.gz -C /usr/lib/ export JINDOSDK_HOME=/usr/lib/jindosdk-6.7.3-linux-el7-aarch64 export JINDOSDK_CONF_DIR=${JINDOSDK_HOME}/conf export PATH=${PATH}:${JINDOSDK_HOME}/bin export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${JINDOSDK_HOME}/lib/native export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:${JINDOSDK_HOME}/lib/* lite\u5b89\u88c5\uff0c\u4ee5\u5b89\u88c5\u5305\u5185\u5bb9\u4e0b\u8f7d\u5230 /share/hadoop/hdfs/lib/ \u76ee\u5f55\u4e3a\u4f8b\uff1a wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-sdk/6.7.3/jindo-sdk-6.7.3.jar wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/com/aliyun/jindodata/jindo-core-linux-el7-aarch64/6.7.3/jindo-core-linux-el7-aarch64-6.7.3.jar cp jindo-core-linux-el7-aarch64-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindo-core-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"Linux aarch64 \u73af\u5883\u793a\u4f8b"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#_4","text":"","title":"\u9644\u5f55"},{"location":"jindosdk/jindosdk_deployment_multi_platform/#_5","text":"\u4ee5\u4e0b x.y.z \u8868\u793a\u7248\u672c\u53f7\u3002 . \u251c\u2500\u2500 bin \u2502 \u251c\u2500\u2500 fusermount3 \u2502 \u251c\u2500\u2500 jindo \u2502 \u251c\u2500\u2500 jindo-fuse \u2502 \u251c\u2500\u2500 jindobench \u2502 \u251c\u2500\u2500 jindodiag \u2502 \u251c\u2500\u2500 jindosync \u2502 \u2514\u2500\u2500 jindotable \u251c\u2500\u2500 conf \u2502 \u251c\u2500\u2500 core-site.xml.template \u2502 \u251c\u2500\u2500 jindosdk.cfg.template \u2502 \u2514\u2500\u2500 log4j.properties.template \u251c\u2500\u2500 include \u2502 \u251c\u2500\u2500 jdo_api.h \u2502 \u251c\u2500\u2500 jdo_common.h \u2502 \u251c\u2500\u2500 jdo_content_summary.h \u2502 \u251c\u2500\u2500 jdo_data_types.h \u2502 \u251c\u2500\u2500 jdo_defines.h \u2502 \u251c\u2500\u2500 jdo_error.h \u2502 \u251c\u2500\u2500 jdo_file_status.h \u2502 \u251c\u2500\u2500 jdo_list_dir_result.h \u2502 \u251c\u2500\u2500 jdo_option_keys.h \u2502 \u2514\u2500\u2500 jdo_options.h \u251c\u2500\u2500 lib \u2502 \u251c\u2500\u2500 jindo-core-x.y.z-nextarch.jar \u2502 \u251c\u2500\u2500 jindo-core-linux-el7-aarch64-x.y.z-nextarch.jar \u2502 \u251c\u2500\u2500 jindo-sdk-x.y.z-nextarch.jar \u2502 \u251c\u2500\u2500 native \u2502 \u2502 \u251c\u2500\u2500 libfuse3.so.3 \u2502 \u2502 \u251c\u2500\u2500 libjemalloc.so \u2502 \u2502 \u251c\u2500\u2500 libjindo-csdk.so \u2502 \u2502 \u251c\u2500\u2500 libjindo-tensorflow1.15.so \u2502 \u2502 \u251c\u2500\u2500 libjindo-tensorflow2.8.so \u2502 \u2502 \u251c\u2500\u2500 libjindosdk.so \u2502 \u2502 \u251c\u2500\u2500 libjindosdk_c.so -> libjindosdk_c.so.x \u2502 \u2502 \u251c\u2500\u2500 libjindosdk_c.so.x -> libjindosdk_c.so.x.y.z \u2502 \u2502 \u2514\u2500\u2500 libjindosdk_c.so.x.y.z \u2502 \u2514\u2500\u2500 site-packages \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp310-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp311-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp312-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp36-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp37-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp38-abi3-linux_x86_64.whl \u2502 \u2514\u2500\u2500 pyjindo-x.y.z-cp39-abi3-linux_x86_64.whl \u251c\u2500\u2500 plugins \u2502 \u2514\u2500\u2500 libjindo-auth-client-plugin-nextarch.so \u2502 \u2514\u2500\u2500 flink \u2502 \u251c\u2500\u2500 jindo-flink-x.y.z-full.jar \u2502 \u2514\u2500\u2500 jindo-flink-x.y.z.jar \u2502 \u251c\u2500\u2500 spark2 \u2502 \u2502 \u2514\u2500\u2500 jindo-spark2-x.y.z.jar \u2502 \u2514\u2500\u2500 spark3 \u2502 \u2514\u2500\u2500 jindo-spark3-x.y.z.jar \u251c\u2500\u2500 tools \u2502 \u251c\u2500\u2500 jindo-dependence-shaded-x.y.z.jar \u2502 \u251c\u2500\u2500 jindo-distcp-tool-x.y.z.jar \u2502 \u251c\u2500\u2500 jindo-distjob-tool-x.y.z.jar \u2502 \u251c\u2500\u2500 jindotable-hive-tool-x.y.z.jar \u2502 \u2514\u2500\u2500 jindotable-shell-x.y.z.jar \u2514\u2500\u2500 versions \u251c\u2500\u2500 JINDOSDK_VERSION \u251c\u2500\u2500 JINDOTABLE_VERSION \u2514\u2500\u2500 JINDO_CONNECTORS_VERSION 9 directories, 29 files","title":"\u5b8c\u6574\u4ea7\u51fa\u7269\u793a\u4f8b"},{"location":"jindosdk/jindosdk_download/","text":"JindoData v6.7.3 \u4e0b\u8f7d \u00a7 \u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72\u8bf4\u660e\uff0c\u53c2\u89c1 \u6587\u6863\u94fe\u63a5 \u5386\u53f2\u7248\u672c\u95ee\u9898\u53ca\u53d1\u5e03\u8bf4\u660e\uff0c\u53c2\u89c1 \u6587\u6863\u94fe\u63a5 \u4e0b\u8f7d\u94fe\u63a5 \u00a7 \u5b8c\u6574\u4ea7\u51fa\u7269 \u00a7 jindosdk-6.7.3-linux.tar.gz jindosdk-6.7.3-linux-el7-aarch64.tar.gz jindosdk-6.7.3-linux-el6-x86_64.tar.gz jindosdk-6.7.3-linux-ubuntu22-x86_64.tar.gz jindosdk-6.7.3-macos-11_0-x86_64.tar.gz jindosdk-6.7.3-macos-11_0-aarch64.tar.gz lite \u4ea7\u51fa\u7269 \u00a7 jindo-core-6.7.3.jar jindo-sdk-6.7.3.jar jindo-core-linux-el7-aarch64-6.7.3.jar jindo-core-linux-el6-x86_64-6.7.3.jar jindo-core-linux-ubuntu22-x86_64-6.7.3.jar jindo-core-macos-11_0-x86_64-6.7.3.jar jindo-core-macos-11_0-aarch64-6.7.3.jar lite \u4ea7\u51fa\u7269 (nextarch classifier) \u00a7 jindo-core-6.7.3-nextarch.jar jindo-sdk-6.7.3-nextarch.jar jindo-core-linux-el7-aarch64-6.7.3-nextarch.jar jindo-core-linux-el6-x86_64-6.7.3-nextarch.jar jindo-core-linux-ubuntu22-x86_64-6.7.3-nextarch.jar jindo-core-macos-11_0-x86_64-6.7.3-nextarch.jar jindo-core-macos-11_0-aarch64-6.7.3-nextarch.jar","title":"JindoSDK \u4e0b\u8f7d"},{"location":"jindosdk/jindosdk_download/#jindodata-v673","text":"\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72\u8bf4\u660e\uff0c\u53c2\u89c1 \u6587\u6863\u94fe\u63a5 \u5386\u53f2\u7248\u672c\u95ee\u9898\u53ca\u53d1\u5e03\u8bf4\u660e\uff0c\u53c2\u89c1 \u6587\u6863\u94fe\u63a5","title":"JindoData v6.7.3 \u4e0b\u8f7d"},{"location":"jindosdk/jindosdk_download/#_1","text":"","title":"\u4e0b\u8f7d\u94fe\u63a5"},{"location":"jindosdk/jindosdk_download/#_2","text":"jindosdk-6.7.3-linux.tar.gz jindosdk-6.7.3-linux-el7-aarch64.tar.gz jindosdk-6.7.3-linux-el6-x86_64.tar.gz jindosdk-6.7.3-linux-ubuntu22-x86_64.tar.gz jindosdk-6.7.3-macos-11_0-x86_64.tar.gz jindosdk-6.7.3-macos-11_0-aarch64.tar.gz","title":"\u5b8c\u6574\u4ea7\u51fa\u7269"},{"location":"jindosdk/jindosdk_download/#lite","text":"jindo-core-6.7.3.jar jindo-sdk-6.7.3.jar jindo-core-linux-el7-aarch64-6.7.3.jar jindo-core-linux-el6-x86_64-6.7.3.jar jindo-core-linux-ubuntu22-x86_64-6.7.3.jar jindo-core-macos-11_0-x86_64-6.7.3.jar jindo-core-macos-11_0-aarch64-6.7.3.jar","title":"lite \u4ea7\u51fa\u7269"},{"location":"jindosdk/jindosdk_download/#lite-nextarch-classifier","text":"jindo-core-6.7.3-nextarch.jar jindo-sdk-6.7.3-nextarch.jar jindo-core-linux-el7-aarch64-6.7.3-nextarch.jar jindo-core-linux-el6-x86_64-6.7.3-nextarch.jar jindo-core-linux-ubuntu22-x86_64-6.7.3-nextarch.jar jindo-core-macos-11_0-x86_64-6.7.3-nextarch.jar jindo-core-macos-11_0-aarch64-6.7.3-nextarch.jar","title":"lite \u4ea7\u51fa\u7269 (nextarch classifier)"},{"location":"jindosdk/jindosdk_endpoint_configuration/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Endpoint \u914d\u7f6e \u00a7 \u6709\u4e09\u79cd\u65b9\u5f0f\u53ef\u4ee5\u6307\u5b9a Endpoint\uff0cJindoSDK \u4f1a\u4ee5\u4e0b\u4f18\u5148\u7ea7\u67e5\u627e Endpoint\uff1a \u65b9\u5f0f\u4e00\uff1a \u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u6307\u5b9a Endpoint\u3002\uff08\u63a8\u8350\uff09 \u00a7 \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u4f7f\u7528\u8fd9\u79cd\u683c\u5f0f\u7684\u8bbf\u95ee\u8def\u5f84 oss://<Bucket>.<Endpoint>/<Object> , \u5982: oss://example-oss-bucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u63a5\u53e3\u3002 \u65b9\u5f0f\u4e8c\uff1a \u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint\u3002 \u00a7 \u5982\u679c\u4f7f\u7528 oss://<Bucket>/<Object> \u8fd9\u79cd\u8bbf\u95ee\u8def\u5f84\uff0c\u5373\u8bbf\u95ee\u8def\u5f84\u4e2d\u6ca1\u6709 Endpoint, JindoSDK \u5219\u4f1a\u5728\u914d\u7f6e\u4e2d\u67e5\u627e Bucket \u7ea7\u522b\u7684 Endpoint\u3002 \u53ef\u5728 Hadoop \u7684 core-site.xml \u4e2d\u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint \u6307\u5411 OSS-HDFS \u670d\u52a1\u7684 Endpoint\u3002 <configuration> <property> <name>fs.oss.bucket.XXX.endpoint</name> <value>cn-xxx.oss-dls.aliyuncs.com</value> </property> </configuration> \u8bf4\u660e: XXX \u4e3a OSS-HDFS \u670d\u52a1 Bucket \u540d\u79f0\u3002 \u65b9\u5f0f\u4e09\uff1a\u914d\u7f6e\u5168\u5c40\u9ed8\u8ba4 Endpoint\u3002 \u00a7 \u5982\u679c\u4f7f\u7528 oss://<Bucket>/<Object> \u8fd9\u79cd\u8bbf\u95ee\u8def\u5f84\u800c\u4e14\u6ca1\u6709\u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint\uff0c\u5219\u4f1a\u7528\u5168\u5c40 Endpoint \u8bbf\u95ee\uff0c\u5982\u679c\u8981\u9ed8\u8ba4\u4f7f\u7528\u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff0c\u5219\u53ef\u7528\u8be5\u65b9\u6cd5\u3002 \u53ef\u5728 Hadoop \u7684 core-site.xml \u4e2d\u914d\u7f6e\u5168\u5c40\u9ed8\u8ba4 Endpoint\u3002 <configuration> <property> <name>fs.oss.endpoint</name> <value>cn-xxx.oss-dls.aliyuncs.com</value> </property> </configuration>","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Endpoint \u914d\u7f6e"},{"location":"jindosdk/jindosdk_endpoint_configuration/#oss-hdfs-jindofs-endpoint","text":"\u6709\u4e09\u79cd\u65b9\u5f0f\u53ef\u4ee5\u6307\u5b9a Endpoint\uff0cJindoSDK \u4f1a\u4ee5\u4e0b\u4f18\u5148\u7ea7\u67e5\u627e Endpoint\uff1a","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09Endpoint \u914d\u7f6e"},{"location":"jindosdk/jindosdk_endpoint_configuration/#endpoint","text":"\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u65f6\uff0c\u63a8\u8350\u4f7f\u7528\u8fd9\u79cd\u683c\u5f0f\u7684\u8bbf\u95ee\u8def\u5f84 oss://<Bucket>.<Endpoint>/<Object> , \u5982: oss://example-oss-bucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u63a5\u53e3\u3002","title":"\u65b9\u5f0f\u4e00\uff1a \u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u6307\u5b9a Endpoint\u3002\uff08\u63a8\u8350\uff09"},{"location":"jindosdk/jindosdk_endpoint_configuration/#bucket-endpoint","text":"\u5982\u679c\u4f7f\u7528 oss://<Bucket>/<Object> \u8fd9\u79cd\u8bbf\u95ee\u8def\u5f84\uff0c\u5373\u8bbf\u95ee\u8def\u5f84\u4e2d\u6ca1\u6709 Endpoint, JindoSDK \u5219\u4f1a\u5728\u914d\u7f6e\u4e2d\u67e5\u627e Bucket \u7ea7\u522b\u7684 Endpoint\u3002 \u53ef\u5728 Hadoop \u7684 core-site.xml \u4e2d\u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint \u6307\u5411 OSS-HDFS \u670d\u52a1\u7684 Endpoint\u3002 <configuration> <property> <name>fs.oss.bucket.XXX.endpoint</name> <value>cn-xxx.oss-dls.aliyuncs.com</value> </property> </configuration> \u8bf4\u660e: XXX \u4e3a OSS-HDFS \u670d\u52a1 Bucket \u540d\u79f0\u3002","title":"\u65b9\u5f0f\u4e8c\uff1a \u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint\u3002"},{"location":"jindosdk/jindosdk_endpoint_configuration/#endpoint_1","text":"\u5982\u679c\u4f7f\u7528 oss://<Bucket>/<Object> \u8fd9\u79cd\u8bbf\u95ee\u8def\u5f84\u800c\u4e14\u6ca1\u6709\u914d\u7f6e Bucket \u7ea7\u522b\u7684 Endpoint\uff0c\u5219\u4f1a\u7528\u5168\u5c40 Endpoint \u8bbf\u95ee\uff0c\u5982\u679c\u8981\u9ed8\u8ba4\u4f7f\u7528\u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff0c\u5219\u53ef\u7528\u8be5\u65b9\u6cd5\u3002 \u53ef\u5728 Hadoop \u7684 core-site.xml \u4e2d\u914d\u7f6e\u5168\u5c40\u9ed8\u8ba4 Endpoint\u3002 <configuration> <property> <name>fs.oss.endpoint</name> <value>cn-xxx.oss-dls.aliyuncs.com</value> </property> </configuration>","title":"\u65b9\u5f0f\u4e09\uff1a\u914d\u7f6e\u5168\u5c40\u9ed8\u8ba4 Endpoint\u3002"},{"location":"jindosdk/jindosdk_memory_best_practice/","text":"JindoSDk \u901a\u8fc7 jemalloc \u63a7\u5236\u5185\u5b58\u4e0a\u6da8\u6700\u4f73\u5b9e\u8df5 \u00a7 \u80cc\u666f \u00a7 jindosdk hadoop sdk \u4f7f\u7528 jni \u8c03\u7528 native \u5b9e\u73b0\uff0c\u56e0\u6b64 jvm \u76f8\u5173 gc \u53c2\u6570\u65e0\u6cd5\u6709\u6548\u63a7\u5236 native \u5185\u5b58\u4e0a\u6da8\uff0c\u53ef\u4ee5\u901a\u8fc7 jemalloc \u914d\u7f6e\u6765\u8fdb\u884c\u63a7\u5236\u3002 \u90e8\u5c5e\u65b9\u6cd5 \u00a7 jindosdk 4.6.8 \u53ca\u4ee5\u4e0a\u7248\u672c\u5b8c\u6574\u4ea7\u51fa\u7269\u5728 tar \u5305\u4e2d\u9ed8\u8ba4\u643a\u5e26\u4e86 jemalloc\uff1a . \u251c\u2500\u2500 bin \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 conf \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 lib \u2502 \u251c\u2500\u2500 jindo-core-6.7.3.jar \u2502 \u251c\u2500\u2500 jindo-sdk-6.7.3.jar \u2502 \u2514\u2500\u2500 native \u2502 \u251c\u2500\u2500 libjemalloc.so \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 plugins \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 tools \u2502 \u251c\u2500\u2500 ... \u2514\u2500\u2500 versions \u2514\u2500\u2500 JINDOSDK_VERSION \u5047\u8bbe\u5b89\u88c5\u76ee\u5f55\u4e3a/opt/apps/JINDOSDK/jindosdk-current/\uff0c\u8981\u4fdd\u8bc1\u6240\u6709\u8c03\u7528jindosdk\u7684\u73af\u5883\uff0c\u90fd\u914d\u7f6e\u4ee5\u4e0b\u7684\u73af\u5883\u53d8\u91cf\uff1a LD_PRELOAD=/opt/apps/JINDOSDK/jindosdk-current/lib/native/libjemalloc.so MALLOC_CONF=narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0,background_thread:true,abort_conf:true \u6548\u679c \u00a7 wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/memcheck/jindo-sdk-memleak-check-1.0.jar \u6267\u884c\u4ee5\u4e0bmemleak-check\u6d4b\u8bd5 HADOOP_HEAPSIZE=2560 HADOOP_ROOT_LOGGER=ERROR,console hadoop jar jindo-sdk-memleak-check-1.0.jar MemLeakTestProgram -baseDir oss:/<your-bucket>/ -duration 8899 \u4f7f\u7528\u9ed8\u8ba4glibc\uff0c\u6700\u7ec8\u5360\u75283627M \u4f7f\u7528jemalloc\u4e0d\u8bbe\u7f6eMALLOC_CONF\uff0c\u6700\u7ec8\u5360\u75282929M \u4f7f\u7528jemalloc\u5e76\u8bbe\u7f6eMALLOC_CONF\uff0c\u6700\u7ec8\u5360\u75282289M \u5176\u4ed6 \u00a7 \u4e0a\u9762\u7684\u914d\u7f6e\u9002\u5408\u5185\u5b58\u7d27\u5f20\u7684\u573a\u666f\uff0c\u53ef\u80fd\u5bf9\u6027\u80fd\u6709\u4e00\u5b9a\u5f71\u54cd\u3002\u5f53\u9700\u8981\u7efc\u5408\u8003\u8651\u5185\u5b58\u5360\u7528\u4e0e\u6027\u80fd\u65f6\uff0c\u53ef\u4ee5\u5c1d\u8bd5\u4ee5\u4e0b\u914d\u7f6e\uff1a LD_PRELOAD=/opt/apps/JINDOSDK/jindosdk-current/lib/native/libjemalloc.so MALLOC_CONF=percpu_arena:percpu,lg_tcache_max:12,dirty_decay_ms:5000,muzzy_decay_ms:5000","title":"JindoSDk \u901a\u8fc7 jemalloc \u63a7\u5236\u5185\u5b58\u4e0a\u6da8\u6700\u4f73\u5b9e\u8df5"},{"location":"jindosdk/jindosdk_memory_best_practice/#jindosdk-jemalloc","text":"","title":"JindoSDk \u901a\u8fc7 jemalloc \u63a7\u5236\u5185\u5b58\u4e0a\u6da8\u6700\u4f73\u5b9e\u8df5"},{"location":"jindosdk/jindosdk_memory_best_practice/#_1","text":"jindosdk hadoop sdk \u4f7f\u7528 jni \u8c03\u7528 native \u5b9e\u73b0\uff0c\u56e0\u6b64 jvm \u76f8\u5173 gc \u53c2\u6570\u65e0\u6cd5\u6709\u6548\u63a7\u5236 native \u5185\u5b58\u4e0a\u6da8\uff0c\u53ef\u4ee5\u901a\u8fc7 jemalloc \u914d\u7f6e\u6765\u8fdb\u884c\u63a7\u5236\u3002","title":"\u80cc\u666f"},{"location":"jindosdk/jindosdk_memory_best_practice/#_2","text":"jindosdk 4.6.8 \u53ca\u4ee5\u4e0a\u7248\u672c\u5b8c\u6574\u4ea7\u51fa\u7269\u5728 tar \u5305\u4e2d\u9ed8\u8ba4\u643a\u5e26\u4e86 jemalloc\uff1a . \u251c\u2500\u2500 bin \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 conf \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 lib \u2502 \u251c\u2500\u2500 jindo-core-6.7.3.jar \u2502 \u251c\u2500\u2500 jindo-sdk-6.7.3.jar \u2502 \u2514\u2500\u2500 native \u2502 \u251c\u2500\u2500 libjemalloc.so \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 plugins \u2502 \u251c\u2500\u2500 ... \u251c\u2500\u2500 tools \u2502 \u251c\u2500\u2500 ... \u2514\u2500\u2500 versions \u2514\u2500\u2500 JINDOSDK_VERSION \u5047\u8bbe\u5b89\u88c5\u76ee\u5f55\u4e3a/opt/apps/JINDOSDK/jindosdk-current/\uff0c\u8981\u4fdd\u8bc1\u6240\u6709\u8c03\u7528jindosdk\u7684\u73af\u5883\uff0c\u90fd\u914d\u7f6e\u4ee5\u4e0b\u7684\u73af\u5883\u53d8\u91cf\uff1a LD_PRELOAD=/opt/apps/JINDOSDK/jindosdk-current/lib/native/libjemalloc.so MALLOC_CONF=narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0,background_thread:true,abort_conf:true","title":"\u90e8\u5c5e\u65b9\u6cd5"},{"location":"jindosdk/jindosdk_memory_best_practice/#_3","text":"wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/memcheck/jindo-sdk-memleak-check-1.0.jar \u6267\u884c\u4ee5\u4e0bmemleak-check\u6d4b\u8bd5 HADOOP_HEAPSIZE=2560 HADOOP_ROOT_LOGGER=ERROR,console hadoop jar jindo-sdk-memleak-check-1.0.jar MemLeakTestProgram -baseDir oss:/<your-bucket>/ -duration 8899 \u4f7f\u7528\u9ed8\u8ba4glibc\uff0c\u6700\u7ec8\u5360\u75283627M \u4f7f\u7528jemalloc\u4e0d\u8bbe\u7f6eMALLOC_CONF\uff0c\u6700\u7ec8\u5360\u75282929M \u4f7f\u7528jemalloc\u5e76\u8bbe\u7f6eMALLOC_CONF\uff0c\u6700\u7ec8\u5360\u75282289M","title":"\u6548\u679c"},{"location":"jindosdk/jindosdk_memory_best_practice/#_4","text":"\u4e0a\u9762\u7684\u914d\u7f6e\u9002\u5408\u5185\u5b58\u7d27\u5f20\u7684\u573a\u666f\uff0c\u53ef\u80fd\u5bf9\u6027\u80fd\u6709\u4e00\u5b9a\u5f71\u54cd\u3002\u5f53\u9700\u8981\u7efc\u5408\u8003\u8651\u5185\u5b58\u5360\u7528\u4e0e\u6027\u80fd\u65f6\uff0c\u53ef\u4ee5\u5c1d\u8bd5\u4ee5\u4e0b\u914d\u7f6e\uff1a LD_PRELOAD=/opt/apps/JINDOSDK/jindosdk-current/lib/native/libjemalloc.so MALLOC_CONF=percpu_arena:percpu,lg_tcache_max:12,dirty_decay_ms:5000,muzzy_decay_ms:5000","title":"\u5176\u4ed6"},{"location":"jindosdk/jindosdk_quickstart/","text":"Hadoop \u73af\u5883\u4f7f\u7528 JindoSDK \u5feb\u901f\u5165\u95e8 \u00a7 JindoSDK \u5168\u9762\u517c\u5bb9 Hadoop FileSystem \u63a5\u53e3\uff0c\u63d0\u4f9b\u4e86\u66f4\u597d\u7684\u517c\u5bb9\u6027\u548c\u6613\u7528\u6027\uff0c\u53c8\u80fd\u517c\u5177OSS\u7684\u6027\u80fd\u548c\u6210\u672c\u4f18\u52bf\u3002 \u76ee\u524d\u652f\u6301\u5e02\u9762\u4e0a\u5927\u90e8\u5206 Hadoop \u7248\u672c\uff0c\u5728 Hadoop 2.3 \u53ca\u4ee5\u4e0a\u7684\u7248\u672c\u4e0a\u9a8c\u8bc1\u901a\u8fc7\uff082.3 \u4ee5\u524d\u7248\u672c\u6682\u672a\u6d4b\u8bd5\uff0c\u5982\u6709\u95ee\u9898\u8bf7 \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988\uff09\u3002 \u672c\u6587\u6863\u5047\u8bbe\u60a8\u5728 Linux x86 \u73af\u5883\u4e2d\uff0c\u5df2\u7ecf\u5b89\u88c5\u90e8\u7f72\u597d\u4e86 Hadoop \u73af\u5883\uff0c\u5982\u9700\u5728\u5176\u4ed6\u5e73\u53f0\u4f7f\u7528\uff0c\u8bf7\u53c2\u8003 \u300a\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK\u300b \u3002 \u6b65\u9aa4 \u00a7 1. \u5b89\u88c5\u90e8\u7f72 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz\uff0c \u4e0b\u8f7d\u9875\u9762 \u3002 \u5b8c\u6574\u90e8\u7f72 JindoSDK\uff0c\u53c2\u89c1 \u6587\u6863\u94fe\u63a5 \u6700\u7b80\u90e8\u7f72 JindoSDK\uff0c\u53c2\u89c1 \u6587\u6863\u94fe\u63a5 2. \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u00a7 \u7528 Hadoop Shell \u8bbf\u95ee OSS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls oss://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<bucket>/<path> \u66f4\u591a Hadoop \u547d\u4ee4\u53c2\u89c1 \u300a\u901a\u8fc7 Hadoop Shell \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS\u300b 3. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"\u5feb\u901f\u5165\u95e8"},{"location":"jindosdk/jindosdk_quickstart/#hadoop-jindosdk","text":"JindoSDK \u5168\u9762\u517c\u5bb9 Hadoop FileSystem \u63a5\u53e3\uff0c\u63d0\u4f9b\u4e86\u66f4\u597d\u7684\u517c\u5bb9\u6027\u548c\u6613\u7528\u6027\uff0c\u53c8\u80fd\u517c\u5177OSS\u7684\u6027\u80fd\u548c\u6210\u672c\u4f18\u52bf\u3002 \u76ee\u524d\u652f\u6301\u5e02\u9762\u4e0a\u5927\u90e8\u5206 Hadoop \u7248\u672c\uff0c\u5728 Hadoop 2.3 \u53ca\u4ee5\u4e0a\u7684\u7248\u672c\u4e0a\u9a8c\u8bc1\u901a\u8fc7\uff082.3 \u4ee5\u524d\u7248\u672c\u6682\u672a\u6d4b\u8bd5\uff0c\u5982\u6709\u95ee\u9898\u8bf7 \u65b0\u5efa ISSUE \u5411\u6211\u4eec\u53cd\u9988\uff09\u3002 \u672c\u6587\u6863\u5047\u8bbe\u60a8\u5728 Linux x86 \u73af\u5883\u4e2d\uff0c\u5df2\u7ecf\u5b89\u88c5\u90e8\u7f72\u597d\u4e86 Hadoop \u73af\u5883\uff0c\u5982\u9700\u5728\u5176\u4ed6\u5e73\u53f0\u4f7f\u7528\uff0c\u8bf7\u53c2\u8003 \u300a\u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK\u300b \u3002","title":"Hadoop \u73af\u5883\u4f7f\u7528 JindoSDK \u5feb\u901f\u5165\u95e8"},{"location":"jindosdk/jindosdk_quickstart/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"jindosdk/jindosdk_quickstart/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz\uff0c \u4e0b\u8f7d\u9875\u9762 \u3002 \u5b8c\u6574\u90e8\u7f72 JindoSDK\uff0c\u53c2\u89c1 \u6587\u6863\u94fe\u63a5 \u6700\u7b80\u90e8\u7f72 JindoSDK\uff0c\u53c2\u89c1 \u6587\u6863\u94fe\u63a5","title":"1. \u5b89\u88c5\u90e8\u7f72 JindoSDK"},{"location":"jindosdk/jindosdk_quickstart/#2-jindosdk-oss","text":"\u7528 Hadoop Shell \u8bbf\u95ee OSS\uff0c\u4e0b\u9762\u5217\u4e3e\u4e86\u51e0\u4e2a\u5e38\u7528\u7684\u547d\u4ee4\u3002 put \u64cd\u4f5c hadoop fs -put <path> oss://<bucket>/ ls \u64cd\u4f5c hadoop fs -ls oss://<bucket>/ mkdir \u64cd\u4f5c hadoop fs -mkdir oss://<bucket>/<path> rm \u64cd\u4f5c hadoop fs -rm oss://<bucket>/<path> \u66f4\u591a Hadoop \u547d\u4ee4\u53c2\u89c1 \u300a\u901a\u8fc7 Hadoop Shell \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS\u300b","title":"2. \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS"},{"location":"jindosdk/jindosdk_quickstart/#3","text":"JindoSDK \u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868","title":"3. \u53c2\u6570\u8c03\u4f18"},{"location":"jindosdk/oss-maven/","text":"JindoData 6.7.3 Maven \u76f8\u5173\u4f9d\u8d56 \u00a7 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 Jindo Flink \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo flink dependency --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-flink</artifactId> <version>${jindodata.version}</version> </dependency> </dependencies> </project> \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoDistCP \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo distcp dependency --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <!-- ... --> <dependencies> <!-- add jindo-core-extended-jar for macos --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian/arm --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> </dependencies> </project>","title":"JindoData 6.7.3 Maven \u76f8\u5173\u4f9d\u8d56"},{"location":"jindosdk/oss-maven/#jindodata-673-maven","text":"\u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 Jindo Flink \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo flink dependency --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-flink</artifactId> <version>${jindodata.version}</version> </dependency> </dependencies> </project> \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoDistCP \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo distcp dependency --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <!-- ... --> <dependencies> <!-- add jindo-core-extended-jar for macos --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian/arm --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> </dependencies> </project>","title":"JindoData 6.7.3 Maven \u76f8\u5173\u4f9d\u8d56"},{"location":"jindosdk/flink/jindosdk_on_flink/","text":"Flink \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 \u5728\u96c6\u7fa4\u4e0a\u6709\u5f00\u6e90\u7248\u672c Flink \u8f6f\u4ef6\uff0c\u7248\u672c\u4e0d\u4f4e\u4e8e 1.10.1\u3002 \u6ce8\uff1aFlink 1.16.0 \u53ca\u66f4\u9ad8\u7248\u672c\u7684\u53ef\u7528\u6027\u5c1a\u672a\u5f97\u5230\u9a8c\u8bc1\u3002 \u4e3a\u4ec0\u4e48 Flink \u9700\u8981\u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u00a7 \u82e5\u9700\u8981 Flink \u4fdd\u6301 EXACTLY_ONCE \u8bed\u4e49\u6d41\u5f0f\u5199\u5165 Aliyun OSS\uff0c\u5219\u9700\u8981\u4f7f\u7528 JindoSDK\u3002 Apache Flink \u662f\u4e00\u79cd\u5f53\u524d\u4e1a\u754c\u6d41\u884c\u7684\u5f00\u6e90\u5927\u6570\u636e\u6d41\u5f0f\u8ba1\u7b97\u5f15\u64ce\uff0c\u652f\u6301 \u201c\u4e25\u683c\u4e00\u6b21\u201d\uff08EXACTLY_ONCE\uff09\u8bed\u4e49\u5199\u5165\u90e8\u5206\u5b58\u50a8\u4ecb\u8d28\u3002Aliyun Object Storage Service (Aliyun OSS) \u5219\u662f\u963f\u91cc\u4e91\u65d7\u4e0b\u7684\u6838\u5fc3\u4ea7\u54c1\u4e4b\u4e00\uff0c\u63d0\u4f9b\u4e91\u539f\u751f\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1\uff0c\u51ed\u501f\u4f18\u5f02\u7684\u6027\u80fd\u548c\u5b89\u5168\u6027\u5df2\u670d\u52a1\u4e8e\u6d77\u91cf\u7528\u6237\u3002 \u76ee\u524d\uff0c\u5f00\u6e90\u7248\u672c Flink \u5bf9\u6d41\u5f0f\u5199\u5165 Aliyun OSS \u5c1a\u4e0d\u80fd\u652f\u6301 EXACTLY_ONCE \u8bed\u4e49\uff0c\u5982\u6709\u8be5\u9700\u6c42\u5219\u9700\u8981\u4f7f\u7528 JindoSDK\u3002 \u6ce8\uff1aApache Flink \u81ea 1.16 \u7248\u672c\u8d77\u539f\u751f\u63d0\u4f9b\u4e86\u6d41\u5f0f\u5199\u5165 Aliyun OSS \u5bf9\u8c61\u5b58\u50a8\u7684\u652f\u6301\u3002JindoSDK \u63d0\u4f9b\u7684\u5b9e\u73b0\u4e0e Apache Flink \u7684\u81ea\u6709\u5b9e\u73b0\u6709\u4e24\u4e2a\u4e3b\u8981\u533a\u522b\uff1a - JindoSDK \u7684\u5b9e\u73b0\u80fd\u591f\u540c\u65f6\u652f\u6301 Aliyun OSS \u5bf9\u8c61\u5b58\u50a8\u548c Aliyun OSS-HDFS \u6587\u4ef6\u5b58\u50a8\u3002\u5bf9 OSS-HDFS \u7684\u4ecb\u7ecd\u53c2\u8003 OSS-HDFS\u670d\u52a1\u6982\u8ff0 - JindoSDK \u7684\u5b9e\u73b0\u5e95\u5c42\u91c7\u7528\u4e86 Native \u65b9\u6848\u3002 JindoSDK \u914d\u7f6e \u00a7 \u9700\u8981\u5728\u6240\u6709 Flink \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flink \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e .jar \u6587\u4ef6\uff1a * jindo-flink-${version}-full.jar \u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 plugins/flink/ \u76ee\u5f55\u4e0b\u627e\u5230\u3002 \u53e6\u5916\uff0c\u5982\u679c\u96c6\u7fa4\u5185\u5b58\u5728 Apache Flink \u63d0\u4f9b\u7684\u81ea\u5e26 Flink OSS Connector\uff0c\u9700\u79fb\u9664\u3002\u79fb\u9664\u65b9\u6cd5\u4e3a\uff0c\u4ece Flink \u96c6\u7fa4\u7684 lib \u76ee\u5f55\u6216 plugins/oss-fs-hadoop \u79fb\u8d70\u5982\u4e0b jar \u5305\uff1a * flink-oss-fs-hadoop-${flink-version}.jar \u6ce8\uff1aJindoSDK \u76ee\u524d\u63d0\u4f9b\u591a\u5e73\u53f0\u652f\u6301\uff0c\u5f53\u524d\u652f\u6301\u7684\u5e73\u53f0\u4ee5\u53ca\u9700\u8981\u989d\u5916\u653e\u7f6e\u7684\u8d44\u6e90\uff0c\u53c2\u8003 \u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK \u5982\u4f55\u4f7f\u7528 \u00a7 \u914d\u7f6e\u597d SDK \u540e\uff0c\u65e0\u9700\u989d\u5916\u914d\u7f6e\uff0c\u4ee5\u5e38\u89c4 Flink \u6d41\u5f0f\u4f5c\u4e1a\u65b9\u6cd5\u4f7f\u7528\u5373\u53ef\uff0c\u6ce8\u610f\u4f7f\u7528\u6b63\u786e\u7684\u8def\u5f84\u3002\u5199\u5165 OSS (Aliyun Object Storage Service) \u987b\u4ee5 oss:// \u4e3a\u524d\u7f00\u3002 \u793a\u4f8b \u00a7 \u4e0b\u9762\u662f\u4e00\u4e2a\u7b80\u5355\u7684\u793a\u4f8b\u7a0b\u5e8f\uff0c\u5982\u679c\u5df2\u7ecf\u6309\u7167\u4e0a\u4e00\u8282\u914d\u7f6e\u597d SDK\uff0c\u5219\u4e0b\u5217\u7a0b\u5e8f\u5e94\u80fd\u6b63\u786e\u8fd0\u884c\uff1a (1) \u901a\u7528\u914d\u7f6e \u6253\u5f00 Flink \u7684 \u201c\u68c0\u67e5\u70b9\u201d\uff08Checkpoint\uff09\u3002\u4f8b\u5982\uff0cenv \u662f\u4e00\u4e2a\u5df2\u7ecf\u5efa\u7acb\u7684 StreamExecutionEnvironment\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u65b9\u5f0f\u5efa\u7acb\uff1a StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); \u5219\u53ef\u4ee5\u8fd9\u6837\u542f\u7528 checkpoint\uff1a env.enableCheckpointing(<userDefinedCheckpointInterval>, CheckpointingMode.EXACTLY_ONCE); \u4f7f\u7528\u53ef\u91cd\u53d1\u7684\u6570\u636e\u6e90\uff0c\u4f8b\u5982 Kafka (2) \u793a\u4f8b\u7a0b\u5e8f \u4e0b\u6587\u4e2d\uff0coutputStream \u662f\u4e00\u4e2a\u9884\u5148\u5f62\u6210\u7684 DataStream<String> \u5bf9\u8c61\uff0c\u5e76\u671f\u671b\u5c06\u5176\u5199\u5165 OSS\uff0c\u90a3\u4e48\u53ef\u4ee5\u8fd9\u6837\u6dfb\u52a0 sink\uff1a String outputPath = \"oss://<user-defined-bucket>/<user-defined-dir>\" StreamingFileSink<String> sink = StreamingFileSink.forRowFormat( new Path(outputPath), new SimpleStringEncoder<String>(\"UTF-8\") ).build(); outputStream.addSink(sink); \u6700\u540e\u7528 env.execute() \u6267\u884c Flink \u4f5c\u4e1a\u5373\u53ef\u3002 \u9ad8\u7ea7\u4f7f\u7528\uff1a\u81ea\u5b9a\u4e49\u53c2\u6570 \u00a7 \u7528\u6237\u5728\u63d0\u4ea4 Flink \u4f5c\u4e1a\u65f6\uff0c\u53ef\u4ee5\u81ea\u5b9a\u4e49\u914d\u7f6e\u4e00\u4e9b\u53c2\u6570\uff0c\u4ee5\u5f00\u542f\u6216\u63a7\u5236\u7279\u5b9a\u529f\u80fd\u3002\u4f8b\u5982\uff0c\u4ee5 yarn-cluster \u6a21\u5f0f\u63d0\u4ea4\u65f6\uff0c\u53ef\u4ee5\u901a\u8fc7 -yD \u8fdb\u884c\u914d\u7f6e\uff0c\u65b9\u5f0f\u7c7b\u4f3c\u4e8e\uff1a <flink_home>/bin/flink run -m yarn-cluster -yD key1=value1 -yD key2=value2 ... \u76ee\u524d\u652f\u6301\u901a\u8fc7\u914d\u7f6e\u5f00\u542f \u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\uff0c\u6216\u8005\u63a7\u5236 \u201c\u5206\u7247\u4e0a\u4f20\u201d \u7684\u5e76\u884c\u5ea6\u3002 \u71b5\u6ce8\u5165 \u00a7 \u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\u7684\u76ee\u7684\u662f\uff0c\u5c06\u5199\u5165\u8def\u5f84\u7684\u4e00\u6bb5\u7279\u5b9a\u5b57\u7b26\u4e32\u5339\u914d\u51fa\u6765\uff0c\u7528\u4e00\u6bb5\u968f\u673a\u7684\u5b57\u7b26\u4e32\u8fdb\u884c\u66ff\u6362\uff0c\u4ee5\u524a\u5f31\u6240\u8c13 \u201c\u7247\u533a\u201d (sharding) \u6548\u5e94\uff0c\u63d0\u9ad8\u5199\u5165\u6548\u7387\u3002\u914d\u7f6e\u53c2\u6570\uff1a oss.entropy.key=<user-defined-key> oss.entropy.length=<user-defined-length> \u5982\u6b64\uff0c\u6bcf\u6b21\u6253\u5f00\u5199\u4e00\u4e2a\u65b0\u6587\u4ef6\u65f6\uff0c\u8def\u5f84\u4e2d\u4e0e <user-defined-key> \u76f8\u540c\u7684\u5b57\u7b26\u4e32\u4f1a\u88ab\u66ff\u6362\u4e3a\u4e00\u4e2a\u968f\u673a\u5b57\u7b26\u4e32\uff0c\u968f\u673a\u4e32\u7684\u957f\u5ea6\u4e3a <user-defined-length> \u3002\u914d\u7f6e\u8be5\u53c2\u6570\u65f6\uff0c <user-defined-length> \u5fc5\u987b\u5927\u4e8e\u96f6\u3002 \u5206\u7247\u4e0a\u4f20\u5e76\u884c\u5ea6 \u00a7 \u5199\u5165 OSS \u4f1a\u81ea\u52a8\u8c03\u7528\u9ad8\u6548\u7684 \u201c\u5206\u7247\u4e0a\u4f20\u201d (Multipart Upload) \u673a\u5236\uff0c\u5c06\u5f85\u4e0a\u4f20\u7684\u6587\u4ef6\u5206\u4e3a\u591a\u4e2a\u6570\u636e\u5757 (part) \u5206\u522b\u4e0a\u4f20\uff0c\u6700\u540e\u7ec4\u5408\u3002\u76ee\u524d\u652f\u6301\u914d\u7f6e\u53c2\u6570 oss.upload.max.concurrent.uploads \uff0c\u7528\u6765\u63a7\u5236\u4e0a\u4f20\u6570\u636e\u5757 (part) \u7684\u5e76\u884c\u5ea6\uff0c\u5982\u679c\u8bbe\u7f6e\u8f83\u9ad8\u7684\u6570\u503c\u5219\u53ef\u80fd\u4f1a\u63d0\u9ad8\u5199\u5165\u6548\u7387\uff08\u4f46\u4e5f\u4f1a\u5360\u7528\u66f4\u591a\u8d44\u6e90\uff09\u3002\u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u8be5\u503c\u4e3a\u5f53\u524d\u53ef\u7528\u7684\u5904\u7406\u5668\u6570\u91cf\u3002","title":"Flink"},{"location":"jindosdk/flink/jindosdk_on_flink/#flink-jindosdk-oss","text":"","title":"Flink \u4f7f\u7528 JindoSDK \u5904\u7406 OSS \u4e0a\u7684\u6570\u636e"},{"location":"jindosdk/flink/jindosdk_on_flink/#_1","text":"\u5728\u96c6\u7fa4\u4e0a\u6709\u5f00\u6e90\u7248\u672c Flink \u8f6f\u4ef6\uff0c\u7248\u672c\u4e0d\u4f4e\u4e8e 1.10.1\u3002 \u6ce8\uff1aFlink 1.16.0 \u53ca\u66f4\u9ad8\u7248\u672c\u7684\u53ef\u7528\u6027\u5c1a\u672a\u5f97\u5230\u9a8c\u8bc1\u3002","title":"\u73af\u5883\u8981\u6c42"},{"location":"jindosdk/flink/jindosdk_on_flink/#flink-jindosdk-oss_1","text":"\u82e5\u9700\u8981 Flink \u4fdd\u6301 EXACTLY_ONCE \u8bed\u4e49\u6d41\u5f0f\u5199\u5165 Aliyun OSS\uff0c\u5219\u9700\u8981\u4f7f\u7528 JindoSDK\u3002 Apache Flink \u662f\u4e00\u79cd\u5f53\u524d\u4e1a\u754c\u6d41\u884c\u7684\u5f00\u6e90\u5927\u6570\u636e\u6d41\u5f0f\u8ba1\u7b97\u5f15\u64ce\uff0c\u652f\u6301 \u201c\u4e25\u683c\u4e00\u6b21\u201d\uff08EXACTLY_ONCE\uff09\u8bed\u4e49\u5199\u5165\u90e8\u5206\u5b58\u50a8\u4ecb\u8d28\u3002Aliyun Object Storage Service (Aliyun OSS) \u5219\u662f\u963f\u91cc\u4e91\u65d7\u4e0b\u7684\u6838\u5fc3\u4ea7\u54c1\u4e4b\u4e00\uff0c\u63d0\u4f9b\u4e91\u539f\u751f\u5bf9\u8c61\u5b58\u50a8\u670d\u52a1\uff0c\u51ed\u501f\u4f18\u5f02\u7684\u6027\u80fd\u548c\u5b89\u5168\u6027\u5df2\u670d\u52a1\u4e8e\u6d77\u91cf\u7528\u6237\u3002 \u76ee\u524d\uff0c\u5f00\u6e90\u7248\u672c Flink \u5bf9\u6d41\u5f0f\u5199\u5165 Aliyun OSS \u5c1a\u4e0d\u80fd\u652f\u6301 EXACTLY_ONCE \u8bed\u4e49\uff0c\u5982\u6709\u8be5\u9700\u6c42\u5219\u9700\u8981\u4f7f\u7528 JindoSDK\u3002 \u6ce8\uff1aApache Flink \u81ea 1.16 \u7248\u672c\u8d77\u539f\u751f\u63d0\u4f9b\u4e86\u6d41\u5f0f\u5199\u5165 Aliyun OSS \u5bf9\u8c61\u5b58\u50a8\u7684\u652f\u6301\u3002JindoSDK \u63d0\u4f9b\u7684\u5b9e\u73b0\u4e0e Apache Flink \u7684\u81ea\u6709\u5b9e\u73b0\u6709\u4e24\u4e2a\u4e3b\u8981\u533a\u522b\uff1a - JindoSDK \u7684\u5b9e\u73b0\u80fd\u591f\u540c\u65f6\u652f\u6301 Aliyun OSS \u5bf9\u8c61\u5b58\u50a8\u548c Aliyun OSS-HDFS \u6587\u4ef6\u5b58\u50a8\u3002\u5bf9 OSS-HDFS \u7684\u4ecb\u7ecd\u53c2\u8003 OSS-HDFS\u670d\u52a1\u6982\u8ff0 - JindoSDK \u7684\u5b9e\u73b0\u5e95\u5c42\u91c7\u7528\u4e86 Native \u65b9\u6848\u3002","title":"\u4e3a\u4ec0\u4e48 Flink \u9700\u8981\u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS"},{"location":"jindosdk/flink/jindosdk_on_flink/#jindosdk","text":"\u9700\u8981\u5728\u6240\u6709 Flink \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flink \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e .jar \u6587\u4ef6\uff1a * jindo-flink-${version}-full.jar \u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 plugins/flink/ \u76ee\u5f55\u4e0b\u627e\u5230\u3002 \u53e6\u5916\uff0c\u5982\u679c\u96c6\u7fa4\u5185\u5b58\u5728 Apache Flink \u63d0\u4f9b\u7684\u81ea\u5e26 Flink OSS Connector\uff0c\u9700\u79fb\u9664\u3002\u79fb\u9664\u65b9\u6cd5\u4e3a\uff0c\u4ece Flink \u96c6\u7fa4\u7684 lib \u76ee\u5f55\u6216 plugins/oss-fs-hadoop \u79fb\u8d70\u5982\u4e0b jar \u5305\uff1a * flink-oss-fs-hadoop-${flink-version}.jar \u6ce8\uff1aJindoSDK \u76ee\u524d\u63d0\u4f9b\u591a\u5e73\u53f0\u652f\u6301\uff0c\u5f53\u524d\u652f\u6301\u7684\u5e73\u53f0\u4ee5\u53ca\u9700\u8981\u989d\u5916\u653e\u7f6e\u7684\u8d44\u6e90\uff0c\u53c2\u8003 \u5728\u591a\u5e73\u53f0\u73af\u5883\u5b89\u88c5\u90e8\u7f72 JindoSDK","title":"JindoSDK \u914d\u7f6e"},{"location":"jindosdk/flink/jindosdk_on_flink/#_2","text":"\u914d\u7f6e\u597d SDK \u540e\uff0c\u65e0\u9700\u989d\u5916\u914d\u7f6e\uff0c\u4ee5\u5e38\u89c4 Flink \u6d41\u5f0f\u4f5c\u4e1a\u65b9\u6cd5\u4f7f\u7528\u5373\u53ef\uff0c\u6ce8\u610f\u4f7f\u7528\u6b63\u786e\u7684\u8def\u5f84\u3002\u5199\u5165 OSS (Aliyun Object Storage Service) \u987b\u4ee5 oss:// \u4e3a\u524d\u7f00\u3002","title":"\u5982\u4f55\u4f7f\u7528"},{"location":"jindosdk/flink/jindosdk_on_flink/#_3","text":"\u4e0b\u9762\u662f\u4e00\u4e2a\u7b80\u5355\u7684\u793a\u4f8b\u7a0b\u5e8f\uff0c\u5982\u679c\u5df2\u7ecf\u6309\u7167\u4e0a\u4e00\u8282\u914d\u7f6e\u597d SDK\uff0c\u5219\u4e0b\u5217\u7a0b\u5e8f\u5e94\u80fd\u6b63\u786e\u8fd0\u884c\uff1a (1) \u901a\u7528\u914d\u7f6e \u6253\u5f00 Flink \u7684 \u201c\u68c0\u67e5\u70b9\u201d\uff08Checkpoint\uff09\u3002\u4f8b\u5982\uff0cenv \u662f\u4e00\u4e2a\u5df2\u7ecf\u5efa\u7acb\u7684 StreamExecutionEnvironment\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u65b9\u5f0f\u5efa\u7acb\uff1a StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); \u5219\u53ef\u4ee5\u8fd9\u6837\u542f\u7528 checkpoint\uff1a env.enableCheckpointing(<userDefinedCheckpointInterval>, CheckpointingMode.EXACTLY_ONCE); \u4f7f\u7528\u53ef\u91cd\u53d1\u7684\u6570\u636e\u6e90\uff0c\u4f8b\u5982 Kafka (2) \u793a\u4f8b\u7a0b\u5e8f \u4e0b\u6587\u4e2d\uff0coutputStream \u662f\u4e00\u4e2a\u9884\u5148\u5f62\u6210\u7684 DataStream<String> \u5bf9\u8c61\uff0c\u5e76\u671f\u671b\u5c06\u5176\u5199\u5165 OSS\uff0c\u90a3\u4e48\u53ef\u4ee5\u8fd9\u6837\u6dfb\u52a0 sink\uff1a String outputPath = \"oss://<user-defined-bucket>/<user-defined-dir>\" StreamingFileSink<String> sink = StreamingFileSink.forRowFormat( new Path(outputPath), new SimpleStringEncoder<String>(\"UTF-8\") ).build(); outputStream.addSink(sink); \u6700\u540e\u7528 env.execute() \u6267\u884c Flink \u4f5c\u4e1a\u5373\u53ef\u3002","title":"\u793a\u4f8b"},{"location":"jindosdk/flink/jindosdk_on_flink/#_4","text":"\u7528\u6237\u5728\u63d0\u4ea4 Flink \u4f5c\u4e1a\u65f6\uff0c\u53ef\u4ee5\u81ea\u5b9a\u4e49\u914d\u7f6e\u4e00\u4e9b\u53c2\u6570\uff0c\u4ee5\u5f00\u542f\u6216\u63a7\u5236\u7279\u5b9a\u529f\u80fd\u3002\u4f8b\u5982\uff0c\u4ee5 yarn-cluster \u6a21\u5f0f\u63d0\u4ea4\u65f6\uff0c\u53ef\u4ee5\u901a\u8fc7 -yD \u8fdb\u884c\u914d\u7f6e\uff0c\u65b9\u5f0f\u7c7b\u4f3c\u4e8e\uff1a <flink_home>/bin/flink run -m yarn-cluster -yD key1=value1 -yD key2=value2 ... \u76ee\u524d\u652f\u6301\u901a\u8fc7\u914d\u7f6e\u5f00\u542f \u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\uff0c\u6216\u8005\u63a7\u5236 \u201c\u5206\u7247\u4e0a\u4f20\u201d \u7684\u5e76\u884c\u5ea6\u3002","title":"\u9ad8\u7ea7\u4f7f\u7528\uff1a\u81ea\u5b9a\u4e49\u53c2\u6570"},{"location":"jindosdk/flink/jindosdk_on_flink/#_5","text":"\u201c\u71b5\u6ce8\u5165\u201d (entropy injection) \u529f\u80fd\u7684\u76ee\u7684\u662f\uff0c\u5c06\u5199\u5165\u8def\u5f84\u7684\u4e00\u6bb5\u7279\u5b9a\u5b57\u7b26\u4e32\u5339\u914d\u51fa\u6765\uff0c\u7528\u4e00\u6bb5\u968f\u673a\u7684\u5b57\u7b26\u4e32\u8fdb\u884c\u66ff\u6362\uff0c\u4ee5\u524a\u5f31\u6240\u8c13 \u201c\u7247\u533a\u201d (sharding) \u6548\u5e94\uff0c\u63d0\u9ad8\u5199\u5165\u6548\u7387\u3002\u914d\u7f6e\u53c2\u6570\uff1a oss.entropy.key=<user-defined-key> oss.entropy.length=<user-defined-length> \u5982\u6b64\uff0c\u6bcf\u6b21\u6253\u5f00\u5199\u4e00\u4e2a\u65b0\u6587\u4ef6\u65f6\uff0c\u8def\u5f84\u4e2d\u4e0e <user-defined-key> \u76f8\u540c\u7684\u5b57\u7b26\u4e32\u4f1a\u88ab\u66ff\u6362\u4e3a\u4e00\u4e2a\u968f\u673a\u5b57\u7b26\u4e32\uff0c\u968f\u673a\u4e32\u7684\u957f\u5ea6\u4e3a <user-defined-length> \u3002\u914d\u7f6e\u8be5\u53c2\u6570\u65f6\uff0c <user-defined-length> \u5fc5\u987b\u5927\u4e8e\u96f6\u3002","title":"\u71b5\u6ce8\u5165"},{"location":"jindosdk/flink/jindosdk_on_flink/#_6","text":"\u5199\u5165 OSS \u4f1a\u81ea\u52a8\u8c03\u7528\u9ad8\u6548\u7684 \u201c\u5206\u7247\u4e0a\u4f20\u201d (Multipart Upload) \u673a\u5236\uff0c\u5c06\u5f85\u4e0a\u4f20\u7684\u6587\u4ef6\u5206\u4e3a\u591a\u4e2a\u6570\u636e\u5757 (part) \u5206\u522b\u4e0a\u4f20\uff0c\u6700\u540e\u7ec4\u5408\u3002\u76ee\u524d\u652f\u6301\u914d\u7f6e\u53c2\u6570 oss.upload.max.concurrent.uploads \uff0c\u7528\u6765\u63a7\u5236\u4e0a\u4f20\u6570\u636e\u5757 (part) \u7684\u5e76\u884c\u5ea6\uff0c\u5982\u679c\u8bbe\u7f6e\u8f83\u9ad8\u7684\u6570\u503c\u5219\u53ef\u80fd\u4f1a\u63d0\u9ad8\u5199\u5165\u6548\u7387\uff08\u4f46\u4e5f\u4f1a\u5360\u7528\u66f4\u591a\u8d44\u6e90\uff09\u3002\u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u8be5\u503c\u4e3a\u5f53\u524d\u53ef\u7528\u7684\u5904\u7406\u5668\u6570\u91cf\u3002","title":"\u5206\u7247\u4e0a\u4f20\u5e76\u884c\u5ea6"},{"location":"jindosdk/flume/jindosdk_on_flume/","text":"Flume \u4f7f\u7528 JindoSDK \u5199\u5165 OSS \u00a7 \u73af\u5883\u8981\u6c42 \u00a7 \u5728\u96c6\u7fa4\u4e0a\u5df2\u7ecf\u90e8\u7f72 Flume\uff0c\u5df2\u90e8\u7f72 JindoSDK 6.0 \u4ee5\u4e0a\u7248\u672c\u3002 JindoSDK \u5b89\u88c5\u90e8\u7f72 \u00a7 \u9700\u8981\u5728 Flume \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flume \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e JindoSDK\u3002 \u4e0b\u8f7d\u5e76\u89e3\u538b\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.y.z.tar.gz\uff08x.y.z\u8868\u7248\u672c\u53f7\uff09\uff0c ( \u4e0b\u8f7d\u9875\u9762 ) cp jindosdk-x.y.z/lib/*.jar $FLUME_HOME/jars/ \u914d\u7f6e \u00a7 Flume Sink \u914d\u7f6e \u00a7 # \u914d\u7f6eOSS Sink\u3002your_bucket\u586b\u5199\u4e3a\u5df2\u5f00\u542fOSS-HDFS\u670d\u52a1\u7684Bucket\u3002 xxx.sinks.oss_sink.hdfs.path = oss://${your_bucket}/flume_dir/%Y-%m-%d/%H # \u5199\u5165Flume\u4e8b\u52a1\u4e2d\u6700\u5927\u7684event\u6570\u91cf\u3002\u63a8\u8350\u6bcf\u6b21Flush\u7684\u91cf\u572832 MB\u4ee5\u4e0a\uff0c\u907f\u514dFlush\u8fc7\u5c0f\u5f71\u54cd\u6574\u4f53\u6027\u80fd\u4ee5\u53ca\u4ea7\u751f\u5927\u91cf\u7684staging\u6587\u4ef6\u3002 # batchSize\u5355\u4f4d\u4e3aevent\u6570\u91cf\uff08\u5373\u65e5\u5fd7\u884c\u6570\uff09\uff0c\u8bbe\u7f6e\u65f6\u9700\u8981\u5148\u8bc4\u4f30event\u5e73\u5747\u5927\u5c0f\uff08\u4f8b\u5982200 Byte\uff09\uff0c\u5047\u8bbe\u6bcf\u6b21Flush\u7684\u5927\u5c0f\u9884\u671f\u4e3a32 MB\uff0c\u5219batchSize\u7ea6\u4e3a160000\uff0832 MB / 200 Byte\uff09\u3002 xxx.sinks.oss_sink.hdfs.batchSize = 100000 ... xxx.sinks.oss_sink.hdfs.round = true xxx.sinks.oss_sink.hdfs.roundValue = 15 xxx.sinks.oss_sink.hdfs.Unit = minute xxx.sinks.oss_sink.hdfs.filePrefix = your_topic xxx.sinks.oss_sink.rollSize = 3600 xxx.sinks.oss_sink.threadsPoolSize = 30 ... OSS-HDFS \u573a\u666f core-sites.xml \u914d\u7f6e \u00a7 \u63a8\u8350\u6bcf\u6b21 Flush \u7684\u91cf\u5728 32 MB\u4ee5\u4e0a\uff0c\u907f\u514d Flush \u8fc7\u4e8e\u9891\u7e41\u5f71\u54cd\u6574\u4f53\u6027\u80fd\u4ee5\u53ca\u4ea7\u751f\u5c0f\u5757\u3002 <configuration> <property> <!-- false\u8868\u793a\u4e0d\u6267\u884chflush\uff0c\u4ec5\u4f9d\u9760close\u6765\u786e\u4fdd\u6267\u884c\u6587\u4ef6\u5199\u5165\uff0c\u4e0d\u63a8\u8350\u5173\u95ed --> <name>fs.oss.hflush.enable</name> <value>true</value> </property> <property> <!-- \u5f53\u524d\u6587\u4ef6\u6267\u884chflush\u7684\u6b21\u6570\u8fbe\u5230\u8be5\u6570\u91cf\u540e\uff0c\u624d\u89e6\u53d1\u4e00\u6b21\u771f\u6b63\u7684flush\uff080\u8868\u793a\u4e0d\u6839\u636e\u8be5\u6570\u91cf\u6765\u89e6\u53d1Flush\uff09 --> <name>fs.oss.hflush.interval.count</name> <value>0</value> </property> <property> <!-- \u5f53\u524d\u6587\u4ef6\u5199\u5165\u8fbe\u5230\u8be5\u5927\u5c0f\u540e\uff0c\u624d\u89e6\u53d1\u4e00\u6b21\u771f\u6b63\u7684flush\uff080\u8868\u793a\u4e0d\u6839\u636e\u5199\u5165\u5927\u5c0f\u6765\u89e6\u53d1Flush\uff09\uff0c\u5355\u4f4d\uff1a\u5b57\u8282--> <name>fs.oss.hflush.interval.size</name> <value>0</value> </property> </configuration> ps: \u5982 fs.oss.hflush.interval.count \u548c fs.oss.hflush.interval.size \u5747\u4e0d\u4e3a0\uff0c\u5219\u4e24\u4e2a\u6761\u4ef6\u5747\u6ee1\u8db3\u65f6\u624d\u6267\u884chflush\u3002","title":"Flume"},{"location":"jindosdk/flume/jindosdk_on_flume/#flume-jindosdk-oss","text":"","title":"Flume \u4f7f\u7528 JindoSDK \u5199\u5165 OSS"},{"location":"jindosdk/flume/jindosdk_on_flume/#_1","text":"\u5728\u96c6\u7fa4\u4e0a\u5df2\u7ecf\u90e8\u7f72 Flume\uff0c\u5df2\u90e8\u7f72 JindoSDK 6.0 \u4ee5\u4e0a\u7248\u672c\u3002","title":"\u73af\u5883\u8981\u6c42"},{"location":"jindosdk/flume/jindosdk_on_flume/#jindosdk","text":"\u9700\u8981\u5728 Flume \u8282\u70b9\u8fdb\u884c\u914d\u7f6e\u3002\u5728\u6bcf\u4e2a\u8282\u70b9 Flume \u6839\u76ee\u5f55\u4e0b\u7684 lib \u6587\u4ef6\u5939\uff0c\u653e\u7f6e JindoSDK\u3002 \u4e0b\u8f7d\u5e76\u89e3\u538b\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.y.z.tar.gz\uff08x.y.z\u8868\u7248\u672c\u53f7\uff09\uff0c ( \u4e0b\u8f7d\u9875\u9762 ) cp jindosdk-x.y.z/lib/*.jar $FLUME_HOME/jars/","title":"JindoSDK \u5b89\u88c5\u90e8\u7f72"},{"location":"jindosdk/flume/jindosdk_on_flume/#_2","text":"","title":"\u914d\u7f6e"},{"location":"jindosdk/flume/jindosdk_on_flume/#flume-sink","text":"# \u914d\u7f6eOSS Sink\u3002your_bucket\u586b\u5199\u4e3a\u5df2\u5f00\u542fOSS-HDFS\u670d\u52a1\u7684Bucket\u3002 xxx.sinks.oss_sink.hdfs.path = oss://${your_bucket}/flume_dir/%Y-%m-%d/%H # \u5199\u5165Flume\u4e8b\u52a1\u4e2d\u6700\u5927\u7684event\u6570\u91cf\u3002\u63a8\u8350\u6bcf\u6b21Flush\u7684\u91cf\u572832 MB\u4ee5\u4e0a\uff0c\u907f\u514dFlush\u8fc7\u5c0f\u5f71\u54cd\u6574\u4f53\u6027\u80fd\u4ee5\u53ca\u4ea7\u751f\u5927\u91cf\u7684staging\u6587\u4ef6\u3002 # batchSize\u5355\u4f4d\u4e3aevent\u6570\u91cf\uff08\u5373\u65e5\u5fd7\u884c\u6570\uff09\uff0c\u8bbe\u7f6e\u65f6\u9700\u8981\u5148\u8bc4\u4f30event\u5e73\u5747\u5927\u5c0f\uff08\u4f8b\u5982200 Byte\uff09\uff0c\u5047\u8bbe\u6bcf\u6b21Flush\u7684\u5927\u5c0f\u9884\u671f\u4e3a32 MB\uff0c\u5219batchSize\u7ea6\u4e3a160000\uff0832 MB / 200 Byte\uff09\u3002 xxx.sinks.oss_sink.hdfs.batchSize = 100000 ... xxx.sinks.oss_sink.hdfs.round = true xxx.sinks.oss_sink.hdfs.roundValue = 15 xxx.sinks.oss_sink.hdfs.Unit = minute xxx.sinks.oss_sink.hdfs.filePrefix = your_topic xxx.sinks.oss_sink.rollSize = 3600 xxx.sinks.oss_sink.threadsPoolSize = 30 ...","title":"Flume Sink \u914d\u7f6e"},{"location":"jindosdk/flume/jindosdk_on_flume/#oss-hdfs-core-sitesxml","text":"\u63a8\u8350\u6bcf\u6b21 Flush \u7684\u91cf\u5728 32 MB\u4ee5\u4e0a\uff0c\u907f\u514d Flush \u8fc7\u4e8e\u9891\u7e41\u5f71\u54cd\u6574\u4f53\u6027\u80fd\u4ee5\u53ca\u4ea7\u751f\u5c0f\u5757\u3002 <configuration> <property> <!-- false\u8868\u793a\u4e0d\u6267\u884chflush\uff0c\u4ec5\u4f9d\u9760close\u6765\u786e\u4fdd\u6267\u884c\u6587\u4ef6\u5199\u5165\uff0c\u4e0d\u63a8\u8350\u5173\u95ed --> <name>fs.oss.hflush.enable</name> <value>true</value> </property> <property> <!-- \u5f53\u524d\u6587\u4ef6\u6267\u884chflush\u7684\u6b21\u6570\u8fbe\u5230\u8be5\u6570\u91cf\u540e\uff0c\u624d\u89e6\u53d1\u4e00\u6b21\u771f\u6b63\u7684flush\uff080\u8868\u793a\u4e0d\u6839\u636e\u8be5\u6570\u91cf\u6765\u89e6\u53d1Flush\uff09 --> <name>fs.oss.hflush.interval.count</name> <value>0</value> </property> <property> <!-- \u5f53\u524d\u6587\u4ef6\u5199\u5165\u8fbe\u5230\u8be5\u5927\u5c0f\u540e\uff0c\u624d\u89e6\u53d1\u4e00\u6b21\u771f\u6b63\u7684flush\uff080\u8868\u793a\u4e0d\u6839\u636e\u5199\u5165\u5927\u5c0f\u6765\u89e6\u53d1Flush\uff09\uff0c\u5355\u4f4d\uff1a\u5b57\u8282--> <name>fs.oss.hflush.interval.size</name> <value>0</value> </property> </configuration> ps: \u5982 fs.oss.hflush.interval.count \u548c fs.oss.hflush.interval.size \u5747\u4e0d\u4e3a0\uff0c\u5219\u4e24\u4e2a\u6761\u4ef6\u5747\u6ee1\u8db3\u65f6\u624d\u6267\u884chflush\u3002","title":"OSS-HDFS \u573a\u666f core-sites.xml \u914d\u7f6e"},{"location":"jindosdk/hadoop/jindosdk_ide_hadoop/","text":"Hadoop \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 \u00a7 \u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.net.URI; public class TestJindoSDK { public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. FileSystem fs = FileSystem.get(URI.create(\"oss://<bucket>/\"), conf); FSDataInputStream in = fs.open(new Path(\"/uttest/file1\")); in.read(); in.close(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <!-- ... --> <dependencies> <!-- add jindo-core-extended-jar for macos --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> </dependencies> </project>","title":"\u5f00\u53d1\u548c\u4f7f\u7528"},{"location":"jindosdk/hadoop/jindosdk_ide_hadoop/#hadoop-jindosdk-ide","text":"\u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.net.URI; public class TestJindoSDK { public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. FileSystem fs = FileSystem.get(URI.create(\"oss://<bucket>/\"), conf); FSDataInputStream in = fs.open(new Path(\"/uttest/file1\")); in.read(); in.close(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <!-- ... --> <dependencies> <!-- add jindo-core-extended-jar for macos --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> </dependencies> </project>","title":"Hadoop \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5"},{"location":"jindosdk/hbase/jindosdk_on_hbase/","text":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8 \u00a7 HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002 OSS-HDFS \u670d\u52a1\u662f\u963f\u91cc\u4e91\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9 HDFS \u63a5\u53e3, JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\u53ca WAL \u6587\u4ef6\u7684\u5b58\u50a8\u5b9e\u73b0\u5b58\u7b97\u5206\u79bb\uff0c\u76f8\u5bf9\u4e8e\u672c\u5730HDFS\u5b58\u50a8\uff0c\u4f7f\u7528\u66f4\u52a0\u7075\u6d3b\uff0c\u51cf\u5c11\u8fd0\u7ef4\u6210\u672c\u3002 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 2. \u5b89\u88c5 jar \u5305 \u00a7 \u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-6.7.3 \u4e3a\u4f8b: cp jindosdk-6.7.3/lib/jindo-core-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-6.7.3/lib/jindo-sdk-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ 3. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06\u5df2\u5f00\u542f OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u00a7 \u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 5. \u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84 \u00a7 \u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u548c hbase.wal.dir \u7684\u503c\u4e3a OSS-HDFS \u670d\u52a1\u5730\u5740\uff0c\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84\u548c WAL \u6587\u4ef6\u7684\u5b58\u50a8\u8def\u5f84\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 OSS-HDFS \u670d\u52a1\u3002\u53c2\u6570\u503c\u4e3a oss://bucket.endpoint/hbase-root-dir hbase.wal.dir \u6307\u5b9a HBase \u7684 WAL \u5b58\u50a8\u76ee\u5f55\u5230 OSS-HDFS \u670d\u52a1\u3002 \u53c2\u6570\u503c\u4e3a oss://bucket.endpoint/hbase","title":"Hbase"},{"location":"jindosdk/hbase/jindosdk_on_hbase/#hbase-oss-hdfs-jindofs","text":"HBase \u662f Hadoop \u751f\u6001\u4e2d\u7684\u5b9e\u65f6\u6570\u636e\u5e93\uff0c\u6709\u5f88\u9ad8\u7684\u5199\u5165\u6027\u80fd\u3002 OSS-HDFS \u670d\u52a1\u662f\u963f\u91cc\u4e91\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9 HDFS \u63a5\u53e3, JindoSDK \u652f\u6301 HBase \u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\u53ca WAL \u6587\u4ef6\u7684\u5b58\u50a8\u5b9e\u73b0\u5b58\u7b97\u5206\u79bb\uff0c\u76f8\u5bf9\u4e8e\u672c\u5730HDFS\u5b58\u50a8\uff0c\u4f7f\u7528\u66f4\u52a0\u7075\u6d3b\uff0c\u51cf\u5c11\u8fd0\u7ef4\u6210\u672c\u3002","title":"HBase \u4f7f\u7528\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8"},{"location":"jindosdk/hbase/jindosdk_on_hbase/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"jindosdk/hbase/jindosdk_on_hbase/#2-jar","text":"\u89e3\u538b\u4e0b\u8f7d\u7684\u5b89\u88c5\u5305\uff0c\u5c06\u5b89\u88c5\u5305\u5185\u7684\u4ee5\u4e0b jar \u6587\u4ef6\u5b89\u88c5\u5230 hadoop \u7684 classpath \u4e0b\uff1a * jindo-core-x.x.x.jar * jindo-sdk-x.x.x.jar jindosdk-6.7.3 \u4e3a\u4f8b: cp jindosdk-6.7.3/lib/jindo-core-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/ cp jindosdk-6.7.3/lib/jindo-sdk-6.7.3.jar <HADOOP_HOME>/share/hadoop/hdfs/lib/","title":"2. \u5b89\u88c5 jar \u5305"},{"location":"jindosdk/hbase/jindosdk_on_hbase/#3-oss-hdfs-access-key","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u5c06\u5df2\u5f00\u542f OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002","title":"3. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"jindosdk/hbase/jindosdk_on_hbase/#4-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002","title":"4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint"},{"location":"jindosdk/hbase/jindosdk_on_hbase/#5-hbase","text":"\u9700\u8981\u4fee\u6539 hbase-site \u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u53c2\u6570 hbase.rootdir \u548c hbase.wal.dir \u7684\u503c\u4e3a OSS-HDFS \u670d\u52a1\u5730\u5740\uff0c\u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84\u548c WAL \u6587\u4ef6\u7684\u5b58\u50a8\u8def\u5f84\u3002\u5982\u679c\u8981\u91ca\u653e\u96c6\u7fa4\uff0c\u9700\u8981\u5148 Disable table\uff0c\u786e\u4fdd WAL \u6587\u4ef6\u5df2\u7ecf\u5b8c\u5168\u66f4\u65b0\u5230 HFile\u3002 \u53c2\u6570 \u63cf\u8ff0 hbase.rootdir \u6307\u5b9a HBase \u7684 ROOT \u5b58\u50a8\u76ee\u5f55\u5230 OSS-HDFS \u670d\u52a1\u3002\u53c2\u6570\u503c\u4e3a oss://bucket.endpoint/hbase-root-dir hbase.wal.dir \u6307\u5b9a HBase \u7684 WAL \u5b58\u50a8\u76ee\u5f55\u5230 OSS-HDFS \u670d\u52a1\u3002 \u53c2\u6570\u503c\u4e3a oss://bucket.endpoint/hbase","title":"5. \u6307\u5b9a HBase \u7684\u5b58\u50a8\u8def\u5f84"},{"location":"jindosdk/hive/jindosdk_on_hive/","text":"Hive \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e \u00a7 Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002 \u5b89\u88c5\u914d\u7f6e \u00a7 1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/ 2. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key \u00a7 \u914d\u7f6e JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b \u5c06 JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key \u5c06\u5df2\u5f00\u542f OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002 3. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u00a7 \u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u6700\u540e\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 4. Hive on MR \u00a7 \u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002 5. Hive on Tez \u00a7 \u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002 6. Hive on Spark \u00a7 \u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002 OSS-HDFS \u670d\u52a1\u7528\u4e8e\u8868\u7684\u5b58\u50a8 \u00a7 \u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS-HDFS \u670d\u52a1\u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'oss://bucket_name.endpoint_name/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'oss://bucket_name.endpoint_name/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS-HDFS \u670d\u52a1\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>oss://bucket_name.endpoint_name/path/to/warehouse</value> </property> </configuration> \u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a \u00a7 ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'oss://bucket_name.endpoint_name/path/to/us/part210301cn'; \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Hive"},{"location":"jindosdk/hive/jindosdk_on_hive/#hive-jindosdk-oss-hdfs-jindofs","text":"Hive \u662f\u5927\u6570\u636e\u7684\u5e38\u7528\u5de5\u5177\u4e4b\u4e00\uff0c\u5f88\u591a\u7528\u6237\u4f7f\u7528 Hive \u642d\u5efa\u79bb\u7ebf\u6570\u4ed3\u3002\u968f\u7740\u6570\u636e\u91cf\u4e0d\u65ad\u589e\u957f\uff0c\u4f20\u7edf\u7684\u57fa\u4e8e HDFS \u5b58\u50a8\u7684\u6570\u4ed3\u53ef\u80fd\u65e0\u6cd5\u4ee5\u8f83\u4f4e\u6210\u672c\u6ee1\u8db3\u7528\u6237\u7684\u9700\u6c42\uff0c\u7ed3\u5408\u5bf9\u8c61\u5b58\u50a8\u7b49\u4e91\u5b58\u50a8\u4f7f\u7528 Hive \u4e5f\u662f\u4e00\u79cd\u5e38\u89c1\u505a\u6cd5\u3002\u5982\u679c\u8981\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u4f5c\u4e3a Hive \u6570\u4ed3\u7684\u5e95\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoSDK \u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u8bfb\u5199\u6027\u80fd\u4ee5\u53ca\u66f4\u5f3a\u5927\u7684\u6280\u672f\u652f\u6301\u3002","title":"Hive \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"jindosdk/hive/jindosdk_on_hive/#_1","text":"","title":"\u5b89\u88c5\u914d\u7f6e"},{"location":"jindosdk/hive/jindosdk_on_hive/#1-hive-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Hive \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $HIVE_HOME/lib/","title":"1. \u5728 Hive \u5ba2\u6237\u7aef\u6216\u670d\u52a1\u6240\u5728\u7ed3\u70b9\u5b89\u88c5 JindoSDK\u3002"},{"location":"jindosdk/hive/jindosdk_on_hive/#2-oss-hdfs-access-key","text":"\u914d\u7f6e JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b \u5c06 JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230 Hadoop \u7684\u914d\u7f6e\u6587\u4ef6 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key \u5c06\u5df2\u5f00\u542f OSS-HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u7b49\u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK OSS-HDFS \u670d\u52a1 Credential Provider \u914d\u7f6e \u3002","title":"2. \u914d\u7f6e OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b\u53ca Access Key"},{"location":"jindosdk/hive/jindosdk_on_hive/#3-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u6700\u540e\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002","title":"3. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint"},{"location":"jindosdk/hive/jindosdk_on_hive/#4-hive-on-mr","text":"\u5f53\u4f7f\u7528 Hive on MR \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u96c6\u7fa4\u6240\u6709\u7ed3\u70b9\u5747\u5b89\u88c5\u4e86 JindoSDK\uff0c\u9700\u8981\u628a jindosdk-${version}.jar \u653e\u5230 $HADOOP_CLASSPATH \u5e76\u91cd\u542f YARN \u670d\u52a1\u3002\u5e76\u91cd\u542f Hive \u6240\u6709\u670d\u52a1\u3002","title":"4. Hive on MR"},{"location":"jindosdk/hive/jindosdk_on_hive/#5-hive-on-tez","text":"\u5f53\u4f7f\u7528 Hive on Tez \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8fd8\u5e94\u4fdd\u8bc1\u914d\u7f6e tez.lib.uris \u6240\u6307\u5411\u8def\u5f84\u4e2d\u5305\u542b jindosdk-${version}.jar \u3002","title":"5. Hive on Tez"},{"location":"jindosdk/hive/jindosdk_on_hive/#6-hive-on-spark","text":"\u5f53\u4f7f\u7528 Hive on Spark \u65b9\u5f0f\u6267\u884c Hive \u4f5c\u4e1a\u65f6\uff0c\u8bf7\u53c2\u8003 Spark \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u540c\u65f6\u914d\u7f6e\u597d Spark\u3002","title":"6. Hive on Spark"},{"location":"jindosdk/hive/jindosdk_on_hive/#oss-hdfs","text":"\u5728\u521b\u5efa\u6570\u636e\u5e93\u548c\u8868\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u628a\u6570\u636e\u5e93\u6216\u8868\u7684\u6570\u636e\u9ed8\u8ba4\u4fdd\u5b58\u5230 OSS-HDFS \u670d\u52a1\u4e0a\uff1a CREATE DATABASE db_on_oss1 LOCATION 'oss://bucket_name.endpoint_name/path/to/db1'; CREATE TABLE db2.table_on_oss ... LOCATION 'oss://bucket_name.endpoint_name/path/to/db2/tablepath'; \u4e5f\u53ef\u4ee5\u5728 Hive Metastore \u7684 hive-site.xml \u914d\u7f6e\u4e2d\u8bbe\u7f6e hive.metastore.warehouse.dir \u5230 OSS-HDFS \u670d\u52a1\u8def\u5f84\uff0c\u5e76\u91cd\u542f Hive Metastore\uff0c\u5219\u540e\u7eed\u521b\u5efa\u7684\u6570\u636e\u5e93\u548c\u8fd9\u4e9b\u6570\u636e\u5e93\u4e0b\u7684\u8868\u90fd\u4f1a\u9ed8\u8ba4\u5b58\u50a8\u4e8e OSS-HDFS \u670d\u52a1\u3002 <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>oss://bucket_name.endpoint_name/path/to/warehouse</value> </property> </configuration>","title":"OSS-HDFS \u670d\u52a1\u7528\u4e8e\u8868\u7684\u5b58\u50a8"},{"location":"jindosdk/hive/jindosdk_on_hive/#oss","text":"ALTER TABLE existed_table ADD PARTITION (dt='2021-03-01', country='cn') LOCATION 'oss://bucket_name.endpoint_name/path/to/us/part210301cn';","title":"\u7ed9\u5df2\u6709\u8868\u6dfb\u52a0\u4f4d\u4e8e OSS \u7684\u5206\u533a"},{"location":"jindosdk/hive/jindosdk_on_hive/#_2","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"\u53c2\u6570\u8c03\u4f18"},{"location":"jindosdk/presto/jindosdk_on_presto/","text":"Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u6570\u636e \u00a7 Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002 \u6b65\u9aa4 \u00a7 1. \u5b89\u88c5 jar \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/ 2. \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b \u00a7 \u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> 3. \u914d\u7f6e OSS Access Key \u00a7 \u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002 4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u00a7 \u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 5. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002 \u00a7 \u4f7f\u7528\u793a\u4f8b \u00a7 \u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236\u53f0 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='oss://<Bucket>.<Endpoint>/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"Presto"},{"location":"jindosdk/presto/jindosdk_on_presto/#presto-jindosdk-oss-hdfs","text":"Presto \u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5206\u5e03\u5f0f SQL \u67e5\u8be2\u5f15\u64ce\uff0c\u9002\u7528\u4e8e\u4ea4\u4e92\u5f0f\u5206\u6790\u67e5\u8be2\u3002\u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u914d\u7f6e Presto \u901a\u8fc7 JindoSDK \u8bbf\u95ee\u963f\u91cc\u4e91OSS\u6570\u636e\u6e56\u5b58\u50a8\u3002","title":"Presto \u4f7f\u7528 JindoSDK \u67e5\u8be2 OSS-HDFS \u670d\u52a1\u4e0a\u7684\u6570\u636e"},{"location":"jindosdk/presto/jindosdk_on_presto/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"jindosdk/presto/jindosdk_on_presto/#1-jar","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230\u6240\u6709 Presto \u8282\u70b9\u3002 cp jindosdk-x.x.x/lib/*.jar $PRESTO_HOME/plugin/hive-hadoop2/","title":"1. \u5b89\u88c5 jar \u5305"},{"location":"jindosdk/presto/jindosdk_on_presto/#2-jindosdk-oss","text":"\u5c06 JindoSDK OSS \u5b9e\u73b0\u7c7b\u914d\u7f6e\u5230\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration>","title":"2. \u914d\u7f6e JindoSDK OSS \u5b9e\u73b0\u7c7b"},{"location":"jindosdk/presto/jindosdk_on_presto/#3-oss-access-key","text":"\u5c06\u5df2\u5f00\u542f HDFS \u670d\u52a1\u7684 Bucket \u5bf9\u5e94\u7684 Access Key ID \u3001 Access Key Secret \u3001 Endpoint \u7b49\u9884\u5148\u914d\u7f6e\u5728\u6240\u6709 Presto \u8282\u70b9\u4e0a\u7684 Hadoop \u7684 core-site.xml \u4e2d\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 OSS AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002","title":"3. \u914d\u7f6e OSS Access Key"},{"location":"jindosdk/presto/jindosdk_on_presto/#4-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002","title":"4. \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint"},{"location":"jindosdk/presto/jindosdk_on_presto/#5-presto","text":"","title":"5. \u91cd\u542f Presto \u6240\u6709\u670d\u52a1\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002"},{"location":"jindosdk/presto/jindosdk_on_presto/#_2","text":"\u4ee5\u4e0b\u4ee5\u6700\u5e38\u7528\u7684 Hive catalog \u4e3a\u4f8b\uff0c\u4f7f\u7528 Presto \u521b\u5efa\u4e00\u4e2a OSS \u4e0a\u7684 schema\uff0c\u5e76\u6267\u884c\u4e00\u4e9b\u7b80\u5355\u7684 sql \u793a\u4f8b\u3002\u7531\u4e8e\u4f9d\u8d56 Hive Metastore\uff0cHive \u670d\u52a1\u4e5f\u9700\u8981\u5b89\u88c5\u90e8\u7f72 JindoSDK\uff0c\u8bf7\u53c2\u8003 Hive \u4f7f\u7528 JindoSDK \u8bbf\u95ee OSS \u3002 \u6267\u884c\u547d\u4ee4\uff0c\u8fdb\u5165 Presto \u63a7\u5236\u53f0 presto --server <presto_server_address>:<presto_server_port> --catalog hive \u521b\u5efa\u5e76\u4f7f\u7528\u4e00\u4e2a location \u4f4d\u4e8e OSS \u4e0a\u7684 schema create schema testDB with (location='oss://<Bucket>.<Endpoint>/<schema_dir>'); use testDB; \u521b\u5efatable\uff0c\u6267\u884csql\u6d4b\u8bd5\u9a8c\u8bc1 create table tbl (key int, val int); insert into tbl values (1,666); select * from tbl;","title":"\u4f7f\u7528\u793a\u4f8b"},{"location":"jindosdk/python/pyjindo_quickstart/","text":"\u4f7f\u7528 PyJindo \u8bbf\u95ee\u963f\u91cc\u4e91 OSS/OSS-HDFS \u00a7 \u80cc\u666f \u00a7 \u672c\u6587\u6307\u5bfc\u5982\u4f55\u4f7f\u7528 Python \u7684\u5de5\u5177\u5305 PyJindo \u64cd\u4f5c OSS/OSS-HDFS\u3002PyJindo \u517c\u5bb9 python3.6 \u4ee5\u4e0a\u7248\u672c\u3002 \u90e8\u7f72\u73af\u5883 \u00a7 \u4e0b\u8f7d\u5bf9\u5e94 Python \u7248\u672c\u7684 PyJindo \u5b89\u88c5\u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.y.z.tar.gz\uff08x.y.z\u8868\u7248\u672c\u53f7\uff09\uff0c\u4e0b\u8f7d\u94fe\u63a5\u89c1 \u4e0b\u8f7d\u9875\u9762 \u3002 \u90e8\u7f72 jindosdk-6.7.0.tar.gz\uff0cwhl\u5b89\u88c5\u5305\u4f4d\u4e8e\u5b8c\u6574\u4ea7\u51fa\u7269 jindosdk-x.x.x/lib/site-packages/ \u7684\u5b50\u76ee\u5f55\u4e2d\u3002\u591a\u5e73\u53f0\u90e8\u7f72\u8bf4\u660e\u53c2\u89c1 \u90e8\u7f72\u6587\u6863 \u3002 \u4ee5 Python3.6 \u7248\u672c\u4e3a\u4f8b\uff0c\u8bf7\u5b89\u88c5pyjindo-x.y.z-cp 36 -abi3-linux_x86_64.whl . \u251c\u2500\u2500 bin \u2502 \u251c\u2500\u2500 xxx \u251c\u2500\u2500 conf \u2502 \u251c\u2500\u2500 xxx \u251c\u2500\u2500 include \u2502 \u251c\u2500\u2500 xxx \u251c\u2500\u2500 lib \u2502 \u251c\u2500\u2500 xxx \u2502 \u251c\u2500\u2500 native \u2502 \u2502 \u251c\u2500\u2500 xxxx \u2502 \u2514\u2500\u2500 site-packages \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp310-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp311-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp312-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp36-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp37-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp38-abi3-linux_x86_64.whl \u2502 \u2514\u2500\u2500 pyjindo-x.y.z-cp39-abi3-linux_x86_64.whl \u251c\u2500\u2500 plugins \u2502 \u2514\u2500\u2500 xxxx \u251c\u2500\u2500 tools \u2502 \u251c\u2500\u2500 xxx \u2514\u2500\u2500 versions \u251c\u2500\u2500 xxx \u786e\u8ba4\u5b58\u5728\u4ee5\u4e0b\u73af\u5883\u53d8\u91cf\u7528\u4ee5\u8bfb\u53d6\u914d\u7f6e \u00a7 export JINDOSDK_CONF_DIR=/etc/taihao-apps/jindosdk-conf export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf \u5728\u963f\u91cc\u4e91EMR\u73af\u5883\u4e2d\u9ed8\u8ba4\u5b58\u5728\u4ee5\u4e0a\u914d\u7f6e\uff0c\u65e0\u9700\u914d\u7f6e\u3002 \u975eEMR\u4e2d\u914d\u7f6e\u65b9\u5f0f\u53c2\u89c1\uff1a \u300a\u5728\u975eEMR\u96c6\u7fa4\u4e2d\u90e8\u7f72JindoSDK\u300b \u3002\u5176\u4e2d\uff0cHadoop\u914d\u7f6e\u6587\u4ef6\u53caHADOOP_CONF_DIR\u4e0d\u662f\u5fc5\u987b\uff0c\u4ec5\u4e3a\u517c\u5bb9HADOOP\u73af\u5883\u4e2d\u7684\u914d\u7f6e\u3002 \u5b89\u88c5 PyJindo \u00a7 \u4ee5 Python3.8 \u73af\u5883\u5b89\u88c5\u6700\u65b0\u7248\u672c\u7684 pyjindo-6.7.3 \u4e3a\u4f8b python3.8 -m ensurepip python3.8 -m pip install pip --upgrade --trusted-host mirrors.aliyun.com -i http://mirrors.aliyun.com/pypi/simple/ python3.8 -m pip install pyjindo-6.7.3-cp38-abi3-linux_x86_64.whl \u7f16\u5199 python \u6d4b\u8bd5\u7a0b\u5e8f \u00a7 from pyjindo import fs bucket = \"jindosdk-yanbin-sh\" endpoint = bucket + \".cn-shanghai.oss-dls.aliyuncs.com\" root_path = \"oss://\" + endpoint + \"/\" sub_dir = root_path + \"pyjindotest/\" file_path = root_path + \"hello.txt\" file_path2 = sub_dir + \"hello.txt\" config = fs.read_config() fs = fs.connect(root_path, \"root\", config) # open\uff08path, mode\uff09\u6a21\u5f0f w,\u6587\u4ef6\u4e0d\u5b58\u5728\u521b\u5efa\u4e00\u4e2a\u6587\u4ef6 out_file = fs.open(file_path,\"wb\") out_file.write(str.encode(\"hello world, pyjindo\")) #\u5199 out_file.close() in_file = fs.open(file_path, \"rb\") data = in_file.read() # \u8bfb print(\"\u5199\u5165\u7684\u6570\u636e\u4e3a%s.\"%(data)) in_file.close() # \u5217\u51fa\u6587\u4ef6 ls_file = fs.listdir(root_path) print(\"\u76ee\u5f55\u6587\u4ef6\u4e3a%s.\" %(ls_file)) # \u521b\u5efa\u76ee\u5f55 fs.mkdir(sub_dir) # \u79fb\u52a8\u5e76\u91cd\u547d\u540d\u6587\u4ef6 fs.rename(file_path, file_path2) # \u5217\u51fa\u6587\u4ef6 mv_file = fs.listdir(sub_dir) print(\"\u79fb\u52a8\u540e\u7684\u76ee\u5f55\u6587\u4ef6\u4e3a%s.\" %(mv_file)) # \u5220\u9664\u6d4b\u8bd5\u6587\u4ef6\uff0c\u91cd\u65b0\u5217\u51fa\u6587\u4ef6 fs.remove(file_path2) de_file = fs.listdir(sub_dir) print(\"\u5220\u9664\u6587\u4ef6\u540e\u7684pyjindotest\u76ee\u5f55\u4e0b\u6587\u4ef6\u4e3a%s.\" %(de_file)) \u6267\u884c\u6d4b\u8bd5 \u00a7 python3.8 fs_test.py \u6267\u884c\u7ed3\u679c \u00a7 \u5199\u5165\u7684\u6570\u636e\u4e3ab'hello world, pyjindo'. \u76ee\u5f55\u6587\u4ef6\u4e3a[<FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/.sysinfo/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/apps/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/flume/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/hbase/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/hello.txt': type=File, size=20>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/pyarrowtest/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/spark-history/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/tmp/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/user/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/yarn/': type=Directory>]. \u79fb\u52a8\u540e\u7684\u76ee\u5f55\u6587\u4ef6\u4e3a[<FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/pyjindotest/hello.txt': type=File, size=20>]. \u5220\u9664\u6587\u4ef6\u540e\u7684pyjindotest\u76ee\u5f55\u4e0b\u6587\u4ef6\u4e3a[]. \u65e5\u5fd7\u7b49\u7ea7 \u00a7 \u8c03\u6574JINDOSDK_CONF_DIR\u4e0b\u7684jindosdk.cfg\u914d\u7f6e\uff0cemr\u4e0a\u5bf9\u5e94/etc/taihao-apps/jindosdk-conf [common] logger.dir = /var/log/emr/jindosdk logger.level = 2 logger.verbose = 0 logger.sync = false logger.jnilogger = true logger.consolelogger = false logger.cleaner.enable = true \u914d\u7f6e\u9879 \u63cf\u8ff0 logger.dir \u65e5\u5fd7\u8f93\u51fa\u76ee\u5f55\u3002 logger.level \u63a8\u8350\u4e3a2\uff0clevel<=1\uff0c\u8868\u793aWARN\uff1blevel>1\uff0c\u8868\u793aINFO\u3002 logger.verbose \u8be6\u7ec6\u65e5\u5fd7\u7b49\u7ea7\uff0c\u8303\u56f4\u4e3a0-99\uff0c\u503c\u8d8a\u5927\u65e5\u5fd7\u8d8a\u8be6\u7ec6\u3002 logger.sync \u63a8\u8350\u4e3afalse\u3002true\u8868\u793a\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\u3002 logger.jnilogger \u4f7f\u7528jni\u65f6\u751f\u6548\uff0c\u4e0epyjindo\u65e0\u5173\u3002 logger.consolelogger \u7ec8\u7aef\u8f93\u51fa\u65f6\u751f\u6548\uff0c\u4e0epyjindo\u65e0\u5173\u3002 logger.cleaner.enable \u63a8\u8350\u4e3atrue\uff0cfalse\u8868\u793a\u5173\u95ed\u65e5\u5fd7\u81ea\u52a8\u6e05\u7406\u3002 API \u8bf4\u660e \u00a7 Config \u7c7b \u00a7 \u6210\u5458\u51fd\u6570 \u8fd4\u56de\u503c\u7c7b\u578b \u63cf\u8ff0 set(key, val) \u65e0 \u8bbe\u7f6e\u5b57\u7b26\u4e32\u7c7b\u578b\u914d\u7f6e\uff0ckey\u4e0eval\u5747\u4e3astr get(key, default='') str \u83b7\u53d6\u5b57\u7b26\u4e32\u7c7b\u578b\u914d\u7f6e contains(key) bool \u662f\u5426\u5b58\u5728\u914d\u7f6ekey \u914d\u7f6e\u8bf4\u660e\u53c2\u89c1\uff1a \u5ba2\u6237\u7aef\u5e38\u7528\u914d\u7f6e FileType \u679a\u4e3e \u00a7 \u679a\u4e3e\u7c7b\u578b \u679a\u4e3e\u503c \u63cf\u8ff0 Unknown 0 - Directory 1 \u76ee\u5f55 File 2 \u6587\u4ef6 Symlink 3 \u8f6f\u94fe FileInfo \u7c7b \u00a7 \u6210\u5458\u5c5e\u6027 \u8fd4\u56de\u503c\u7c7b\u578b \u63cf\u8ff0 type FileType \u6587\u4ef6\u7c7b\u578b is_file bool \u662f\u5426\u4e3a\u6587\u4ef6 is_dir bool \u662f\u5426\u4e3a\u76ee\u5f55 is_symlink bool \u662f\u5426\u4e3a\u8f6f\u94fe path str \u8def\u5f84 user str \u7528\u6237 group str \u7528\u6237\u7ec4 size int \u6587\u4ef6\u5927\u5c0f perm int \u6587\u4ef6\u6743\u9650 atime datetime \u6587\u4ef6\u6700\u540e\u8bbf\u95ee\u65f6\u95f4 mtime datetime \u6587\u4ef6\u6700\u540e\u4fee\u6539\u65f6\u95f4 FileStream \u7c7b \u00a7 \u6210\u5458\u51fd\u6570 \u8fd4\u56de\u503c\u7c7b\u578b \u63cf\u8ff0 readable() bool \u662f\u5426\u53ef\u8bfb writable() bool \u662f\u5426\u53ef\u5199 seekable() bool \u662f\u5426\u53efseek closed() bool \u662f\u5426\u5df2\u5173\u95ed close() \u65e0\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u5173\u95ed\u6587\u4ef6 size() int, \u5931\u8d25\u65f6\u629b\u51faIOError \u6587\u4ef6\u5927\u5c0f\uff08\u4ec5\u53ef\u8bfb\u65f6\u4f7f\u7528\uff09 tell() int, \u5931\u8d25\u65f6\u629b\u51faIOError \u6587\u4ef6\u6d41\u4f4d\u7f6e flush() \u65e0\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u5237\u65b0\u7f13\u5b58 write(data) \u65e0\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u5199\u6570\u636e\uff0cdata\u4e3abytes\u7c7b\u578b\uff0c\u4e3a\u5199\u5165buffer read(nbytes) bytes, \u5931\u8d25\u65f6\u629b\u51faIOError \u8bfb\u6570\u636e\uff0cnbytes\u4e3aint\uff0c\u4e3a\u8bfb\u53d6\u5927\u5c0f pread(nbytes, offset) bytes, \u5931\u8d25\u65f6\u629b\u51faIOError \u968f\u673a\u8bfb\u6570\u636e\uff0cnbytes\u4e3aint\uff0c\u8868\u8bfb\u53d6\u5927\u5c0f\uff1boffset\u4e3aint\uff0c\u8868\u6587\u4ef6\u4f4d\u79fb readall() bytes, \u5931\u8d25\u65f6\u629b\u51faIOError \u8bfb\u53d6\u6574\u4e2a\u6587\u4ef6 download(stream_or_path, buffer_size) \u65e0, \u5931\u8d25\u65f6\u629b\u51faIOError \u4e0b\u8f7d\uff0c\u8bfb\u53d6\u5f53\u524d\u6587\u4ef6\uff0c\u5199\u5165\u5230\u672c\u5730\u8def\u5f84\u6216\u7740\u76ee\u6807\u6d41\u3002stream_or_path\u53ef\u4ee5\u662f\u672c\u5730\u8def\u5f84\u4e5f\u53ef\u4ee5\u6587\u4ef6\u6d41\u3002 upload(stream, buffer_size) \u65e0\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u4e0a\u4f20\uff0c\u8bfb\u53d6stream\u6d41\uff0c\u5199\u5165\u5f53\u524d\u6587\u4ef6\u3002 FileSystem \u7c7b\u578b \u00a7 \u6210\u5458\u51fd\u6570 \u8fd4\u56de\u503c\u7c7b\u578b \u63cf\u8ff0 mkdir(path, recursive) bool\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u521b\u5efa\u76ee\u5f55\uff0crecursive\u4e3abool\uff0c\u8868\u662f\u5426\u9012\u5f52\u521b\u5efa\u7236\u76ee\u5f55 rename(src, dest) bool\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError rename\u6587\u4ef6\uff0c\u628asrc\u8def\u5f84\u79fb\u52a8\u5230dest\u8def\u5f84\uff0csrc\u548cdest\u8def\u5f84\u4e3astr get_file_info(path) FileInfo\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u83b7\u53d6\u6587\u4ef6\u4fe1\u606f exists(path) bool\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u6587\u4ef6\u662f\u5426\u5b58\u5728 listdir(path, recursive) FileInfo\u5217\u8868\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u5217\u4e3e\u6587\u4ef6\uff0crecursive\u4e3abool\uff0c\u8868\u662f\u5426\u9012\u5f52 chmod(path, perm) bool\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u7c7b\u4f3csetPermission\uff0cperm\u4e3a\u6743\u9650\uff0c\u59820o777 chown(path, owner, group) bool\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u7c7b\u4f3csetOwner\uff0cowner\u4e3astr\uff0c\u8868\u7528\u6237\u540d\uff0cgroup\u4e3astr\uff0c\u8868\u7528\u6237\u7ec4 open(path, mode, buffer_size=None) FileStream\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u6253\u5f00\u6587\u4ef6\uff0cmode\u652f\u6301 rb , wb \u3002buffer_size\u9ed8\u8ba4\u4e3a64k\uff0c\u4e3aupload\u3001download\u3001copy_file\u65f6\u7684buffer\u5927\u5c0f\u3002 download(path, stream_or_path, buffer_size=None) \u65e0, \u5931\u8d25\u65f6\u629b\u51faIOError \u4e0b\u8f7d\uff0c\u8bfb\u53d6\u8fdc\u7aefpath\u7684\u6587\u4ef6\uff0c\u5230\u672c\u5730\u8def\u5f84\u6216\u7740\u76ee\u6807\u6d41\u3002stream_or_path\u53ef\u4ee5\u662f\u672c\u5730\u8def\u5f84\u4e5f\u53ef\u4ee5\u6587\u4ef6\u6d41\u3002buffer_size\u9ed8\u8ba4\u4e3a64k\uff0c\u5982\u914d\u7f6e\u4e2d\u5b58\u5728 fs.oss.read.buffer.size \uff0c\u5219\u4ee5\u914d\u7f6e\u4e3a\u51c6\u3002 upload(path, stream, buffer_size=None) \u65e0, \u5931\u8d25\u65f6\u629b\u51faIOError \u4e0a\u4f20stream\u6d41\u5230\u8fdc\u7aefpath\u3002buffer_size\u9ed8\u8ba4\u4e3a64k\uff0c\u5982\u914d\u7f6e\u4e2d\u5b58\u5728 fs.oss.write.buffer.size \uff0c\u5219\u4ee5\u914d\u7f6e\u4e3a\u51c6\u3002 copy_file(src, dest, buffer_size=None) \u65e0, \u5931\u8d25\u65f6\u629b\u51faIOError \u62f7\u8d1d\u6587\u4ef6\uff0c\u4ecesrc\u8def\u5f84\u62f7\u8d1d\u5230dest\u8def\u5f84\u3002buffer_size\u9ed8\u8ba4\u4e3a64k\uff0c\u5982\u914d\u7f6e\u4e2d\u5b58\u5728 fs.oss.read.buffer.size \uff0c\u5219\u4ee5\u914d\u7f6e\u4e3a\u51c6\u3002 fs \u6a21\u5757 \u00a7 \u5168\u5c40\u51fd\u6570 \u8fd4\u56de\u503c\u7c7b\u578b \u63cf\u8ff0 read_config() Config \u8bfb\u53d6\u914d\u7f6e 1. \u68c0\u67e5\u73af\u5883\u53d8\u91cfJINDOSDK_CONF_DIR\uff0c\u5982\u5b58\u5728\uff0c\u5219\u4ece$JINDOSDK_CONF_DIR/jindosdk.cfg\u4e2d\u8bfb\u53d6\u914d\u7f6e\u3002 2. \u68c0\u67e5\u73af\u5883\u53d8\u91cfHADOOP_CONF_DIR\uff0c\u5982\u5b58\u5728\u4e14jindosdk.cfg\u4e2d\u672a\u914d\u7f6ehadoopConf.enable\u4e3afalse\uff0c\u5219\u4ece$HADOOP_CONF_DIR/core-sites.xml\u4e2d\u8bfb\u53d6\u914d\u7f6e\u3002 connect(uri, user, config) FileSystem\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u521d\u59cb\u5316 FileSystem API \u517c\u5bb9\u6027 \u00a7 ossfs \u6a21\u5757\u517c\u5bb9 fsspec \u63a5\u53e3 \u00a7 pyjindo \u540c\u6837\u517c\u5bb9 fsspec \u540c\u6b65\u63a5\u53e3\uff0c\u4f46\u4ec5\u517c\u5bb9 python3.7 \u4ee5\u4e0a\u7248\u672c\u3002\u4ec5\u63a5\u53e3\u517c\u5bb9\uff0c\u8bfb\u53d6\u914d\u7f6e\u3001\u65e5\u5fd7\u7b49\u7ea7\u7b49\u4f7f\u7528\u65b9\u5f0f\u4e0d\u53d8\u3002 \u4f7f\u7528\u524d\u9700\u8981\u5b89\u88c5\u4f9d\u8d56 fsspec \u00a7 \u4ee5 Python3.8 \u73af\u5883\u5b89\u88c5 fssepc \u4e3a\u4f8b python3.8 -m pip install fsspec --trusted-host mirrors.aliyun.com -i http://mirrors.aliyun.com/pypi/simple/ \u7f16\u5199 python \u6d4b\u8bd5\u7a0b\u5e8f \u00a7 from pyjindo.ossfs import JindoOssFileSystem bucket = \"jindosdk-yanbin-sh\" endpoint = bucket + \".cn-shanghai.oss-dls.aliyuncs.com\" root_path = \"oss://\" + endpoint + \"/\" sub_dir = root_path + \"pyjindotest/\" file_path = root_path + \"hello.txt\" file_path2 = sub_dir + \"hello.txt\" fs = JindoOssFileSystem(root_path) # open\uff08path, mode\uff09\u6a21\u5f0f w,\u6587\u4ef6\u4e0d\u5b58\u5728\u521b\u5efa\u4e00\u4e2a\u6587\u4ef6 out_file = fs.open(file_path,\"wb\") out_file.write(str.encode(\"hello world, pyjindo\")) #\u5199 out_file.close() in_file = fs.open(file_path, \"rb\") data = in_file.read() # \u8bfb print(\"\u5199\u5165\u7684\u6570\u636e\u4e3a%s.\"%(data)) in_file.close() # \u5217\u51fa\u6587\u4ef6 ls_file = fs.ls(root_path, detail=False) print(\"\u76ee\u5f55\u6587\u4ef6\u4e3a%s.\" %(ls_file)) assert file_path in fs.glob(root_path + \"*\") # \u521b\u5efa\u76ee\u5f55 fs.mkdir(sub_dir) # \u79fb\u52a8\u5e76\u91cd\u547d\u540d\u6587\u4ef6 fs.rename(file_path, file_path2) # \u5217\u51fa\u6587\u4ef6 mv_file = fs.listdir(sub_dir, detail=False) print(\"\u79fb\u52a8\u540e\u7684\u76ee\u5f55\u6587\u4ef6\u4e3a%s.\" %(mv_file)) # \u5220\u9664\u6d4b\u8bd5\u6587\u4ef6\uff0c\u91cd\u65b0\u5217\u51fa\u6587\u4ef6 fs.rm(file_path2) de_file = fs.ls(sub_dir) print(\"\u5220\u9664\u6587\u4ef6\u540e\u7684pyjindotest\u76ee\u5f55\u4e0b\u6587\u4ef6\u4e3a%s.\" %(de_file)) \u6267\u884c\u6d4b\u8bd5 \u00a7 python3.8 ossfs_test.py \u6267\u884c\u7ed3\u679c\uff1a \u00a7 \u5199\u5165\u7684\u6570\u636e\u4e3ab'hello world, pyjindo'. \u76ee\u5f55\u6587\u4ef6\u4e3a['oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/.sysinfo/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/apps/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/flume/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/hbase/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/hello.txt', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/pyarrowtest/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/pyjindotest/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/spark-history/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/test/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/tmp/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/user/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/yarn/']. \u79fb\u52a8\u540e\u7684\u76ee\u5f55\u6587\u4ef6\u4e3a['oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/pyjindotest/hello.txt']. \u5220\u9664\u6587\u4ef6\u540e\u7684pyjindotest\u76ee\u5f55\u4e0b\u6587\u4ef6\u4e3a[]. \u66f4\u591a\u63a5\u53e3\u8bf4\u660e\u53c2\u89c1 fssepc \u5b98\u7f51\u6587\u6863 \u3002","title":"Python"},{"location":"jindosdk/python/pyjindo_quickstart/#pyjindo-ossoss-hdfs","text":"","title":"\u4f7f\u7528 PyJindo \u8bbf\u95ee\u963f\u91cc\u4e91 OSS/OSS-HDFS"},{"location":"jindosdk/python/pyjindo_quickstart/#_1","text":"\u672c\u6587\u6307\u5bfc\u5982\u4f55\u4f7f\u7528 Python \u7684\u5de5\u5177\u5305 PyJindo \u64cd\u4f5c OSS/OSS-HDFS\u3002PyJindo \u517c\u5bb9 python3.6 \u4ee5\u4e0a\u7248\u672c\u3002","title":"\u80cc\u666f"},{"location":"jindosdk/python/pyjindo_quickstart/#_2","text":"","title":"\u90e8\u7f72\u73af\u5883"},{"location":"jindosdk/python/pyjindo_quickstart/#python-pyjindo","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.y.z.tar.gz\uff08x.y.z\u8868\u7248\u672c\u53f7\uff09\uff0c\u4e0b\u8f7d\u94fe\u63a5\u89c1 \u4e0b\u8f7d\u9875\u9762 \u3002 \u90e8\u7f72 jindosdk-6.7.0.tar.gz\uff0cwhl\u5b89\u88c5\u5305\u4f4d\u4e8e\u5b8c\u6574\u4ea7\u51fa\u7269 jindosdk-x.x.x/lib/site-packages/ \u7684\u5b50\u76ee\u5f55\u4e2d\u3002\u591a\u5e73\u53f0\u90e8\u7f72\u8bf4\u660e\u53c2\u89c1 \u90e8\u7f72\u6587\u6863 \u3002 \u4ee5 Python3.6 \u7248\u672c\u4e3a\u4f8b\uff0c\u8bf7\u5b89\u88c5pyjindo-x.y.z-cp 36 -abi3-linux_x86_64.whl . \u251c\u2500\u2500 bin \u2502 \u251c\u2500\u2500 xxx \u251c\u2500\u2500 conf \u2502 \u251c\u2500\u2500 xxx \u251c\u2500\u2500 include \u2502 \u251c\u2500\u2500 xxx \u251c\u2500\u2500 lib \u2502 \u251c\u2500\u2500 xxx \u2502 \u251c\u2500\u2500 native \u2502 \u2502 \u251c\u2500\u2500 xxxx \u2502 \u2514\u2500\u2500 site-packages \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp310-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp311-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp312-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp36-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp37-abi3-linux_x86_64.whl \u2502 \u251c\u2500\u2500 pyjindo-x.y.z-cp38-abi3-linux_x86_64.whl \u2502 \u2514\u2500\u2500 pyjindo-x.y.z-cp39-abi3-linux_x86_64.whl \u251c\u2500\u2500 plugins \u2502 \u2514\u2500\u2500 xxxx \u251c\u2500\u2500 tools \u2502 \u251c\u2500\u2500 xxx \u2514\u2500\u2500 versions \u251c\u2500\u2500 xxx","title":"\u4e0b\u8f7d\u5bf9\u5e94 Python \u7248\u672c\u7684 PyJindo \u5b89\u88c5\u5305"},{"location":"jindosdk/python/pyjindo_quickstart/#_3","text":"export JINDOSDK_CONF_DIR=/etc/taihao-apps/jindosdk-conf export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf \u5728\u963f\u91cc\u4e91EMR\u73af\u5883\u4e2d\u9ed8\u8ba4\u5b58\u5728\u4ee5\u4e0a\u914d\u7f6e\uff0c\u65e0\u9700\u914d\u7f6e\u3002 \u975eEMR\u4e2d\u914d\u7f6e\u65b9\u5f0f\u53c2\u89c1\uff1a \u300a\u5728\u975eEMR\u96c6\u7fa4\u4e2d\u90e8\u7f72JindoSDK\u300b \u3002\u5176\u4e2d\uff0cHadoop\u914d\u7f6e\u6587\u4ef6\u53caHADOOP_CONF_DIR\u4e0d\u662f\u5fc5\u987b\uff0c\u4ec5\u4e3a\u517c\u5bb9HADOOP\u73af\u5883\u4e2d\u7684\u914d\u7f6e\u3002","title":"\u786e\u8ba4\u5b58\u5728\u4ee5\u4e0b\u73af\u5883\u53d8\u91cf\u7528\u4ee5\u8bfb\u53d6\u914d\u7f6e"},{"location":"jindosdk/python/pyjindo_quickstart/#pyjindo","text":"\u4ee5 Python3.8 \u73af\u5883\u5b89\u88c5\u6700\u65b0\u7248\u672c\u7684 pyjindo-6.7.3 \u4e3a\u4f8b python3.8 -m ensurepip python3.8 -m pip install pip --upgrade --trusted-host mirrors.aliyun.com -i http://mirrors.aliyun.com/pypi/simple/ python3.8 -m pip install pyjindo-6.7.3-cp38-abi3-linux_x86_64.whl","title":"\u5b89\u88c5 PyJindo"},{"location":"jindosdk/python/pyjindo_quickstart/#python","text":"from pyjindo import fs bucket = \"jindosdk-yanbin-sh\" endpoint = bucket + \".cn-shanghai.oss-dls.aliyuncs.com\" root_path = \"oss://\" + endpoint + \"/\" sub_dir = root_path + \"pyjindotest/\" file_path = root_path + \"hello.txt\" file_path2 = sub_dir + \"hello.txt\" config = fs.read_config() fs = fs.connect(root_path, \"root\", config) # open\uff08path, mode\uff09\u6a21\u5f0f w,\u6587\u4ef6\u4e0d\u5b58\u5728\u521b\u5efa\u4e00\u4e2a\u6587\u4ef6 out_file = fs.open(file_path,\"wb\") out_file.write(str.encode(\"hello world, pyjindo\")) #\u5199 out_file.close() in_file = fs.open(file_path, \"rb\") data = in_file.read() # \u8bfb print(\"\u5199\u5165\u7684\u6570\u636e\u4e3a%s.\"%(data)) in_file.close() # \u5217\u51fa\u6587\u4ef6 ls_file = fs.listdir(root_path) print(\"\u76ee\u5f55\u6587\u4ef6\u4e3a%s.\" %(ls_file)) # \u521b\u5efa\u76ee\u5f55 fs.mkdir(sub_dir) # \u79fb\u52a8\u5e76\u91cd\u547d\u540d\u6587\u4ef6 fs.rename(file_path, file_path2) # \u5217\u51fa\u6587\u4ef6 mv_file = fs.listdir(sub_dir) print(\"\u79fb\u52a8\u540e\u7684\u76ee\u5f55\u6587\u4ef6\u4e3a%s.\" %(mv_file)) # \u5220\u9664\u6d4b\u8bd5\u6587\u4ef6\uff0c\u91cd\u65b0\u5217\u51fa\u6587\u4ef6 fs.remove(file_path2) de_file = fs.listdir(sub_dir) print(\"\u5220\u9664\u6587\u4ef6\u540e\u7684pyjindotest\u76ee\u5f55\u4e0b\u6587\u4ef6\u4e3a%s.\" %(de_file))","title":"\u7f16\u5199 python \u6d4b\u8bd5\u7a0b\u5e8f"},{"location":"jindosdk/python/pyjindo_quickstart/#_4","text":"python3.8 fs_test.py","title":"\u6267\u884c\u6d4b\u8bd5"},{"location":"jindosdk/python/pyjindo_quickstart/#_5","text":"\u5199\u5165\u7684\u6570\u636e\u4e3ab'hello world, pyjindo'. \u76ee\u5f55\u6587\u4ef6\u4e3a[<FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/.sysinfo/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/apps/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/flume/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/hbase/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/hello.txt': type=File, size=20>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/pyarrowtest/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/spark-history/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/tmp/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/user/': type=Directory>, <FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/yarn/': type=Directory>]. \u79fb\u52a8\u540e\u7684\u76ee\u5f55\u6587\u4ef6\u4e3a[<FileInfo for 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/pyjindotest/hello.txt': type=File, size=20>]. \u5220\u9664\u6587\u4ef6\u540e\u7684pyjindotest\u76ee\u5f55\u4e0b\u6587\u4ef6\u4e3a[].","title":"\u6267\u884c\u7ed3\u679c"},{"location":"jindosdk/python/pyjindo_quickstart/#_6","text":"\u8c03\u6574JINDOSDK_CONF_DIR\u4e0b\u7684jindosdk.cfg\u914d\u7f6e\uff0cemr\u4e0a\u5bf9\u5e94/etc/taihao-apps/jindosdk-conf [common] logger.dir = /var/log/emr/jindosdk logger.level = 2 logger.verbose = 0 logger.sync = false logger.jnilogger = true logger.consolelogger = false logger.cleaner.enable = true \u914d\u7f6e\u9879 \u63cf\u8ff0 logger.dir \u65e5\u5fd7\u8f93\u51fa\u76ee\u5f55\u3002 logger.level \u63a8\u8350\u4e3a2\uff0clevel<=1\uff0c\u8868\u793aWARN\uff1blevel>1\uff0c\u8868\u793aINFO\u3002 logger.verbose \u8be6\u7ec6\u65e5\u5fd7\u7b49\u7ea7\uff0c\u8303\u56f4\u4e3a0-99\uff0c\u503c\u8d8a\u5927\u65e5\u5fd7\u8d8a\u8be6\u7ec6\u3002 logger.sync \u63a8\u8350\u4e3afalse\u3002true\u8868\u793a\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\u3002 logger.jnilogger \u4f7f\u7528jni\u65f6\u751f\u6548\uff0c\u4e0epyjindo\u65e0\u5173\u3002 logger.consolelogger \u7ec8\u7aef\u8f93\u51fa\u65f6\u751f\u6548\uff0c\u4e0epyjindo\u65e0\u5173\u3002 logger.cleaner.enable \u63a8\u8350\u4e3atrue\uff0cfalse\u8868\u793a\u5173\u95ed\u65e5\u5fd7\u81ea\u52a8\u6e05\u7406\u3002","title":"\u65e5\u5fd7\u7b49\u7ea7"},{"location":"jindosdk/python/pyjindo_quickstart/#api","text":"","title":"API \u8bf4\u660e"},{"location":"jindosdk/python/pyjindo_quickstart/#config","text":"\u6210\u5458\u51fd\u6570 \u8fd4\u56de\u503c\u7c7b\u578b \u63cf\u8ff0 set(key, val) \u65e0 \u8bbe\u7f6e\u5b57\u7b26\u4e32\u7c7b\u578b\u914d\u7f6e\uff0ckey\u4e0eval\u5747\u4e3astr get(key, default='') str \u83b7\u53d6\u5b57\u7b26\u4e32\u7c7b\u578b\u914d\u7f6e contains(key) bool \u662f\u5426\u5b58\u5728\u914d\u7f6ekey \u914d\u7f6e\u8bf4\u660e\u53c2\u89c1\uff1a \u5ba2\u6237\u7aef\u5e38\u7528\u914d\u7f6e","title":"Config \u7c7b"},{"location":"jindosdk/python/pyjindo_quickstart/#filetype","text":"\u679a\u4e3e\u7c7b\u578b \u679a\u4e3e\u503c \u63cf\u8ff0 Unknown 0 - Directory 1 \u76ee\u5f55 File 2 \u6587\u4ef6 Symlink 3 \u8f6f\u94fe","title":"FileType \u679a\u4e3e"},{"location":"jindosdk/python/pyjindo_quickstart/#fileinfo","text":"\u6210\u5458\u5c5e\u6027 \u8fd4\u56de\u503c\u7c7b\u578b \u63cf\u8ff0 type FileType \u6587\u4ef6\u7c7b\u578b is_file bool \u662f\u5426\u4e3a\u6587\u4ef6 is_dir bool \u662f\u5426\u4e3a\u76ee\u5f55 is_symlink bool \u662f\u5426\u4e3a\u8f6f\u94fe path str \u8def\u5f84 user str \u7528\u6237 group str \u7528\u6237\u7ec4 size int \u6587\u4ef6\u5927\u5c0f perm int \u6587\u4ef6\u6743\u9650 atime datetime \u6587\u4ef6\u6700\u540e\u8bbf\u95ee\u65f6\u95f4 mtime datetime \u6587\u4ef6\u6700\u540e\u4fee\u6539\u65f6\u95f4","title":"FileInfo \u7c7b"},{"location":"jindosdk/python/pyjindo_quickstart/#filestream","text":"\u6210\u5458\u51fd\u6570 \u8fd4\u56de\u503c\u7c7b\u578b \u63cf\u8ff0 readable() bool \u662f\u5426\u53ef\u8bfb writable() bool \u662f\u5426\u53ef\u5199 seekable() bool \u662f\u5426\u53efseek closed() bool \u662f\u5426\u5df2\u5173\u95ed close() \u65e0\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u5173\u95ed\u6587\u4ef6 size() int, \u5931\u8d25\u65f6\u629b\u51faIOError \u6587\u4ef6\u5927\u5c0f\uff08\u4ec5\u53ef\u8bfb\u65f6\u4f7f\u7528\uff09 tell() int, \u5931\u8d25\u65f6\u629b\u51faIOError \u6587\u4ef6\u6d41\u4f4d\u7f6e flush() \u65e0\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u5237\u65b0\u7f13\u5b58 write(data) \u65e0\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u5199\u6570\u636e\uff0cdata\u4e3abytes\u7c7b\u578b\uff0c\u4e3a\u5199\u5165buffer read(nbytes) bytes, \u5931\u8d25\u65f6\u629b\u51faIOError \u8bfb\u6570\u636e\uff0cnbytes\u4e3aint\uff0c\u4e3a\u8bfb\u53d6\u5927\u5c0f pread(nbytes, offset) bytes, \u5931\u8d25\u65f6\u629b\u51faIOError \u968f\u673a\u8bfb\u6570\u636e\uff0cnbytes\u4e3aint\uff0c\u8868\u8bfb\u53d6\u5927\u5c0f\uff1boffset\u4e3aint\uff0c\u8868\u6587\u4ef6\u4f4d\u79fb readall() bytes, \u5931\u8d25\u65f6\u629b\u51faIOError \u8bfb\u53d6\u6574\u4e2a\u6587\u4ef6 download(stream_or_path, buffer_size) \u65e0, \u5931\u8d25\u65f6\u629b\u51faIOError \u4e0b\u8f7d\uff0c\u8bfb\u53d6\u5f53\u524d\u6587\u4ef6\uff0c\u5199\u5165\u5230\u672c\u5730\u8def\u5f84\u6216\u7740\u76ee\u6807\u6d41\u3002stream_or_path\u53ef\u4ee5\u662f\u672c\u5730\u8def\u5f84\u4e5f\u53ef\u4ee5\u6587\u4ef6\u6d41\u3002 upload(stream, buffer_size) \u65e0\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u4e0a\u4f20\uff0c\u8bfb\u53d6stream\u6d41\uff0c\u5199\u5165\u5f53\u524d\u6587\u4ef6\u3002","title":"FileStream \u7c7b"},{"location":"jindosdk/python/pyjindo_quickstart/#filesystem","text":"\u6210\u5458\u51fd\u6570 \u8fd4\u56de\u503c\u7c7b\u578b \u63cf\u8ff0 mkdir(path, recursive) bool\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u521b\u5efa\u76ee\u5f55\uff0crecursive\u4e3abool\uff0c\u8868\u662f\u5426\u9012\u5f52\u521b\u5efa\u7236\u76ee\u5f55 rename(src, dest) bool\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError rename\u6587\u4ef6\uff0c\u628asrc\u8def\u5f84\u79fb\u52a8\u5230dest\u8def\u5f84\uff0csrc\u548cdest\u8def\u5f84\u4e3astr get_file_info(path) FileInfo\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u83b7\u53d6\u6587\u4ef6\u4fe1\u606f exists(path) bool\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u6587\u4ef6\u662f\u5426\u5b58\u5728 listdir(path, recursive) FileInfo\u5217\u8868\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u5217\u4e3e\u6587\u4ef6\uff0crecursive\u4e3abool\uff0c\u8868\u662f\u5426\u9012\u5f52 chmod(path, perm) bool\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u7c7b\u4f3csetPermission\uff0cperm\u4e3a\u6743\u9650\uff0c\u59820o777 chown(path, owner, group) bool\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u7c7b\u4f3csetOwner\uff0cowner\u4e3astr\uff0c\u8868\u7528\u6237\u540d\uff0cgroup\u4e3astr\uff0c\u8868\u7528\u6237\u7ec4 open(path, mode, buffer_size=None) FileStream\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u6253\u5f00\u6587\u4ef6\uff0cmode\u652f\u6301 rb , wb \u3002buffer_size\u9ed8\u8ba4\u4e3a64k\uff0c\u4e3aupload\u3001download\u3001copy_file\u65f6\u7684buffer\u5927\u5c0f\u3002 download(path, stream_or_path, buffer_size=None) \u65e0, \u5931\u8d25\u65f6\u629b\u51faIOError \u4e0b\u8f7d\uff0c\u8bfb\u53d6\u8fdc\u7aefpath\u7684\u6587\u4ef6\uff0c\u5230\u672c\u5730\u8def\u5f84\u6216\u7740\u76ee\u6807\u6d41\u3002stream_or_path\u53ef\u4ee5\u662f\u672c\u5730\u8def\u5f84\u4e5f\u53ef\u4ee5\u6587\u4ef6\u6d41\u3002buffer_size\u9ed8\u8ba4\u4e3a64k\uff0c\u5982\u914d\u7f6e\u4e2d\u5b58\u5728 fs.oss.read.buffer.size \uff0c\u5219\u4ee5\u914d\u7f6e\u4e3a\u51c6\u3002 upload(path, stream, buffer_size=None) \u65e0, \u5931\u8d25\u65f6\u629b\u51faIOError \u4e0a\u4f20stream\u6d41\u5230\u8fdc\u7aefpath\u3002buffer_size\u9ed8\u8ba4\u4e3a64k\uff0c\u5982\u914d\u7f6e\u4e2d\u5b58\u5728 fs.oss.write.buffer.size \uff0c\u5219\u4ee5\u914d\u7f6e\u4e3a\u51c6\u3002 copy_file(src, dest, buffer_size=None) \u65e0, \u5931\u8d25\u65f6\u629b\u51faIOError \u62f7\u8d1d\u6587\u4ef6\uff0c\u4ecesrc\u8def\u5f84\u62f7\u8d1d\u5230dest\u8def\u5f84\u3002buffer_size\u9ed8\u8ba4\u4e3a64k\uff0c\u5982\u914d\u7f6e\u4e2d\u5b58\u5728 fs.oss.read.buffer.size \uff0c\u5219\u4ee5\u914d\u7f6e\u4e3a\u51c6\u3002","title":"FileSystem \u7c7b\u578b"},{"location":"jindosdk/python/pyjindo_quickstart/#fs","text":"\u5168\u5c40\u51fd\u6570 \u8fd4\u56de\u503c\u7c7b\u578b \u63cf\u8ff0 read_config() Config \u8bfb\u53d6\u914d\u7f6e 1. \u68c0\u67e5\u73af\u5883\u53d8\u91cfJINDOSDK_CONF_DIR\uff0c\u5982\u5b58\u5728\uff0c\u5219\u4ece$JINDOSDK_CONF_DIR/jindosdk.cfg\u4e2d\u8bfb\u53d6\u914d\u7f6e\u3002 2. \u68c0\u67e5\u73af\u5883\u53d8\u91cfHADOOP_CONF_DIR\uff0c\u5982\u5b58\u5728\u4e14jindosdk.cfg\u4e2d\u672a\u914d\u7f6ehadoopConf.enable\u4e3afalse\uff0c\u5219\u4ece$HADOOP_CONF_DIR/core-sites.xml\u4e2d\u8bfb\u53d6\u914d\u7f6e\u3002 connect(uri, user, config) FileSystem\uff0c\u5931\u8d25\u65f6\u629b\u51faIOError \u521d\u59cb\u5316 FileSystem","title":"fs \u6a21\u5757"},{"location":"jindosdk/python/pyjindo_quickstart/#api_1","text":"","title":"API \u517c\u5bb9\u6027"},{"location":"jindosdk/python/pyjindo_quickstart/#ossfs-fsspec","text":"pyjindo \u540c\u6837\u517c\u5bb9 fsspec \u540c\u6b65\u63a5\u53e3\uff0c\u4f46\u4ec5\u517c\u5bb9 python3.7 \u4ee5\u4e0a\u7248\u672c\u3002\u4ec5\u63a5\u53e3\u517c\u5bb9\uff0c\u8bfb\u53d6\u914d\u7f6e\u3001\u65e5\u5fd7\u7b49\u7ea7\u7b49\u4f7f\u7528\u65b9\u5f0f\u4e0d\u53d8\u3002","title":"ossfs \u6a21\u5757\u517c\u5bb9 fsspec \u63a5\u53e3"},{"location":"jindosdk/python/pyjindo_quickstart/#fsspec","text":"\u4ee5 Python3.8 \u73af\u5883\u5b89\u88c5 fssepc \u4e3a\u4f8b python3.8 -m pip install fsspec --trusted-host mirrors.aliyun.com -i http://mirrors.aliyun.com/pypi/simple/","title":"\u4f7f\u7528\u524d\u9700\u8981\u5b89\u88c5\u4f9d\u8d56 fsspec"},{"location":"jindosdk/python/pyjindo_quickstart/#python_1","text":"from pyjindo.ossfs import JindoOssFileSystem bucket = \"jindosdk-yanbin-sh\" endpoint = bucket + \".cn-shanghai.oss-dls.aliyuncs.com\" root_path = \"oss://\" + endpoint + \"/\" sub_dir = root_path + \"pyjindotest/\" file_path = root_path + \"hello.txt\" file_path2 = sub_dir + \"hello.txt\" fs = JindoOssFileSystem(root_path) # open\uff08path, mode\uff09\u6a21\u5f0f w,\u6587\u4ef6\u4e0d\u5b58\u5728\u521b\u5efa\u4e00\u4e2a\u6587\u4ef6 out_file = fs.open(file_path,\"wb\") out_file.write(str.encode(\"hello world, pyjindo\")) #\u5199 out_file.close() in_file = fs.open(file_path, \"rb\") data = in_file.read() # \u8bfb print(\"\u5199\u5165\u7684\u6570\u636e\u4e3a%s.\"%(data)) in_file.close() # \u5217\u51fa\u6587\u4ef6 ls_file = fs.ls(root_path, detail=False) print(\"\u76ee\u5f55\u6587\u4ef6\u4e3a%s.\" %(ls_file)) assert file_path in fs.glob(root_path + \"*\") # \u521b\u5efa\u76ee\u5f55 fs.mkdir(sub_dir) # \u79fb\u52a8\u5e76\u91cd\u547d\u540d\u6587\u4ef6 fs.rename(file_path, file_path2) # \u5217\u51fa\u6587\u4ef6 mv_file = fs.listdir(sub_dir, detail=False) print(\"\u79fb\u52a8\u540e\u7684\u76ee\u5f55\u6587\u4ef6\u4e3a%s.\" %(mv_file)) # \u5220\u9664\u6d4b\u8bd5\u6587\u4ef6\uff0c\u91cd\u65b0\u5217\u51fa\u6587\u4ef6 fs.rm(file_path2) de_file = fs.ls(sub_dir) print(\"\u5220\u9664\u6587\u4ef6\u540e\u7684pyjindotest\u76ee\u5f55\u4e0b\u6587\u4ef6\u4e3a%s.\" %(de_file))","title":"\u7f16\u5199 python \u6d4b\u8bd5\u7a0b\u5e8f"},{"location":"jindosdk/python/pyjindo_quickstart/#_7","text":"python3.8 ossfs_test.py","title":"\u6267\u884c\u6d4b\u8bd5"},{"location":"jindosdk/python/pyjindo_quickstart/#_8","text":"\u5199\u5165\u7684\u6570\u636e\u4e3ab'hello world, pyjindo'. \u76ee\u5f55\u6587\u4ef6\u4e3a['oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/.sysinfo/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/apps/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/flume/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/hbase/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/hello.txt', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/pyarrowtest/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/pyjindotest/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/spark-history/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/test/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/tmp/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/user/', 'oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/yarn/']. \u79fb\u52a8\u540e\u7684\u76ee\u5f55\u6587\u4ef6\u4e3a['oss://jindosdk-yanbin-sh.cn-shanghai.oss-dls.aliyuncs.com/pyjindotest/hello.txt']. \u5220\u9664\u6587\u4ef6\u540e\u7684pyjindotest\u76ee\u5f55\u4e0b\u6587\u4ef6\u4e3a[]. \u66f4\u591a\u63a5\u53e3\u8bf4\u660e\u53c2\u89c1 fssepc \u5b98\u7f51\u6587\u6863 \u3002","title":"\u6267\u884c\u7ed3\u679c\uff1a"},{"location":"jindosdk/spark/jindosdk_ide_spark/","text":"Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5 \u00a7 \u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.spark.sql.SparkSession; public class TestJindoSDK { public static void main(String[] args) throws Exception { SparkSession spark = SparkSession .builder() .config(\"spark.hadoop.fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\") .config(\"spark.hadoop.fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\") .config(\"spark.hadoop.fs.oss.accessKeyId\", \"xxx\") .config(\"spark.hadoop.fs.oss.accessKeySecret\", \"xxx\") // set accessKey, secret, endpoint and so on. .appName(\"TestJindoSDK\") .getOrCreate(); spark.read().parquet(\"oss://bucket.endpoint/xxx\").count(); spark.stop(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <!-- ... --> <dependencies> <!-- add jindo-core-extended-jar for macos --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> </dependencies> </project>","title":"Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5"},{"location":"jindosdk/spark/jindosdk_ide_spark/#spark-jindosdk-ide","text":"\u76ee\u524d JindoSDK \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff09 \u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoSDK \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-core --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar if you need support other platform --> <!-- add jindo-hadoop-sdk --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-sdk</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> </dependencies> </project> \u7136\u540e\u60a8\u53ef\u4ee5\u7f16\u5199Java\u7a0b\u5e8f\u4f7f\u7528SDK import org.apache.spark.sql.SparkSession; public class TestJindoSDK { public static void main(String[] args) throws Exception { SparkSession spark = SparkSession .builder() .config(\"spark.hadoop.fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\") .config(\"spark.hadoop.fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\") .config(\"spark.hadoop.fs.oss.accessKeyId\", \"xxx\") .config(\"spark.hadoop.fs.oss.accessKeySecret\", \"xxx\") // set accessKey, secret, endpoint and so on. .appName(\"TestJindoSDK\") .getOrCreate(); spark.read().parquet(\"oss://bucket.endpoint/xxx\").count(); spark.stop(); } } \u5176\u4ed6\u5e73\u53f0\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <!-- ... --> <dependencies> <!-- add jindo-core-extended-jar for macos --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-macos-11_0-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for centos6 or el6 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el6-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for ubuntu22 --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-ubuntu22-x86_64</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add jindo-core-extended-jar for aliyun yitian & alios (beta)--> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-core-linux-el7-aarch64</artifactId> <version>${jindodata.version}</version> </dependency> </dependencies> </project>","title":"Spark \u4f7f\u7528 JindoSDK \u5728 IDE \u5f00\u53d1\u8c03\u8bd5"},{"location":"jindosdk/spark/jindosdk_on_spark/","text":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e \u00a7 JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411Hadoop/Spark\u751f\u6001\u7684OSS\u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684Hadoop FileSystem\u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002 \u6b65\u9aa4 \u00a7 1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e \u00a7 \u786e\u4fddSpark\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002 2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/ 3. \u914d\u7f6e JindoSDK \u00a7 \u5168\u5c40\u914d\u7f6e, \u4fee\u6539 Spark \u4f7f\u7528\u7684 core-site.xml \u00a7 \u914d\u7f6e JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002 \u4efb\u52a1\u7ea7\u522b\u914d\u7f6e \u00a7 \u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx 4. \u4f7f\u7528 Spark \u8bbf\u95ee OSS \u00a7 \u521b\u5efa\u8868 create table test_oss (c1 string) location \"oss://bucket.endpoint/dir\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss; 5. \u53c2\u6570\u8c03\u4f18 \u00a7 JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"jindosdk/spark/jindosdk_on_spark/#spark-jindosdk-oss-hdfs-jindofs","text":"JindoSDK\u662f\u4e00\u4e2a\u7b80\u5355\u6613\u7528\u9762\u5411Hadoop/Spark\u751f\u6001\u7684OSS\u5ba2\u6237\u7aef\uff0c\u4e3a\u963f\u91cc\u4e91OSS\u63d0\u4f9b\u9ad8\u5ea6\u4f18\u5316\u7684Hadoop FileSystem\u5b9e\u73b0, Spark \u4f7f\u7528 JindoSDK \u76f8\u5bf9\u4e8e\u4f7f\u7528 Hadoop \u793e\u533a OSS \u5ba2\u6237\u7aef\uff0c\u53ef\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd,\u540c\u65f6\u8fd8\u80fd\u83b7\u5f97\u963f\u91cc\u4e91 E-MapReduce \u4ea7\u54c1\u6280\u672f\u56e2\u961f\u66f4\u4e13\u4e1a\u7684\u652f\u6301\u3002","title":"Spark \u4f7f\u7528 JindoSDK \u5904\u7406\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u4e0a\u7684\u6570\u636e"},{"location":"jindosdk/spark/jindosdk_on_spark/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"jindosdk/spark/jindosdk_on_spark/#1-spark-hadoop","text":"\u786e\u4fddSpark\u7684\u914d\u7f6e\u6587\u4ef6\u4e2d\u5305\u542b\u4e86 HADOOP \u76f8\u5173\u7684\u914d\u7f6e\u6587\u4ef6\u3002","title":"1. Spark \u5df2\u7ecf\u914d\u7f6e HADOOP \u76f8\u5173\u914d\u7f6e"},{"location":"jindosdk/spark/jindosdk_on_spark/#2-spark-classpath-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\uff0c\u89e3\u538b\u540e\u5c06sdk\u5305\u5b89\u88c5\u5230 Spark \u7684 classpath\u4e0b\u3002 cp jindosdk-x.x.x/lib/*.jar $SPARK_HOME/jars/","title":"2. \u5728 Spark CLASSPATH \u4e2d\u6dfb\u52a0 JindoSDK"},{"location":"jindosdk/spark/jindosdk_on_spark/#3-jindosdk","text":"","title":"3. \u914d\u7f6e JindoSDK"},{"location":"jindosdk/spark/jindosdk_on_spark/#spark-core-sitexml","text":"\u914d\u7f6e JindoSDK OSS-HDFS \u670d\u52a1\u5b9e\u73b0\u7c7b <configuration> <property> <name>fs.AbstractFileSystem.oss.impl</name> <value>com.aliyun.jindodata.oss.OSS</value> </property> <property> <name>fs.oss.impl</name> <value>com.aliyun.jindodata.oss.JindoOssFileSystem</value> </property> </configuration> \u914d\u7f6e OSS-HDFS \u670d\u52a1 Access Key <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 AccessKey \u7684\u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 JindoSDK Credential Provider \u914d\u7f6e \u3002 \u914d\u7f6e OSS-HDFS \u670d\u52a1 Endpoint \u8bbf\u95ee OSS/OSS-HDFS Bucket\u65f6\u9700\u8981\u914d\u7f6e Endpoint\u3002\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> . <Endpoint> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket.cn-hangzhou.oss-dls.aliyuncs.com/exampleobject.txt\u3002\u914d\u7f6e\u5b8c\u6210\u540e\uff0cJindoSDK\u4f1a\u6839\u636e\u8bbf\u95ee\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS/OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 \u6b64\u5916\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u914d\u7f6e\u9ed8\u8ba4 Endpoint\uff0c\u4ee5\u7b80\u5316\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a oss:// <Bucket> / <Object> \uff0c\u4f8b\u5982 oss://examplebucket/exampleobject.txt <configuration> <property> <name>fs.oss.endpoint</name> <value>xxx</value> </property> </configuration> \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u670d\u52a1\u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u670d\u52a1\u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS-HDFS \u670d\u52a1 Endpoint \u914d\u7f6e \u3002","title":"\u5168\u5c40\u914d\u7f6e, \u4fee\u6539 Spark \u4f7f\u7528\u7684core-site.xml"},{"location":"jindosdk/spark/jindosdk_on_spark/#_2","text":"\u4f7f\u7528\u53c2\u6570\u5728 Spark \u4efb\u52a1\u63d0\u4ea4\u7684\u65f6\u5019\u8bbe\u7f6e JindoSDK, \u4ee5\u4e0b\u4e3a\u793a\u4f8b: spark-submit --conf spark.hadoop.fs.AbstractFileSystem.oss.impl=com.aliyun.jindodata.oss.OSS --conf spark.hadoop.fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem --conf spark.hadoop.fs.oss.accessKeyId=xxx --conf spark.hadoop.fs.oss.accessKeySecret=xxx","title":"\u4efb\u52a1\u7ea7\u522b\u914d\u7f6e"},{"location":"jindosdk/spark/jindosdk_on_spark/#4-spark-oss","text":"\u521b\u5efa\u8868 create table test_oss (c1 string) location \"oss://bucket.endpoint/dir\"; \u63d2\u5165\u6570\u636e insert into table test_oss values (\"testdata\"); \u67e5\u8be2 OSS \u8868 select * from test_oss;","title":"4. \u4f7f\u7528 Spark \u8bbf\u95ee OSS"},{"location":"jindosdk/spark/jindosdk_on_spark/#5","text":"JindoSDK\u5305\u542b\u4e00\u4e9b\u9ad8\u7ea7\u8c03\u4f18\u53c2\u6570\uff0c\u914d\u7f6e\u65b9\u5f0f\u4ee5\u53ca\u914d\u7f6e\u9879\u53c2\u8003\u6587\u6863 JindoSDK \u914d\u7f6e\u9879\u5217\u8868 \u3002","title":"5. \u53c2\u6570\u8c03\u4f18"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/","text":"\u4f7f\u7528 Jindo Tensorflow Collector \u8bbf\u95ee OSS/OSS-HDFS \u00a7 Jindo Tensorflow Collector \u652f\u6301\u4f7f\u7528 Tensoflow \u539f\u751f API \u652f\u6301\u8bbf\u95ee OSS/OSS-HDFS\u3002 1. \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.y.z.tar.gz\uff08x.y.z\u8868\u7248\u672c\u53f7\uff09\uff0c ( \u4e0b\u8f7d\u9875\u9762 )\u3002 \u90e8\u7f72\u65b9\u5f0f\uff0c\u53c2\u89c1 \u90e8\u7f72\u6587\u6863 2. \u4fee\u6539\u914d\u7f6e \u00a7 2.1 \u5b89\u88c5 tensorflow \u00a7 \u76ee\u524d\u4ec5\u652f\u6301 tensorflow 1.15 \u6216 tensorflow 2.8 \u7248\u672c\uff0c\u4ee5\u4e0b\u4ee5 tensorflow 2.8 \u4e3a\u4f8b\u3002 pip3 install tensorflow==2.8 2.2 \u6dfb\u52a0\u73af\u5883\u53d8\u91cf \u00a7 2.2.1 JindoSDK \u73af\u5883\u53d8\u91cf \u00a7 \u786e\u8ba4 LD_LIBRARY_PATH \u5305\u542b /path/to/jindosdk-x.x.x/lib/native 2.2.2 \u4f9d\u8d56\u7684 Tensorflow \u73af\u5883\u53d8\u91cf \u00a7 \u540c\u65f6\uff0cLD_LIBRARY_PATH \u9700\u8981\u6dfb\u52a0 tensorflow \u6240\u5728\u7684\u4f9d\u8d56\u8def\u5f84\uff0c\u4e0d\u540c\u7684 python \u73af\u5883\u5b89\u88c5\u8def\u5f84\u53ef\u80fd\u4e0d\u540c\uff0c\u5982\uff1a # tensorflow1.15 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow/\":$LD_LIBRARY_PATH # tensorflow2.8 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow/\":$LD_LIBRARY_PATH 2.3 \u914d\u7f6e\u6587\u4ef6 \u00a7 \u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \u3002\u53ef\u4ee5\u9009\u62e9\u56fa\u5b9a\u6216\u8005\u514d\u5bc6\u4e24\u79cd\u914d\u7f6e\u65b9\u5f0f\u3002 2.3.1 \u56fa\u5b9a AK \u8bbf\u95ee \u00a7 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/tfio-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95eeOSS\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret> \u914d\u7f6e\u9009\u9879 \u00a7 \u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u914d\u7f6e\u8282 \u8bf4\u660e logger.dir /tmp/tfio-log common \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false common \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false common \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\u3002 \u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002 logger.verbose 0 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false common \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u5730\u5740\uff0c\u5982cn-xxx.oss-dls.aliyuncs.com fs.oss.accessKeyId jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeySecret \u66f4\u591a jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002 2.3.2 \u514d\u5bc6\u8bbf\u95ee \u00a7 \u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7\u89d2\u8272\u6388\u6743\u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/tensorflow-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE 3. \u4f7f\u7528 Jindo Tensorflow \u00a7 \u4ee5\u52a0\u8f7d jindo-tensorflow2.8 \u4e3a\u4f8b 3.1 \u901a\u8fc7 load_library \u6dfb\u52a0 collector \u00a7 import tensorflow as tf tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\") 3.2 \u901a\u8fc7 gfile \u8bbf\u95ee oss-hdfs \u00a7 \u4e3a\u907f\u514d\u4e0e tfio \u4e2d\u7684 oss \u524d\u7f00\u51b2\u7a81\uff0cjindo-tensorflow \u4f7f\u7528 joss \u524d\u7f00\u3002 # \u76ee\u5f55\u64cd\u4f5c tf.io.gfile.mkdir(\"joss://<bucket>/test_mkdir\") content = tf.io.gfile.listdir(\"joss://<bucket>/test_mkdir\") # \u8bfb gmnist = tf.io.gfile.GFile(\"joss://<bucket>/mnist.npz\", \"rb\") gmnist_file = gmnist.read() # \u5199 fh = gfile.Open(f, mode=\"w\") content = \"file content\" fh.write(content) fh.close() 3.3 \u8bfb\u53d6 TFRecordDataset \u00a7 tfRecordOssPath = \"joss://<bucket>/test.tfrecord\" filenames = [tfRecordOssPath] raw_dataset = tf.data.TFRecordDataset(filenames) 3.4 \u5b8c\u6574\u793a\u4f8b \u00a7 import tensorflow as tf import numpy as np tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\") gmnist = tf.io.gfile.GFile(\"joss://<bucket>/mnist.npz\", 'rb') print(gmnist.size()) with np.load(gmnist) as f: x_train, y_train = f['x_train'], f['y_train'] x_test, y_test = f['x_test'], f['y_test'] model = tf.keras.models.Sequential([ tf.keras.layers.Flatten(input_shape=(28, 28)), tf.keras.layers.Dense(128, activation='relu'), tf.keras.layers.Dropout(0.2), tf.keras.layers.Dense(10, activation='softmax') ]) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) model.fit(x_train, y_train, epochs=5)","title":"Tensorflow"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#jindo-tensorflow-collector-ossoss-hdfs","text":"Jindo Tensorflow Collector \u652f\u6301\u4f7f\u7528 Tensoflow \u539f\u751f API \u652f\u6301\u8bbf\u95ee OSS/OSS-HDFS\u3002","title":"\u4f7f\u7528 Jindo Tensorflow Collector \u8bbf\u95ee OSS/OSS-HDFS"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#1-jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.y.z.tar.gz\uff08x.y.z\u8868\u7248\u672c\u53f7\uff09\uff0c ( \u4e0b\u8f7d\u9875\u9762 )\u3002 \u90e8\u7f72\u65b9\u5f0f\uff0c\u53c2\u89c1 \u90e8\u7f72\u6587\u6863","title":"1. \u4e0b\u8f7d JindoSDK \u5305"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#2","text":"","title":"2. \u4fee\u6539\u914d\u7f6e"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#21-tensorflow","text":"\u76ee\u524d\u4ec5\u652f\u6301 tensorflow 1.15 \u6216 tensorflow 2.8 \u7248\u672c\uff0c\u4ee5\u4e0b\u4ee5 tensorflow 2.8 \u4e3a\u4f8b\u3002 pip3 install tensorflow==2.8","title":"2.1 \u5b89\u88c5 tensorflow"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#22","text":"","title":"2.2 \u6dfb\u52a0\u73af\u5883\u53d8\u91cf"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#221-jindosdk","text":"\u786e\u8ba4 LD_LIBRARY_PATH \u5305\u542b /path/to/jindosdk-x.x.x/lib/native","title":"2.2.1 JindoSDK \u73af\u5883\u53d8\u91cf"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#222-tensorflow","text":"\u540c\u65f6\uff0cLD_LIBRARY_PATH \u9700\u8981\u6dfb\u52a0 tensorflow \u6240\u5728\u7684\u4f9d\u8d56\u8def\u5f84\uff0c\u4e0d\u540c\u7684 python \u73af\u5883\u5b89\u88c5\u8def\u5f84\u53ef\u80fd\u4e0d\u540c\uff0c\u5982\uff1a # tensorflow1.15 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow/\":$LD_LIBRARY_PATH # tensorflow2.8 export LD_LIBRARY_PATH=\"/usr/local/lib/python3.9/site-packages/tensorflow/\":$LD_LIBRARY_PATH","title":"2.2.2 \u4f9d\u8d56\u7684 Tensorflow \u73af\u5883\u53d8\u91cf"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#23","text":"\u4f7f\u7528 INI \u98ce\u683c\u914d\u7f6e\u6587\u4ef6\uff0c\u914d\u7f6e\u6587\u4ef6\u7684\u6587\u4ef6\u540d\u4e3a jindosdk.cfg \u3002\u53ef\u4ee5\u9009\u62e9\u56fa\u5b9a\u6216\u8005\u514d\u5bc6\u4e24\u79cd\u914d\u7f6e\u65b9\u5f0f\u3002","title":"2.3 \u914d\u7f6e\u6587\u4ef6"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#231-ak","text":"\u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/tfio-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> # \u7528\u4e8e\u8bbf\u95eeOSS\u7684AccessKey ID\u548cAccessKey Secret\u3002\u963f\u91cc\u4e91\u8d26\u53f7AccessKey\u62e5\u6709\u6240\u6709API\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u98ce\u9669\u5f88\u9ad8\u3002\u5f3a\u70c8\u5efa\u8bae\u60a8\u521b\u5efa\u5e76\u4f7f\u7528RAM\u7528\u6237\u8fdb\u884cAPI\u8bbf\u95ee\u6216\u65e5\u5e38\u8fd0\u7ef4\uff0c\u8bf7\u767b\u5f55RAM\u63a7\u5236\u53f0\u521b\u5efaRAM\u7528\u6237\u3002 fs.oss.accessKeyId = <your_key_id> fs.oss.accessKeySecret = <your_key_secret>","title":"2.3.1 \u56fa\u5b9a AK \u8bbf\u95ee"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#_1","text":"\u914d\u7f6e\u9879 \u9ed8\u8ba4\u503c \u914d\u7f6e\u8282 \u8bf4\u660e logger.dir /tmp/tfio-log common \u65e5\u5fd7\u76ee\u5f55\uff0c\u4e0d\u5b58\u5728\u4f1a\u521b\u5efa logger.sync false common \u662f\u5426\u540c\u6b65\u8f93\u51fa\u65e5\u5fd7\uff0cfalse\u8868\u793a\u5f02\u6b65\u8f93\u51fa logger.consolelogger false common \u6253\u5370\u65e5\u5fd7\u5230\u7ec8\u7aef logger.level 2 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684\u65e5\u5fd7\u3002 \u5f00\u542f\u7ec8\u7aef\u65e5\u5fd7\u65f6\uff0c\u65e5\u5fd7\u7b49\u7ea7\u8303\u56f4\u4e3a0-6\uff0c\u5206\u522b\u8868\u793a\uff1aTRACE\u3001DEBUG\u3001INFO\u3001WARN\u3001ERROR\u3001CRITICAL\u3001OFF\u3002 \u5173\u95ed\u7ec8\u7aef\u65e5\u5fd7\uff0c\u4f7f\u7528\u6587\u4ef6\u65e5\u5fd7\u65f6\uff1a\u65e5\u5fd7\u7b49\u7ea7<=1\uff0c\u8868\u793aWARN\uff1b\u65e5\u5fd7\u7b49\u7ea7>1\uff0c\u8868\u793aINFO\u3002 logger.verbose 0 common \u8f93\u51fa\u5927\u4e8e\u7b49\u4e8e\u8be5\u7b49\u7ea7\u7684VERBOSE\u65e5\u5fd7\uff0c\u7b49\u7ea7\u8303\u56f4\u4e3a0-99\uff0c0\u8868\u793a\u4e0d\u8f93\u51fa logger.cleaner.enable false common \u662f\u5426\u5f00\u542f\u65e5\u5fd7\u6e05\u7406 fs.oss.endpoint jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u7684\u5730\u5740\uff0c\u5982cn-xxx.oss-dls.aliyuncs.com fs.oss.accessKeyId jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeyId fs.oss.accessKeySecret jindosdk \u8bbf\u95ee OSS-HDFS \u670d\u52a1\u9700\u8981\u7684 accessKeySecret \u66f4\u591a jindosdk \u914d\u7f6e\u8282\u53c2\u6570\u53ef\u89c1 \u76f8\u5173\u6587\u6863 \u3002","title":"\u914d\u7f6e\u9009\u9879"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#232","text":"\u524d\u63d0\uff1a\u4f7f\u7528\u7684\u662f\u963f\u91cc\u4e91 ECS\uff0c\u5e76\u4e14\u8be5\u673a\u5668\u5df2\u7ed1\u5b9a\u8fc7\u89d2\u8272\u6388\u6743\u3002 \u793a\u4f8b\u5982\u4e0b\uff1a [common] logger.dir = /tmp/tensorflow-log [jindosdk] # \u5df2\u521b\u5efa\u7684Bucket\u5bf9\u5e94\u7684Endpoint\u3002\u4ee5\u534e\u4e1c1\uff08\u676d\u5dde\uff09\u4e3a\u4f8b\uff0c\u586b\u5199\u4e3aoss-cn-hangzhou.aliyuncs.com\u3002 fs.oss.endpoint = <your_endpoint> fs.oss.provider.endpoint = ECS_ROLE","title":"2.3.2 \u514d\u5bc6\u8bbf\u95ee"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#3-jindo-tensorflow","text":"\u4ee5\u52a0\u8f7d jindo-tensorflow2.8 \u4e3a\u4f8b","title":"3. \u4f7f\u7528 Jindo Tensorflow"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#31-load_library-collector","text":"import tensorflow as tf tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\")","title":"3.1 \u901a\u8fc7 load_library \u6dfb\u52a0 collector"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#32-gfile-oss-hdfs","text":"\u4e3a\u907f\u514d\u4e0e tfio \u4e2d\u7684 oss \u524d\u7f00\u51b2\u7a81\uff0cjindo-tensorflow \u4f7f\u7528 joss \u524d\u7f00\u3002 # \u76ee\u5f55\u64cd\u4f5c tf.io.gfile.mkdir(\"joss://<bucket>/test_mkdir\") content = tf.io.gfile.listdir(\"joss://<bucket>/test_mkdir\") # \u8bfb gmnist = tf.io.gfile.GFile(\"joss://<bucket>/mnist.npz\", \"rb\") gmnist_file = gmnist.read() # \u5199 fh = gfile.Open(f, mode=\"w\") content = \"file content\" fh.write(content) fh.close()","title":"3.2 \u901a\u8fc7 gfile \u8bbf\u95ee oss-hdfs"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#33-tfrecorddataset","text":"tfRecordOssPath = \"joss://<bucket>/test.tfrecord\" filenames = [tfRecordOssPath] raw_dataset = tf.data.TFRecordDataset(filenames)","title":"3.3 \u8bfb\u53d6 TFRecordDataset"},{"location":"jindosdk/tensorflow/jindosdk_on_tensorflow/#34","text":"import tensorflow as tf import numpy as np tf.load_library(\"/path/to/jindosdk-x.x.x/lib/native/libjindo-tensorflow2.8.so\") gmnist = tf.io.gfile.GFile(\"joss://<bucket>/mnist.npz\", 'rb') print(gmnist.size()) with np.load(gmnist) as f: x_train, y_train = f['x_train'], f['y_train'] x_test, y_test = f['x_test'], f['y_test'] model = tf.keras.models.Sequential([ tf.keras.layers.Flatten(input_shape=(28, 28)), tf.keras.layers.Dense(128, activation='relu'), tf.keras.layers.Dropout(0.2), tf.keras.layers.Dense(10, activation='softmax') ]) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) model.fit(x_train, y_train, epochs=5)","title":"3.4 \u5b8c\u6574\u793a\u4f8b"},{"location":"jindotools/jindodistcp_api_quickstart/","text":"JindoDistCp API \u4f7f\u7528\u8bf4\u660e \u00a7 \u4e0b\u8f7d JindoSDK \u5305 \u00a7 \u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002 \u5b89\u88c5\u4f9d\u8d56\u5305 \u00a7 \u76ee\u524d JindoSDK/JindoDistCP \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff0cMac M1 \u7cfb\u5217\u4e5f\u6682\u4e0d\u652f\u6301\uff09 \u4ee56.7.3\u7248\u672c\u4e3a\u4f8b\uff0c\u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoDistCP \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-distcp --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop with provided scope, your runtime should install hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> </dependencies> </project> \u793a\u4f8b\u4ee3\u7801 \u00a7 \u4ee5\u62f7\u8d1d HDFS \u6587\u4ef6\u5939\u5230\u963f\u91cc\u4e91 OSS \u4e3a\u4f8b\u3002 import com.aliyun.jindodata.distcp.util.DistCpCounter; import com.aliyun.jindodata.distjob.request.DistJobResponse; import org.apache.hadoop.conf.Configuration; import java.io.IOException; public class DistcpExample { static public void main(String[] args) throws IOException { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. conf.set(\"fs.oss.credentials.provider\", \"com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider\"); conf.set(\"fs.oss.accessKeyId\", \"XXX\"); conf.set(\"fs.oss.accessKeySecret\", \"XXX\"); DistCpRequest request = new DistCpRequest(); request.setSrcDir(\"hdfs:///user/root/random-data\"); request.setDestDir(\"oss://dls-bucket.cn-xxx.oss-dls.aliyuncs.com/dest\"); request.setWorkTempDir(\"hdfs:///user/root/tmp\"); request.setConf(conf); // Jindo DistCp Service DistCpService service = new DistCpService(); service.buildAllJobs(request); service.runAllJobs(); DistJobResponse response = service.collectAllJobResult(); printResponse(response); } static public void printResponse(DistJobResponse response) { System.out.println(\"ErrorCode: \" + response.getErrorCode()); System.out.println(\"ErrorMsg: \" + response.getErrorMsg()); System.out.println(\"FileCopied: \" + response.getCount(DistCpCounter.FILES_COPIED.getName())); System.out.println(\"BytesCopied: \" + response.getCount(DistCpCounter.BYTES_COPIED.getName())); } } \u8bf7\u6c42\u53c2\u6570 \u00a7 \u76f8\u5173\u65b9\u6cd5\u7684\u5177\u4f53\u63cf\u8ff0\u53ef\u53c2\u8003 \u300a\u4f7f\u7528Jindo DistCp\u300b \u65b9\u6cd5 \u53c2\u6570\u7c7b\u578b \u8bf4\u660e \u9ed8\u8ba4\u503c \u517c\u5bb9 void setSrc(String src) \u5fc5\u9009 \u8bbe\u7f6e\u6e90\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setDest(String dest) \u5fc5\u9009 \u8bbe\u7f6e\u76ee\u6807\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setConf(Configuration conf) \u5fc5\u9009 \u8bbe\u7f6e Configuration - 4.3.0+ void setMode(DistCpMode mode) \u53ef\u9009 \u8bbe\u7f6e DistCp \u6a21\u5f0f\uff0c\u652f\u6301\u7684\u6a21\u5f0f\u6709 COPY\uff08\u62f7\u8d1d\uff09 DIFF\uff08\u5dee\u5f02\u6bd4\u8f83\uff09 UPADTE\uff08\u589e\u91cf\u62f7\u8d1d\uff09 DistCpMode.COPY 4.3.0+ void setBandWidthLimit(int bandWidthLimit) \u53ef\u9009 \u8bbe\u7f6e\u5355\u4e2a\u8282\u70b9\u7684\u5e26\u5bbd\u9650\u5236\uff0c\u5355\u4f4dM -1 4.3.0+ void setOutputCodec(String outputCodec) \u53ef\u9009 \u8bbe\u7f6e\u538b\u7f29\u7c7b\u578b\uff0c\u652f\u6301\u7f16\u89e3\u7801\u5668\u6709 gzip\u3001gz\u3001lzo\u3001lzop\u3001lzop\u3001snappy keep\uff08\u4e0d\u66f4\u6539\u538b\u7f29\u7c7b\u578b\uff09 4.3.0+ void setStorageClass(String storageClass) \u53ef\u9009 \u8bbe\u7f6e\u76ee\u6807\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301Standard\u3001IA\u3001Archive\u3001ColdArchive Standard 4.3.0+ void setIncludeMatcherFile(String includeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setExcludeMatcherFile(String excludeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setNumWorkers(int numWorkers) \u53ef\u9009 \u8bbe\u7f6eDistCp\u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\uff0c\u5bf9\u5e94MR\u4efb\u52a1\u4e2d\u7684 mapreduce.job.maps 10 4.3.0+ void setBatchPerJob(int batchPerJob) \u53ef\u9009 \u8bbe\u7f6e\u6bcf\u4e2a distcp job \u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf 10 4.3.0+ void setBatchPerTask(int batchPerTask) \u53ef\u9009 \u8bbe\u7f6e\u6bcf\u4e2a distcp task \u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf 10 4.3.0+ void setWorkTempDir(String workTempDir) \u53ef\u9009 \u8bbe\u7f6e\u4e34\u65f6\u76ee\u5f55 /tmp 4.3.0+ void setDisableChecksum(boolean disableChecksum) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5173\u95ed checksum \u68c0\u67e5 false 4.3.0+ void setDeleteOnSuccess(boolean deleteOnSuccess) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5220\u9664\u6e90\u6587\u4ef6\uff0c\u7528\u4e8e\u79fb\u52a8\u6570\u636e false 4.3.0+ void setEnableTransaction(boolean enableTransaction) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4e8b\u52a1\uff0c\u4ee5\u4fdd\u8bc1Job\u7ea7\u522b\u7684\u539f\u5b50\u6027 false 4.3.0+ void setIgnoreFailures(boolean ignoreFailures) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5ffd\u7565\u62f7\u8d1d\u4efb\u52a1\u4e2d\u629b\u51fa\u7684\u5f02\u5e38\uff0c\u907f\u514d\u4e2d\u65ad\u4efb\u52a1 False 4.3.0+ void setEnableCMS(boolean enableCMS) \u53ef\u9009 \u662f\u5426\u5f00\u542f\u76d1\u63a7\u544a\u8b66 false 4.3.0+ void setEnablePreserveMetaStatus(boolean enablePreserveMetaStatus) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f false 4.4.0+ \u54cd\u5e94\u53c2\u6570 \u00a7 \u65b9\u6cd5 \u8bf4\u660e \u9ed8\u8ba4\u503c int getErrorCode() \u83b7\u53d6\u9519\u8bef\u7801 0 String getErrorMsg() \u83b7\u53d6\u9519\u8bef\u539f\u56e0 - long getStartTime() \u83b7\u53d6\u4efb\u52a1\u7684\u5f00\u59cb\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - long getEndTime() \u83b7\u53d6\u4efb\u52a1\u7684\u7ed3\u675f\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - getCount(String key) \u83b7\u53d6\u8ba1\u6570\u5668\u503c\uff0ckey\u4e3a\u8ba1\u6570\u5668\u540d\u79f0\uff0c - \u8ba1\u6570\u5668\u8bf4\u660e \u00a7 \u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf BYTES_EXPECTED \u9884\u671f\u7684copy\u5b57\u8282\u6570 FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 BYTES_COPIED copy\u6210\u529f\u7684\u5b57\u8282\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"\u4f7f\u7528 JindoDistcp API \u5f00\u53d1"},{"location":"jindotools/jindodistcp_api_quickstart/#jindodistcp-api","text":"","title":"JindoDistCp API \u4f7f\u7528\u8bf4\u660e"},{"location":"jindotools/jindodistcp_api_quickstart/#jindosdk","text":"\u4e0b\u8f7d\u6700\u65b0\u7684 tar.gz \u5305 jindosdk-x.x.x.tar.gz ( \u4e0b\u8f7d\u9875\u9762 )\u3002","title":"\u4e0b\u8f7d JindoSDK \u5305"},{"location":"jindotools/jindodistcp_api_quickstart/#_1","text":"\u76ee\u524d JindoSDK/JindoDistCP \u652f\u6301\u4e3b\u6d41 Intel X86 \u7684 Linux \u548c Mac\uff08\u4e0d\u652f\u6301 Windows\u7cfb\u7edf\uff0cMac M1 \u7cfb\u5217\u4e5f\u6682\u4e0d\u652f\u6301\uff09 \u4ee56.7.3\u7248\u672c\u4e3a\u4f8b\uff0c\u5728 maven pom.xml \u4e2d\u6dfb\u52a0 JindoDistCP \u7684\u4f9d\u8d56 <project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\"> <modelVersion>4.0.0</modelVersion> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp-example</artifactId> <version>1.0</version> <properties> <jindodata.version>6.7.3</jindodata.version> <hadoop.version>2.8.5</hadoop.version> </properties> <repositories> <!-- Add JindoData Maven Repository --> <repository> <id>jindodata</id> <url>https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url> </repository> </repositories> <dependencies> <!-- add jindo-distcp --> <dependency> <groupId>com.aliyun.jindodata</groupId> <artifactId>jindo-distcp</artifactId> <version>${jindodata.version}</version> </dependency> <!-- add hadoop with provided scope, your runtime should install hadoop dependency. --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>${hadoop.version}</version> <scope>provided</scope> </dependency> </dependencies> </project>","title":"\u5b89\u88c5\u4f9d\u8d56\u5305"},{"location":"jindotools/jindodistcp_api_quickstart/#_2","text":"\u4ee5\u62f7\u8d1d HDFS \u6587\u4ef6\u5939\u5230\u963f\u91cc\u4e91 OSS \u4e3a\u4f8b\u3002 import com.aliyun.jindodata.distcp.util.DistCpCounter; import com.aliyun.jindodata.distjob.request.DistJobResponse; import org.apache.hadoop.conf.Configuration; import java.io.IOException; public class DistcpExample { static public void main(String[] args) throws IOException { Configuration conf = new Configuration(); conf.set(\"fs.oss.impl\", \"com.aliyun.jindodata.oss.JindoOssFileSystem\"); conf.set(\"fs.AbstractFileSystem.oss.impl\", \"com.aliyun.jindodata.oss.OSS\"); // set accessKey, secret, endpoint and so on. conf.set(\"fs.oss.credentials.provider\", \"com.aliyun.jindodata.oss.auth.SimpleCredentialsProvider\"); conf.set(\"fs.oss.accessKeyId\", \"XXX\"); conf.set(\"fs.oss.accessKeySecret\", \"XXX\"); DistCpRequest request = new DistCpRequest(); request.setSrcDir(\"hdfs:///user/root/random-data\"); request.setDestDir(\"oss://dls-bucket.cn-xxx.oss-dls.aliyuncs.com/dest\"); request.setWorkTempDir(\"hdfs:///user/root/tmp\"); request.setConf(conf); // Jindo DistCp Service DistCpService service = new DistCpService(); service.buildAllJobs(request); service.runAllJobs(); DistJobResponse response = service.collectAllJobResult(); printResponse(response); } static public void printResponse(DistJobResponse response) { System.out.println(\"ErrorCode: \" + response.getErrorCode()); System.out.println(\"ErrorMsg: \" + response.getErrorMsg()); System.out.println(\"FileCopied: \" + response.getCount(DistCpCounter.FILES_COPIED.getName())); System.out.println(\"BytesCopied: \" + response.getCount(DistCpCounter.BYTES_COPIED.getName())); } }","title":"\u793a\u4f8b\u4ee3\u7801"},{"location":"jindotools/jindodistcp_api_quickstart/#_3","text":"\u76f8\u5173\u65b9\u6cd5\u7684\u5177\u4f53\u63cf\u8ff0\u53ef\u53c2\u8003 \u300a\u4f7f\u7528Jindo DistCp\u300b \u65b9\u6cd5 \u53c2\u6570\u7c7b\u578b \u8bf4\u660e \u9ed8\u8ba4\u503c \u517c\u5bb9 void setSrc(String src) \u5fc5\u9009 \u8bbe\u7f6e\u6e90\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setDest(String dest) \u5fc5\u9009 \u8bbe\u7f6e\u76ee\u6807\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ void setConf(Configuration conf) \u5fc5\u9009 \u8bbe\u7f6e Configuration - 4.3.0+ void setMode(DistCpMode mode) \u53ef\u9009 \u8bbe\u7f6e DistCp \u6a21\u5f0f\uff0c\u652f\u6301\u7684\u6a21\u5f0f\u6709 COPY\uff08\u62f7\u8d1d\uff09 DIFF\uff08\u5dee\u5f02\u6bd4\u8f83\uff09 UPADTE\uff08\u589e\u91cf\u62f7\u8d1d\uff09 DistCpMode.COPY 4.3.0+ void setBandWidthLimit(int bandWidthLimit) \u53ef\u9009 \u8bbe\u7f6e\u5355\u4e2a\u8282\u70b9\u7684\u5e26\u5bbd\u9650\u5236\uff0c\u5355\u4f4dM -1 4.3.0+ void setOutputCodec(String outputCodec) \u53ef\u9009 \u8bbe\u7f6e\u538b\u7f29\u7c7b\u578b\uff0c\u652f\u6301\u7f16\u89e3\u7801\u5668\u6709 gzip\u3001gz\u3001lzo\u3001lzop\u3001lzop\u3001snappy keep\uff08\u4e0d\u66f4\u6539\u538b\u7f29\u7c7b\u578b\uff09 4.3.0+ void setStorageClass(String storageClass) \u53ef\u9009 \u8bbe\u7f6e\u76ee\u6807\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301Standard\u3001IA\u3001Archive\u3001ColdArchive Standard 4.3.0+ void setIncludeMatcherFile(String includeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setExcludeMatcherFile(String excludeMatcherFile) \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ void setNumWorkers(int numWorkers) \u53ef\u9009 \u8bbe\u7f6eDistCp\u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\uff0c\u5bf9\u5e94MR\u4efb\u52a1\u4e2d\u7684 mapreduce.job.maps 10 4.3.0+ void setBatchPerJob(int batchPerJob) \u53ef\u9009 \u8bbe\u7f6e\u6bcf\u4e2a distcp job \u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf 10 4.3.0+ void setBatchPerTask(int batchPerTask) \u53ef\u9009 \u8bbe\u7f6e\u6bcf\u4e2a distcp task \u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf 10 4.3.0+ void setWorkTempDir(String workTempDir) \u53ef\u9009 \u8bbe\u7f6e\u4e34\u65f6\u76ee\u5f55 /tmp 4.3.0+ void setDisableChecksum(boolean disableChecksum) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5173\u95ed checksum \u68c0\u67e5 false 4.3.0+ void setDeleteOnSuccess(boolean deleteOnSuccess) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5220\u9664\u6e90\u6587\u4ef6\uff0c\u7528\u4e8e\u79fb\u52a8\u6570\u636e false 4.3.0+ void setEnableTransaction(boolean enableTransaction) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4e8b\u52a1\uff0c\u4ee5\u4fdd\u8bc1Job\u7ea7\u522b\u7684\u539f\u5b50\u6027 false 4.3.0+ void setIgnoreFailures(boolean ignoreFailures) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5ffd\u7565\u62f7\u8d1d\u4efb\u52a1\u4e2d\u629b\u51fa\u7684\u5f02\u5e38\uff0c\u907f\u514d\u4e2d\u65ad\u4efb\u52a1 False 4.3.0+ void setEnableCMS(boolean enableCMS) \u53ef\u9009 \u662f\u5426\u5f00\u542f\u76d1\u63a7\u544a\u8b66 false 4.3.0+ void setEnablePreserveMetaStatus(boolean enablePreserveMetaStatus) \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f false 4.4.0+","title":"\u8bf7\u6c42\u53c2\u6570"},{"location":"jindotools/jindodistcp_api_quickstart/#_4","text":"\u65b9\u6cd5 \u8bf4\u660e \u9ed8\u8ba4\u503c int getErrorCode() \u83b7\u53d6\u9519\u8bef\u7801 0 String getErrorMsg() \u83b7\u53d6\u9519\u8bef\u539f\u56e0 - long getStartTime() \u83b7\u53d6\u4efb\u52a1\u7684\u5f00\u59cb\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - long getEndTime() \u83b7\u53d6\u4efb\u52a1\u7684\u7ed3\u675f\u65f6\u95f4\uff0c\u5355\u4f4d\u79d2 - getCount(String key) \u83b7\u53d6\u8ba1\u6570\u5668\u503c\uff0ckey\u4e3a\u8ba1\u6570\u5668\u540d\u79f0\uff0c -","title":"\u54cd\u5e94\u53c2\u6570"},{"location":"jindotools/jindodistcp_api_quickstart/#_5","text":"\u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf BYTES_EXPECTED \u9884\u671f\u7684copy\u5b57\u8282\u6570 FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 BYTES_COPIED copy\u6210\u529f\u7684\u5b57\u8282\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"\u8ba1\u6570\u5668\u8bf4\u660e"},{"location":"jindotools/jindodistcp_faq/","text":"JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u00a7 1\u3001list \u6027\u80fd\u4f4e \u00a7 \u73b0\u8c61 \u00a7 \u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u53d1\u73b0 list \u6027\u80fd\u8f83\u6162\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Successfully list objects with prefix xxx/yyy/ in bucket xxx recursive 0 result 315 dur 100036.615031MS \u5176\u4e2d dur 100036.615031MS \u4ee3\u8868\u6b64\u6b21 list \u7528\u65f6\uff0c\u5355\u4f4d\u6beb\u79d2\uff0c\u5982\u4e0a\u8ff0\u4e00\u6b21 list \u8017\u65f6 100 \u79d2\uff0cOSS \u4e0a\u6587\u4ef6\u6b63\u5e38\u7684 list \u901f\u5ea6\u662f 1000 \u6587\u4ef6\u5728 1s \u4ee5\u4e0b\uff0c\u60a8\u53ef\u6839\u636e\u5f53\u524d\u76ee\u5f55\u4e0b\u6587\u4ef6\u6570\u91cf\u6765\u5224\u65ad\u8be5 list \u8017\u65f6\u662f\u5426\u5f02\u5e38\uff0c\u5982\u4e0a\u8ff0\u4fe1\u606f\u663e\u793a list 315 \u4e2a\u6587\u4ef6\u7684\u76ee\u5f55\u9700\u8981 100s \u663e\u7136\u662f\u4e0d\u592a\u6b63\u5e38\u7684\u3002 \u89e3\u51b3\u529e\u6cd5 \u00a7 \u589e\u52a0\u5ba2\u6237\u7aef\u5185\u5b58\uff0c\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u589e\u5927\u5ba2\u6237\u7aef\u5185\u5b58 export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx4096m\" 2. checksum \u62a5\u9519 \u00a7 \u73b0\u8c61 \u00a7 \u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Failed to get checksum store. \u89e3\u51b3\u529e\u6cd5 \u00a7 OSS-HDFS \u9ed8\u8ba4\u7684 checksum \u7b97\u6cd5\u662f COMPOSITE_CRC\uff0c\u5982\u679c HDFS \u914d\u7f6e\u7684 dfs.checksum.combine.mode \u4e3a MD5MD5CRC\uff0c\u5219\u9700\u8981\u53d8\u66f4 OSS-HDFS \u7684 fs.oss.checksum.combine.mode \u914d\u7f6e\u4e3a MD5MD5CRC\u3002 hadoop jar jindo-distcp-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.checksum.combine.mode=MD5MD5CRC 3. OSS \u62f7\u8d1d\u5230 OSS-HDFS \u65f6\u62a5 verify checksum failed \u00a7 \u73b0\u8c61 \u00a7 \u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Exception raised while copying data file, verify checksum failed \u89e3\u51b3\u529e\u6cd5 \u00a7 \u5982\u679c OSS \u4e2d\u7684\u7684\u6587\u4ef6\u4e0d\u662f\u901a\u8fc7 JindoDistCp \u4ece HDFS \u8fc1\u79fb\u5230 OSS\u7684\uff0c\u5219\u9700\u8981\u901a\u8fc7 --disableChecksum \u8df3\u8fc7 checksum \u68c0\u67e5\u3002 hadoop jar jindo-distcp-${version}.jar --src oss://ossBucket/ --dest oss://dlsBucket/ --disableChecksum 4. \u6267\u884c JindoDistCp \u540e\uff0c\u67e5\u770b\u662f\u5426\u6210\u529f \u00a7 \u5982\u679c\u6ca1\u6709\u6dfb\u52a0 --ignore \u53c2\u6570\uff0c\u6267\u884c\u8fc7\u7a0b\u4e2d\u5982\u679c\u6709\u95ee\u9898\u4f1a\u62a5\u9519\u9000\u51fa\u3002\u5982\u679c\u6ca1\u6dfb\u52a0\u4e86--ignore \u53c2\u6570\uff0c\u67e5\u770b JindoDistCp Counters\uff0c\u5982 COPY_FAILED\uff0cCHECKSUM_DIFF \u7b49\uff0c\u8be6\u89c1 JindoDistCp Counters \u8bf4\u660e\u3002","title":"\u5e38\u89c1\u95ee\u9898"},{"location":"jindotools/jindodistcp_faq/#jindodistcp","text":"","title":"JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357"},{"location":"jindotools/jindodistcp_faq/#1list","text":"","title":"1\u3001list \u6027\u80fd\u4f4e"},{"location":"jindotools/jindodistcp_faq/#_1","text":"\u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u53d1\u73b0 list \u6027\u80fd\u8f83\u6162\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Successfully list objects with prefix xxx/yyy/ in bucket xxx recursive 0 result 315 dur 100036.615031MS \u5176\u4e2d dur 100036.615031MS \u4ee3\u8868\u6b64\u6b21 list \u7528\u65f6\uff0c\u5355\u4f4d\u6beb\u79d2\uff0c\u5982\u4e0a\u8ff0\u4e00\u6b21 list \u8017\u65f6 100 \u79d2\uff0cOSS \u4e0a\u6587\u4ef6\u6b63\u5e38\u7684 list \u901f\u5ea6\u662f 1000 \u6587\u4ef6\u5728 1s \u4ee5\u4e0b\uff0c\u60a8\u53ef\u6839\u636e\u5f53\u524d\u76ee\u5f55\u4e0b\u6587\u4ef6\u6570\u91cf\u6765\u5224\u65ad\u8be5 list \u8017\u65f6\u662f\u5426\u5f02\u5e38\uff0c\u5982\u4e0a\u8ff0\u4fe1\u606f\u663e\u793a list 315 \u4e2a\u6587\u4ef6\u7684\u76ee\u5f55\u9700\u8981 100s \u663e\u7136\u662f\u4e0d\u592a\u6b63\u5e38\u7684\u3002","title":"\u73b0\u8c61"},{"location":"jindotools/jindodistcp_faq/#_2","text":"\u589e\u52a0\u5ba2\u6237\u7aef\u5185\u5b58\uff0c\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u589e\u5927\u5ba2\u6237\u7aef\u5185\u5b58 export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx4096m\"","title":"\u89e3\u51b3\u529e\u6cd5"},{"location":"jindotools/jindodistcp_faq/#2-checksum","text":"","title":"2. checksum \u62a5\u9519"},{"location":"jindotools/jindodistcp_faq/#_3","text":"\u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Failed to get checksum store.","title":"\u73b0\u8c61"},{"location":"jindotools/jindodistcp_faq/#_4","text":"OSS-HDFS \u9ed8\u8ba4\u7684 checksum \u7b97\u6cd5\u662f COMPOSITE_CRC\uff0c\u5982\u679c HDFS \u914d\u7f6e\u7684 dfs.checksum.combine.mode \u4e3a MD5MD5CRC\uff0c\u5219\u9700\u8981\u53d8\u66f4 OSS-HDFS \u7684 fs.oss.checksum.combine.mode \u914d\u7f6e\u4e3a MD5MD5CRC\u3002 hadoop jar jindo-distcp-${version}.jar --src /data --dest oss://destBucket/ --hadoopConf fs.oss.checksum.combine.mode=MD5MD5CRC","title":"\u89e3\u51b3\u529e\u6cd5"},{"location":"jindotools/jindodistcp_faq/#3-oss-oss-hdfs-verify-checksum-failed","text":"","title":"3. OSS \u62f7\u8d1d\u5230 OSS-HDFS \u65f6\u62a5 verify checksum failed"},{"location":"jindotools/jindodistcp_faq/#_5","text":"\u5982\u60a8\u5728\u4f7f\u7528 JindoDistCp \u7684\u8fc7\u7a0b\u4e2d\uff0c\u5982\u9047\u5230\u5982\u4e0b\u4fe1\u606f Exception raised while copying data file, verify checksum failed","title":"\u73b0\u8c61"},{"location":"jindotools/jindodistcp_faq/#_6","text":"\u5982\u679c OSS \u4e2d\u7684\u7684\u6587\u4ef6\u4e0d\u662f\u901a\u8fc7 JindoDistCp \u4ece HDFS \u8fc1\u79fb\u5230 OSS\u7684\uff0c\u5219\u9700\u8981\u901a\u8fc7 --disableChecksum \u8df3\u8fc7 checksum \u68c0\u67e5\u3002 hadoop jar jindo-distcp-${version}.jar --src oss://ossBucket/ --dest oss://dlsBucket/ --disableChecksum","title":"\u89e3\u51b3\u529e\u6cd5"},{"location":"jindotools/jindodistcp_faq/#4-jindodistcp","text":"\u5982\u679c\u6ca1\u6709\u6dfb\u52a0 --ignore \u53c2\u6570\uff0c\u6267\u884c\u8fc7\u7a0b\u4e2d\u5982\u679c\u6709\u95ee\u9898\u4f1a\u62a5\u9519\u9000\u51fa\u3002\u5982\u679c\u6ca1\u6dfb\u52a0\u4e86--ignore \u53c2\u6570\uff0c\u67e5\u770b JindoDistCp Counters\uff0c\u5982 COPY_FAILED\uff0cCHECKSUM_DIFF \u7b49\uff0c\u8be6\u89c1 JindoDistCp Counters \u8bf4\u660e\u3002","title":"4. \u6267\u884c JindoDistCp \u540e\uff0c\u67e5\u770b\u662f\u5426\u6210\u529f"},{"location":"jindotools/jindodistcp_hdfs_to_oss/","text":"\u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS/OSS-HDFS \u00a7 \u4f7f\u7528\u524d\u987b\u77e5 \u00a7 \u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d\u3002 \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\u3002 1\u3001\u62f7\u8d1d\u6570\u636e\u5230\u963f\u91cc\u4e91 OSS/OSS-HDFS \u4e0a \u00a7 \u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06 HDFS \u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS/OSS-HDFS \u4e0a hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src HDFS \u7684\u6e90\u8def\u5f84\u3002 /data --dest OSS/OSS-HDFS \u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/ --hadoopConf \u6307\u5b9a OSS/OSS-HDFS \u7684 Access Key ID , Access Key Secret * \u914d\u7f6e OSS/OSS-HDFS \u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS/OSS-HDFS \u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10 2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6 \u00a7 \u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53eaCopy\u5269\u4e0b\u672aCopy\u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a \u4f7f\u7528 --update \u547d\u4ee4\uff0c\u83b7\u5f97\u589e\u91cf\u7684\u6587\u4ef6\u5217\u8868 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --update --parallelism 20 3\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9 \u00a7 \u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB 4\u3001\u914d\u7f6e OSS/OSS-HDFS \u7684 AK \u00a7 \u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS/OSS-HDFS AccessKey/AccessSecret \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662f JindoDistCp \u53ef\u4ee5\u5c06\u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> 5\u3001\u914d\u7f6e OSS-HDFS Endpoint \u00a7 \u8bbf\u95ee OSS-HDFS \u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS/OSS-HDFS Endpoint \u914d\u7f6e \u3002 \u6ce8\u610f \uff1a JindoSDK 4.4.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u9ed8\u8ba4\u4f7f\u7528\u57df\u540d\u5206\u79bb\u7684\u65b9\u5f0f\uff0c\u6570\u636e\u8bfb\u5199\u57df\u540d\u9ed8\u8ba4\u4f7f\u7528\u5185\u7f51\u6807\u51c6 OSS \u57df\u540d\uff0c\u5982\u679c\u5728\u975e\u963f\u91cc\u4e91\u5185\u7f51\u73af\u5883\u6267\u884c distcp \u547d\u4ee4\uff0c\u5219\u9700\u8981\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u914d\u7f6e\u6570\u636e\u6d41\u8bbf\u95ee\u7684\u516c\u7f51 OSS endpoint\u3002 <configuration> <property> <name>fs.oss.data.endpoint</name> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> 6\u3001\u5176\u4ed6\u529f\u80fd \u00a7 \u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003: * JindoDistCp \u4f7f\u7528\u8bf4\u660e","title":"\u4ece HDFS \u8fc1\u79fb\u5230 OSS \u4e0a"},{"location":"jindotools/jindodistcp_hdfs_to_oss/#hdfs-ossoss-hdfs","text":"","title":"\u6570\u636e\u4ece HDFS \u8fc1\u79fb\u5230 OSS/OSS-HDFS"},{"location":"jindotools/jindodistcp_hdfs_to_oss/#_1","text":"\u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d\u3002 \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\u3002","title":"\u4f7f\u7528\u524d\u987b\u77e5"},{"location":"jindotools/jindodistcp_hdfs_to_oss/#1-ossoss-hdfs","text":"\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06 HDFS \u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS/OSS-HDFS \u4e0a hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src HDFS \u7684\u6e90\u8def\u5f84\u3002 /data --dest OSS/OSS-HDFS \u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/ --hadoopConf \u6307\u5b9a OSS/OSS-HDFS \u7684 Access Key ID , Access Key Secret * \u914d\u7f6e OSS/OSS-HDFS \u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS/OSS-HDFS \u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10","title":"1\u3001\u62f7\u8d1d\u6570\u636e\u5230\u963f\u91cc\u4e91 OSS/OSS-HDFS \u4e0a"},{"location":"jindotools/jindodistcp_hdfs_to_oss/#2","text":"\u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53eaCopy\u5269\u4e0b\u672aCopy\u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a","title":"2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6"},{"location":"jindotools/jindodistcp_hdfs_to_oss/#-update","text":"hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --update --parallelism 20","title":"\u4f7f\u7528 --update \u547d\u4ee4\uff0c\u83b7\u5f97\u589e\u91cf\u7684\u6587\u4ef6\u5217\u8868"},{"location":"jindotools/jindodistcp_hdfs_to_oss/#3yarn","text":"\u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src /data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB","title":"3\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9"},{"location":"jindotools/jindodistcp_hdfs_to_oss/#4-ossoss-hdfs-ak","text":"\u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS/OSS-HDFS AccessKey/AccessSecret \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662f JindoDistCp \u53ef\u4ee5\u5c06\u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration>","title":"4\u3001\u914d\u7f6e OSS/OSS-HDFS \u7684 AK"},{"location":"jindotools/jindodistcp_hdfs_to_oss/#5-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS-HDFS \u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS/OSS-HDFS Endpoint \u914d\u7f6e \u3002 \u6ce8\u610f \uff1a JindoSDK 4.4.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u9ed8\u8ba4\u4f7f\u7528\u57df\u540d\u5206\u79bb\u7684\u65b9\u5f0f\uff0c\u6570\u636e\u8bfb\u5199\u57df\u540d\u9ed8\u8ba4\u4f7f\u7528\u5185\u7f51\u6807\u51c6 OSS \u57df\u540d\uff0c\u5982\u679c\u5728\u975e\u963f\u91cc\u4e91\u5185\u7f51\u73af\u5883\u6267\u884c distcp \u547d\u4ee4\uff0c\u5219\u9700\u8981\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u914d\u7f6e\u6570\u636e\u6d41\u8bbf\u95ee\u7684\u516c\u7f51 OSS endpoint\u3002 <configuration> <property> <name>fs.oss.data.endpoint</name> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration>","title":"5\u3001\u914d\u7f6e OSS-HDFS Endpoint"},{"location":"jindotools/jindodistcp_hdfs_to_oss/#6","text":"\u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003: * JindoDistCp \u4f7f\u7528\u8bf4\u660e","title":"6\u3001\u5176\u4ed6\u529f\u80fd"},{"location":"jindotools/jindodistcp_how_to_cms/","text":"JindoDistCp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66 \u00a7 JindoDistCp\uff08\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u5de5\u5177\uff09\u662f\u7528\u4e8e\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u62f7\u8d1d\u6587\u4ef6\u7684\u5de5\u5177\u3002\u4f7f\u7528MapReduce\u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3amap/reduce\u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002 CMS\uff08\u4e91\u76d1\u63a7\u670d\u52a1\uff09\uff0c\u53ef\u7528\u4e8e\u6536\u96c6\u963f\u91cc\u4e91\u8d44\u6e90\u7684\u76d1\u63a7\u6307\u6807\u6216\u7528\u6237\u81ea\u5b9a\u4e49\u7684\u76d1\u63a7\u6307\u6807\uff0c\u63a2\u6d4b\u670d\u52a1\u53ef\u7528\u6027\uff0c\u4ee5\u53ca\u9488\u5bf9\u6307\u6807\u8bbe\u7f6e\u8b66\u62a5\u3002\u4f7f\u60a8\u5168\u9762\u4e86\u89e3\u963f\u91cc\u4e91\u4e0a\u8d44\u6e90\u7684\u4f7f\u7528\u60c5\u51b5\u548c\u4e1a\u52a1\u8fd0\u884c\u72b6\u51b5\uff0c\u5e76\u53ca\u65f6\u5bf9\u6545\u969c\u8d44\u6e90\u8fdb\u884c\u5904\u7406\uff0c\u4fdd\u8bc1\u4e1a\u52a1\u6b63\u5e38\u8fd0\u884c\u3002 \u60a8\u53ef\u4ee5\u6307\u5b9a\u672c\u6b21JindoDistCp\u4efb\u52a1\u7ed3\u675f\u540e\u662f\u5426\u9700\u8981\u5f00\u542fCMS\uff0c\u4ee5\u4e0a\u62a5\u4efb\u52a1\u5931\u8d25\u4fe1\u606f\uff0c\u5e76\u4f7f\u7528CMS\u63a7\u5236\u53f0\u6765\u914d\u7f6e\u544a\u8b66\u529f\u80fd\u3002\u544a\u8b66\u6761\u4ef6\u53c2\u89c1 \u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668 \u3002 \u6b65\u9aa4 \u00a7 1. \u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u548c\u62a5\u8b66\u8054\u7cfb\u7ec4 \u00a7 \u5177\u4f53\u8bf7\u53c2\u8003\u6587\u6863 \u300a\u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u6216\u62a5\u8b66\u8054\u7cfb\u7ec4\u300b 2. \u83b7\u53d6\u62a5\u8b66Token \u00a7 \u70b9\u51fb\u5de6\u4fa7\u5bfc\u822a\u680f\u201c\u62a5\u8b66\u670d\u52a1\u201d->\"\u62a5\u8b66\u8054\u7cfb\u4eba\"->\u5207\u6362\u4e0a\u6d4b\u9875\u7b7e\u5230\"\u62a5\u8b66\u8054\u7cfb\u7ec4\"->\u9009\u62e9\u5bf9\u5e94\u7684\u62a5\u8b66\u8054\u7cfb\u7ec4->\u70b9\u51fb\u53f3\u4fa7\u63a5\u5165\u5916\u90e8\u62a5\u8b66 \u67e5\u770b \u62a5\u8b66\u670d\u52a1\u8c03\u7528\u5730\u5740 \u7684\u5185\u5bb9\uff0c\u83b7\u53d6 token= \u4e4b\u540e & \u4e4b\u524d\u7684\u5185\u5bb9\u3002 \u793a\u4f8b\uff1a\u5982\u4e0b\u53ef\u83b7\u5f97 cmsToken \u4e3a XXXX \uff1a https://metrichub-cms-cn-hangzhou.aliyuncs.com/event/notify?token=XXXX&level=CRITICAL 3. \u914d\u7f6e\u73af\u5883\u53d8\u91cf \u00a7 \u73af\u5883\u53d8\u91cf \u8bf4\u660e cmsAccessKeyId \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94AccessKey ID cmsAccessSecret \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94Access Key Secret cmsRegion \u8bbe\u7f6eCMS\u6240\u5728Region\uff0c\u9700\u8981\u4e0e\u5176\u4ed6\u76f8\u5173\u7684\u4e91\u4e0a\u8d44\u6e90\u4e00\u81f4\uff0c\u4f8b\u5982ECS\uff0cOSS\uff0cEMR\u6240\u5bf9\u5e94\u7684Region cmsLevel \u8bbe\u7f6e\u6807\u8bc6\u544a\u8b66\u901a\u77e5\u65b9\u5f0f\uff0c\u5982\u679c\u672a\u8bbe\u7f6e\uff0c\u9ed8\u8ba4\u4e3aWARN\u3002INFO\u4e3a \u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cWARN\u4e3a \u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cCRITICAL\u4e3a \u7535\u8bdd+\u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \u793a\u4f8b\u5982\u4e0b export cmsAccessKeyId=<your_key_id> export cmsAccessSecret=<your_key_secret> export cmsRegion=cn-hangzhou export cmsToken=<your_cms_token> export cmsLevel=WARN 4. \u5f00\u542fCMS\u5bf9\u5931\u8d25\u4efb\u52a1\u8fdb\u884c\u544a\u8b66 \u00a7 copy\u64cd\u4f5c \u00a7 hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://example-oss-bucket/hourly_table \\ --enableCMS diff\u64cd\u4f5c \u00a7 hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://example-oss-bucket/hourly_table \\ --diff \\ --enableCMS update\u64cd\u4f5c \u00a7 hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://example-oss-bucket/hourly_table \\ --update \\ --enableCMS \u544a\u8b66\u5185\u5bb9 \u00a7 \u544a\u8b66\u5185\u5bb9\u683c\u5f0f \u00a7 { \"JobTime\": \"2021-03-09 19:45:27~2021-03-09 19:45:28\", \"SrcPath\": \"hdfs://cluster-xxx:9000//data/incoming/hourly_table\", \"DestPath\": \"oss://example-oss-bucket/hourly_table\", \"Counter\": { \"COUNTER_1\": XXX, \"COUNTER_2\": XXX, ... } } ... \u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668 \u00a7 \u544a\u8b66\u5185\u5bb9\u4e2d\u7684counter\u8ba1\u6570\u5668\u6709\u4ee5\u4e0b\u51e0\u79cd\uff1a copy\u64cd\u4f5c \u00a7 \u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 diff\u64cd\u4f5c \u00a7 \u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"JindoDistCp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66"},{"location":"jindotools/jindodistcp_how_to_cms/#jindodistcpcms","text":"JindoDistCp\uff08\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u5de5\u5177\uff09\u662f\u7528\u4e8e\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u62f7\u8d1d\u6587\u4ef6\u7684\u5de5\u5177\u3002\u4f7f\u7528MapReduce\u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3amap/reduce\u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002 CMS\uff08\u4e91\u76d1\u63a7\u670d\u52a1\uff09\uff0c\u53ef\u7528\u4e8e\u6536\u96c6\u963f\u91cc\u4e91\u8d44\u6e90\u7684\u76d1\u63a7\u6307\u6807\u6216\u7528\u6237\u81ea\u5b9a\u4e49\u7684\u76d1\u63a7\u6307\u6807\uff0c\u63a2\u6d4b\u670d\u52a1\u53ef\u7528\u6027\uff0c\u4ee5\u53ca\u9488\u5bf9\u6307\u6807\u8bbe\u7f6e\u8b66\u62a5\u3002\u4f7f\u60a8\u5168\u9762\u4e86\u89e3\u963f\u91cc\u4e91\u4e0a\u8d44\u6e90\u7684\u4f7f\u7528\u60c5\u51b5\u548c\u4e1a\u52a1\u8fd0\u884c\u72b6\u51b5\uff0c\u5e76\u53ca\u65f6\u5bf9\u6545\u969c\u8d44\u6e90\u8fdb\u884c\u5904\u7406\uff0c\u4fdd\u8bc1\u4e1a\u52a1\u6b63\u5e38\u8fd0\u884c\u3002 \u60a8\u53ef\u4ee5\u6307\u5b9a\u672c\u6b21JindoDistCp\u4efb\u52a1\u7ed3\u675f\u540e\u662f\u5426\u9700\u8981\u5f00\u542fCMS\uff0c\u4ee5\u4e0a\u62a5\u4efb\u52a1\u5931\u8d25\u4fe1\u606f\uff0c\u5e76\u4f7f\u7528CMS\u63a7\u5236\u53f0\u6765\u914d\u7f6e\u544a\u8b66\u529f\u80fd\u3002\u544a\u8b66\u6761\u4ef6\u53c2\u89c1 \u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668 \u3002","title":"JindoDistCp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66"},{"location":"jindotools/jindodistcp_how_to_cms/#_1","text":"","title":"\u6b65\u9aa4"},{"location":"jindotools/jindodistcp_how_to_cms/#1","text":"\u5177\u4f53\u8bf7\u53c2\u8003\u6587\u6863 \u300a\u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u6216\u62a5\u8b66\u8054\u7cfb\u7ec4\u300b","title":"1. \u521b\u5efa\u62a5\u8b66\u8054\u7cfb\u4eba\u548c\u62a5\u8b66\u8054\u7cfb\u7ec4"},{"location":"jindotools/jindodistcp_how_to_cms/#2-token","text":"\u70b9\u51fb\u5de6\u4fa7\u5bfc\u822a\u680f\u201c\u62a5\u8b66\u670d\u52a1\u201d->\"\u62a5\u8b66\u8054\u7cfb\u4eba\"->\u5207\u6362\u4e0a\u6d4b\u9875\u7b7e\u5230\"\u62a5\u8b66\u8054\u7cfb\u7ec4\"->\u9009\u62e9\u5bf9\u5e94\u7684\u62a5\u8b66\u8054\u7cfb\u7ec4->\u70b9\u51fb\u53f3\u4fa7\u63a5\u5165\u5916\u90e8\u62a5\u8b66 \u67e5\u770b \u62a5\u8b66\u670d\u52a1\u8c03\u7528\u5730\u5740 \u7684\u5185\u5bb9\uff0c\u83b7\u53d6 token= \u4e4b\u540e & \u4e4b\u524d\u7684\u5185\u5bb9\u3002 \u793a\u4f8b\uff1a\u5982\u4e0b\u53ef\u83b7\u5f97 cmsToken \u4e3a XXXX \uff1a https://metrichub-cms-cn-hangzhou.aliyuncs.com/event/notify?token=XXXX&level=CRITICAL","title":"2. \u83b7\u53d6\u62a5\u8b66Token"},{"location":"jindotools/jindodistcp_how_to_cms/#3","text":"\u73af\u5883\u53d8\u91cf \u8bf4\u660e cmsAccessKeyId \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94AccessKey ID cmsAccessSecret \u8bbe\u7f6e\u8d26\u53f7\u5bf9\u5e94Access Key Secret cmsRegion \u8bbe\u7f6eCMS\u6240\u5728Region\uff0c\u9700\u8981\u4e0e\u5176\u4ed6\u76f8\u5173\u7684\u4e91\u4e0a\u8d44\u6e90\u4e00\u81f4\uff0c\u4f8b\u5982ECS\uff0cOSS\uff0cEMR\u6240\u5bf9\u5e94\u7684Region cmsLevel \u8bbe\u7f6e\u6807\u8bc6\u544a\u8b66\u901a\u77e5\u65b9\u5f0f\uff0c\u5982\u679c\u672a\u8bbe\u7f6e\uff0c\u9ed8\u8ba4\u4e3aWARN\u3002INFO\u4e3a \u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cWARN\u4e3a \u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \uff0cCRITICAL\u4e3a \u7535\u8bdd+\u77ed\u4fe1+\u90ae\u4ef6+\u9489\u9489\u673a\u5668\u4eba \u793a\u4f8b\u5982\u4e0b export cmsAccessKeyId=<your_key_id> export cmsAccessSecret=<your_key_secret> export cmsRegion=cn-hangzhou export cmsToken=<your_cms_token> export cmsLevel=WARN","title":"3. \u914d\u7f6e\u73af\u5883\u53d8\u91cf"},{"location":"jindotools/jindodistcp_how_to_cms/#4-cms","text":"","title":"4. \u5f00\u542fCMS\u5bf9\u5931\u8d25\u4efb\u52a1\u8fdb\u884c\u544a\u8b66"},{"location":"jindotools/jindodistcp_how_to_cms/#copy","text":"hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://example-oss-bucket/hourly_table \\ --enableCMS","title":"copy\u64cd\u4f5c"},{"location":"jindotools/jindodistcp_how_to_cms/#diff","text":"hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://example-oss-bucket/hourly_table \\ --diff \\ --enableCMS","title":"diff\u64cd\u4f5c"},{"location":"jindotools/jindodistcp_how_to_cms/#update","text":"hadoop jar jindo-distcp-tool-${version}.jar \\ --src /data/incoming/hourly_table \\ --dest oss://example-oss-bucket/hourly_table \\ --update \\ --enableCMS","title":"update\u64cd\u4f5c"},{"location":"jindotools/jindodistcp_how_to_cms/#_2","text":"","title":"\u544a\u8b66\u5185\u5bb9"},{"location":"jindotools/jindodistcp_how_to_cms/#_3","text":"{ \"JobTime\": \"2021-03-09 19:45:27~2021-03-09 19:45:28\", \"SrcPath\": \"hdfs://cluster-xxx:9000//data/incoming/hourly_table\", \"DestPath\": \"oss://example-oss-bucket/hourly_table\", \"Counter\": { \"COUNTER_1\": XXX, \"COUNTER_2\": XXX, ... } } ...","title":"\u544a\u8b66\u5185\u5bb9\u683c\u5f0f"},{"location":"jindotools/jindodistcp_how_to_cms/#_4","text":"\u544a\u8b66\u5185\u5bb9\u4e2d\u7684counter\u8ba1\u6570\u5668\u6709\u4ee5\u4e0b\u51e0\u79cd\uff1a","title":"\u544a\u8b66\u4efb\u52a1\u8ba1\u6570\u5668"},{"location":"jindotools/jindodistcp_how_to_cms/#copy_1","text":"\u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570","title":"copy\u64cd\u4f5c"},{"location":"jindotools/jindodistcp_how_to_cms/#diff_1","text":"\u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570\uff0c \u4e0d\u4e3a0\u65f6\u544a\u8b66 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"diff\u64cd\u4f5c"},{"location":"jindotools/jindodistcp_quickstart/","text":"JindoDistCp \u4f7f\u7528\u8bf4\u660e \u00a7 JindoDistCp \u4ecb\u7ecd \u00a7 JindoDistCp \u662f\u963f\u91cc\u4e91\u6570\u636e\u6e56\u5b58\u50a8\u56e2\u961f\u5f00\u53d1\u7684\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u7684\u5de5\u5177\u3002\u5176\u4f7f\u7528 MapReduce \u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3a MapReduce \u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002\u76ee\u524d\u5168\u91cf\u652f\u6301 HDFS/OSS-HDFS/OSS/S3 \u4e4b\u95f4\u7684\u7684\u6570\u636e\u62f7\u8d1d\u573a\u666f\uff0c\u63d0\u4f9b\u591a\u79cd\u4e2a\u6027\u5316\u62f7\u8d1d\u53c2\u6570\u548c\u591a\u79cd\u62f7\u8d1d\u7b56\u7565\u3002\u91cd\u70b9\u4f18\u5316 HDFS \u5230 OSS-HDFS \u7684\u6570\u636e\u62f7\u8d1d\uff0c\u901a\u8fc7\u5b9a\u5236\u5316 CopyCommitter\uff0c\u5b9e\u73b0 No-Rename \u62f7\u8d1d\uff0c\u5e76\u4fdd\u8bc1\u6570\u636e\u62f7\u8d1d\u843d\u5730\u7684\u4e00\u81f4\u6027\u3002\u529f\u80fd\u5168\u91cf\u5bf9\u9f50 S3 DistCp \u548c HDFS DistCp\uff0c\u6027\u80fd\u8f83 HDFS DistCp \u6709\u8f83\u5927\u63d0\u5347\uff0c\u81f4\u529b\u4e8e\u63d0\u4f9b\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177\u3002 \u73af\u5883\u8981\u6c42 \u00a7 JDK 1.8\u53ca\u4ee5\u4e0a Hadoop 2.3+ \u7248\u672c\uff0c\u8bf7\u4e0b\u8f7d\u6700\u65b0\u7248\u7684 jindo-distcp-tool-x.x.x.jar\uff0c\u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz \u5185\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 tools/ \u76ee\u5f55\u4e0b\u627e\u5230\uff0c \u4e0b\u8f7d\u9875\u9762 \u3002 \u6ce8\uff1aE-MapReduce EMR-5.6.0/EMR-3.40.0 \u6216\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u5728\u96c6\u7fa4\u4e2d\u5df2\u90e8\u7f72 JindoDistCp\uff0c\u53ef\u5728 /opt/apps/JINDOSDK/jindosdk-current/tools \u76ee\u5f55\u4e0b\u627e\u5230 jindo-distcp-tool-x.x.x.jar\u3002 \u53c2\u6570\u8bf4\u660e \u00a7 JindoDistCp \u63d0\u4f9b jar \u5305\u5f62\u5f0f\u4f7f\u7528\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 hadoop jar \u547d\u4ee4\u914d\u5408\u4e00\u7cfb\u5217\u53c2\u6570\u6765\u5b8c\u6210\u8fc1\u79fb\u64cd\u4f5c\u3002 \u53c2\u6570 \u53c2\u6570\u7c7b\u578b \u8bf4\u660e \u9ed8\u8ba4\u503c \u7248\u672c OSS OSS-HDFS --src \u5fc5\u9009 \u8bbe\u7f6e\u6e90\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ \u652f\u6301 \u652f\u6301 --dest \u5fc5\u9009 \u8bbe\u7f6e\u76ee\u6807\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ \u652f\u6301 \u652f\u6301 --bandWidth \u53ef\u9009 \u8bbe\u7f6e\u5355\u4e2a\u8282\u70b9\u7684\u5e26\u5bbd\u9650\u5236\uff0c\u5355\u4f4d M -1 4.3.0+ \u652f\u6301 \u652f\u6301 --codec \u53ef\u9009 \u8bbe\u7f6e\u538b\u7f29\u7c7b\u578b\uff0c\u652f\u6301\u7f16\u89e3\u7801\u5668\u6709 gzip\u3001gz\u3001lzo\u3001lzop\u3001lzop\u3001snappy keep\uff08\u4e0d\u66f4\u6539\u538b\u7f29\u7c7b\u578b\uff09 4.3.0+ \u652f\u6301 \u652f\u6301 --policy \u53ef\u9009 \u8bbe\u7f6e\u76ee\u6807\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301 Standard\u3001IA\u3001Archive\u3001ColdArchive Standard 4.3.0+ \u652f\u6301 \u4e0d\u652f\u6301 --filters \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ \u652f\u6301 \u652f\u6301 --srcPrefixesFile \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ \u652f\u6301 \u652f\u6301 --parallelism \u53ef\u9009 \u8bbe\u7f6eDistCp\u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\uff0c\u5bf9\u5e94MR\u4efb\u52a1\u4e2d\u7684 mapreduce.job.maps 10 4.3.0+ \u652f\u6301 \u652f\u6301 --jobBatch \u53ef\u9009 \u8bbe\u7f6e\u6bcf\u4e2a distcp job \u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf 10000 4.5.1+ \u652f\u6301 \u652f\u6301 --taskBatch \u53ef\u9009 \u8bbe\u7f6e\u6bcf\u4e2a distcp task \u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf 1 4.3.0+ \u652f\u6301 \u652f\u6301 --tmp \u53ef\u9009 \u8bbe\u7f6e\u4e34\u65f6\u76ee\u5f55 /tmp 4.3.0+ \u652f\u6301 \u652f\u6301 --hadoopConf \u53ef\u9009 \u8bbe\u7f6e Configuration - 4.3.0+ \u652f\u6301 \u652f\u6301 --disableChecksum \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5173\u95ed Checksum \u68c0\u67e5 false 4.3.0+ \u652f\u6301 \u652f\u6301 --deleteOnSuccess \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5220\u9664\u6e90\u6587\u4ef6\uff0c\u7528\u4e8e\u79fb\u52a8\u6570\u636e false 4.3.0+ \u652f\u6301 \u652f\u6301 --enableTransaction \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4e8b\u52a1\uff0c\u4ee5\u4fdd\u8bc1 Job \u7ea7\u522b\u7684\u539f\u5b50\u6027 false 4.3.0+ \u652f\u6301 \u652f\u6301 --ignore \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5ffd\u7565\u62f7\u8d1d\u4efb\u52a1\u4e2d\u629b\u51fa\u7684\u5f02\u5e38\uff0c\u907f\u514d\u4e2d\u65ad\u4efb\u52a1 false 4.3.0+ \u652f\u6301 \u652f\u6301 --enableCMS \u53ef\u9009 \u662f\u5426\u5f00\u542f\u76d1\u63a7\u544a\u8b66 false 4.5.1+ \u652f\u6301 \u652f\u6301 --diff \u53ef\u9009 \u8bbe\u7f6e DistCp \u6a21\u5f0f\u4e3a DIFF, \u67e5\u770b src \u548c dest \u7684\u6587\u4ef6\u5dee\u5f02 DistCpMode.COPY 4.3.0+ \u652f\u6301 \u652f\u6301 --update \u53ef\u9009 \u8bbe\u7f6e DistCp \u6a21\u5f0f\u4e3a UPDATE, \u6307\u5b9a\u589e\u91cf\u540c\u6b65\u529f\u80fd\uff0c\u8df3\u8fc7\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u548c\u76ee\u5f55\uff0c\u76f4\u63a5\u5c06 src \u4e2d\u65b0\u589e\u6216\u53d1\u751f\u6539\u53d8\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u540c\u6b65\u5230 dest \u4e0a DistCpMode.COPY 4.3.0+ \u652f\u6301 \u652f\u6301 --preserveMeta \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f false 4.4.0+ \u4e0d\u652f\u6301 \u652f\u6301 1\u3001\u4f7f\u7528 --src \u548c --dest (\u5fc5\u9009) \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --src \u8868\u793a\u6307\u5b9a\u6e90\u6587\u4ef6\u7684\u8def\u5f84\u3002 --dest \u8868\u793a\u76ee\u6807\u6587\u4ef6\u7684\u8def\u5f84\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table \u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9a dest \u8def\u5f84\u6765\u786e\u5b9a\u62f7\u8d1d\u540e\u7684\u6587\u4ef6\u5c42\u6b21\uff0c\u5982\u60a8\u9700\u8981\u5c06 /data/hourly_table \u4e0b\u7684\u6587\u4ef6\u62f7\u8d1d\u5230 example-oss-bucket \u8fd9\u4e2a bucket \u4e0b\u7684 hourly_table \u76ee\u5f55\u4e0b\uff0c\u5219\u53ef\u4ee5\u4f7f\u7528\u4e0a\u8ff0\u8bed\u53e5\u6765\u5b8c\u6210\u3002\u6b64\u5904\u548c Hadoop \u7684 DistCp \u884c\u4e3a\u6709\u6240\u4e0d\u540c\uff0cJindoDistCp \u4f1a\u9ed8\u8ba4\u5c06 src \u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6587\u4ef6\u62f7\u8d1d\u5230\u60a8\u6307\u5b9a dest \u8def\u5f84\u4e0b\uff0c\u5e76\u4e0d\u5305\u62ec\u5f53\u524d\u7684\u6839\u76ee\u5f55\u540d\u79f0\uff0c\u60a8\u53ef\u4ee5\u5728 dest \u4e2d\u6307\u5b9a\u62f7\u8d1d\u8def\u5f84\u7684\u6839\u76ee\u5f55\uff0c\u5982\u679c\u4e0d\u5b58\u5728\u4f1a\u81ea\u52a8\u521b\u5efa\u3002 \u6ce8\u610f\uff1a\u5982\u9700\u62f7\u8d1d\u5355\u4e2a\u6587\u4ef6, dest \u9700\u6307\u5b9a\u4e3a\u76ee\u5f55\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /test.txt --dest oss://example-oss-bucket/tmp 2\u3001\u4f7f\u7528 --bandWidth \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --bandWidth \u8868\u793a\u672c\u6b21distcp\u4efb\u52a1\u6240\u7528\u7684\u5355\u673a\u5e26\u5bbd(\u4ee5MB\u4e3a\u5355\u4f4d)\uff0c\u907f\u514d\u5355\u673a\u5360\u7528\u8fc7\u5927\u5e26\u5bbd\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --bandWidth 6 3\u3001\u4f7f\u7528 --codec \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 \u539f\u59cb\u6587\u4ef6\u901a\u5e38\u4ee5\u672a\u538b\u7f29\u7684\u6587\u672c\u683c\u5f0f\u8fdb\u5165 OSS/OSS-HDFS\u3002\u65e0\u8bba\u662f\u5b58\u50a8\u6210\u672c\u8fd8\u662f\u5bf9\u8be5\u6570\u636e\u8fdb\u884c\u5206\u6790\uff0c\u6b64\u683c\u5f0f\u90fd\u4e0d\u751a\u7406\u60f3\u3002JindoDistCp \u53ef\u4ee5\u4f7f\u7528 --codec \u9009\u9879\u5e2e\u52a9\u60a8\u5728\u7ebf\u9ad8\u6548\u5730\u5b58\u50a8\u6570\u636e\u548c\u538b\u7f29\u6587\u4ef6\u3002 --codec \u6307\u5b9a\u6587\u4ef6\u538b\u7f29\u7f16\u89e3\u7801\u5668\u3002 \u652f\u6301 gzip\u3001gz\u3001lzo\u3001lzop \u548c snappy \u5173\u952e\u5b57\u3002 none \u4fdd\u5b58\u4e3a\u672a\u538b\u7f29\u7684\u6587\u4ef6\u3002\u5982\u679c\u6587\u4ef6\u5df2\u538b\u7f29\uff0c\u5219 JindoDistCp \u4f1a\u5c06\u5176\u89e3\u538b\u7f29\u3002 keep(\u9ed8\u8ba4\u503c) \u4e0d\u66f4\u6539\u6587\u4ef6\u538b\u7f29\u5f62\u6001\uff0c\u6309\u539f\u6837\u590d\u5236\u3002 \u547d\u4ee4\u793a\u4f8b\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --codec gz \u6211\u4eec\u68c0\u67e5\u4e00\u4e0b\u76ee\u6807\u6587\u4ef6\u5939\u4e2d\u7684\u6587\u4ef6\uff0c\u8fd9\u4e9b\u6587\u4ef6\u73b0\u5728\u5df2\u7ecf\u7528 gz \u7f16\u89e3\u7801\u5668\u538b\u7f29\u4e86\uff1a [root@emr-header-1 opt]# hdfs dfs -ls oss://example-oss-bucket/hourly_table/2017-02-01/03 Found 6 items -rw-rw-rw- 1 938 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/000151.sst.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/1.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/2.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/OPTIONS-000109.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/emp01.txt.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/emp06.txt.gz \u5982\u60a8\u5728\u5f00\u6e90Hadoop\u96c6\u7fa4\u73af\u5883\u4e2d\u4f7f\u7528lzo\u538b\u7f29\u529f\u80fd\uff0c\u5219\u60a8\u9700\u8981\u5b89\u88c5 gplcompression \u7684 native \u5e93\u548chadoop-lzo\u5305\uff0c\u5982\u60a8\u7f3a\u5c11\u76f8\u5173\u73af\u5883\uff0c\u5efa\u8bae\u4f7f\u7528\u5176\u4ed6\u538b\u7f29\u65b9\u5f0f\u8fdb\u884c\u538b\u7f29\u3002 4\u3001\u4f7f\u7528 --filters \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --filters \u6307\u5b9a\u6709\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --filters filter.txt \u5047\u8bbe filter.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u4e0d\u4f1a\u5c06\u8def\u5f84\u4e2d\u5e26\u6709 test \u5b57\u7b26\u7684\u6587\u4ef6\u62f7\u8d1d\u5230OSS\u3002 5\u3001\u4f7f\u7528 --srcPrefixesFile \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --srcPrefixesFile \u6307\u5b9a\u6709\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --srcPrefixesFile prefixes.txt \u5047\u8bbe prefixes.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u53ea\u6709\u8def\u5f84\u4e2d\u5e26\u6709 test \u5b57\u7b26\u7684\u6587\u4ef6\u624d\u4f1a\u62f7\u8d1d\u5230OSS\u3002 6\u3001\u4f7f\u7528 --parallelism \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --parallelism \u6307\u5b9aMR\u4efb\u52a1\u91cc\u7684 mapreduce.job.reduces \u53c2\u6570\uff0c\u8be5\u53c2\u6570\u5728EMR\u73af\u5883\u4e2d\u9ed8\u8ba4\u4e3a7\uff0c\u60a8\u53ef\u4ee5\u6839\u636e\u96c6\u7fa4\u7684\u8d44\u6e90\u60c5\u51b5\u81ea\u5b9a\u4e49 parallelism \u7684\u5927\u5c0f\u6765\u63a7\u5236 distcp \u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://example-oss-bucket/tmp --parallelism 20 7\u3001\u4f7f\u7528 --taskBatch \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --taskBatch \u6307\u5b9a\u6bcf\u4e2a distcp task \u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --taskBatch 10 8\u3001\u4f7f\u7528 --tmp \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --tmp \u6307\u5b9a\u4e00\u4e2a\u4e34\u65f6\u76ee\u5f55\u5728hdfs\u4e0a\u4f20\u5b58\u653e\u4e34\u65f6\u6570\u636e\uff0c\u9ed8\u8ba4\u503c\u4e3a/tmp\uff0c\u5373hdfs:///tmp/\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --tmp /tmp 9\u3001\u914d\u7f6e\u8bbf\u95ee OSS/OSS-HDFS \u7684 AK \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 \u5728 EMR \u5916\u6216\u514d\u5bc6\u670d\u52a1\u51fa\u73b0\u95ee\u9898\u7684\u60c5\u51b5\u4e0b\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9aAK\u6765\u83b7\u5f97\u8bbf\u95ee OSS/OSS-HDFS \u670d\u52a1\u7684\u6743\u9650\u3002\u60a8\u53ef\u4ee5\u5728\u547d\u4ee4\u4e2d\u4f7f\u7528 --hadoopConf \u9009\u9879\u6765\u6307\u5b9a AK\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret \u60a8\u4e5f\u53ef\u4ee5\u5c06 OSS/OSS-HDFS \u670d\u52a1\u7684 ak\u3001secret \u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u907f\u514d\u6bcf\u6b21\u4f7f\u7528\u65f6\u4e34\u65f6\u586b\u5199 ak\u3002\u60a8\u53ef\u4ee5\u5728 E-MapReduce \u63a7\u5236\u53f0 Hadoop-Common \u670d\u52a1\u7684 core-site.xml \u9875\u9762\u914d\u7f6e\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> 10\u3001\u4f7f\u7528 --disableChecksum \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --disableChecksum \u5173\u95ed\u68c0\u67e5\u6587\u4ef6 checksum\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --disableChecksum 11\u3001\u4f7f\u7528 --deleteOnSuccess \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --deleteOnSuccess \u6307\u5b9a\u79fb\u52a8\u6570\u636e\u800c\u4e0d\u662f\u590d\u5236\u6570\u636e\u3002\u6b64\u9009\u9879\u7c7b\u4f3c\u4e8e mv \u64cd\u4f5c\uff0c\u9996\u5148\u590d\u5236\u6587\u4ef6\uff0c\u7136\u540e\u4ece\u6e90\u4f4d\u7f6e\u5220\u9664\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --deleteOnSuccess 12\u3001\u4f7f\u7528 --enableTransaction \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u4e0d\u652f\u6301 \u652f\u6301 --enableTransaction JindoDistCp \u9ed8\u8ba4\u4f7f\u7528 Task \u7ea7\u522b\u5b8c\u6574\u6027\uff0c\u5982\u60a8\u9700\u8981\u4fdd\u8bc1 Job \u7ea7\u522b\u7684\u5b8c\u6574\u6027\u4ee5\u53ca\u4fdd\u8bc1 Job \u4e4b\u95f4\u7684\u4e8b\u52a1\u652f\u6301\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528\u8be5\u53c2\u6570\u6307\u5b9a\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --enableTransaction 13\u3001\u4f7f\u7528 --ignore \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --ignore \u5ffd\u7565\u6570\u636e\u8fc1\u79fb\u671f\u95f4\u53d1\u751f\u7684\u5f02\u5e38\uff0c\u76f8\u5173\u62a5\u9519\u4e0d\u4f1a\u4e2d\u65ad\u4efb\u52a1\uff0c\u5e76\u6700\u7ec8\u4ee5 JindoCounter \u7684\u5f62\u5f0f\u900f\u51fa\u3002\uff08\u5982\u679c\u5f00\u542fCMS\u7684\u8bdd\uff0c\u4e5f\u4f1a\u4ee5\u6307\u5b9a\u65b9\u5f0f\u8fdb\u884c\u901a\u77e5\u3002\uff09 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --ignore 14\u3001\u4f7f\u7528 --diff \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --diff \u67e5\u770b src \u548c dest \u7684\u6587\u4ef6\u5dee\u5f02\u3002\u5982\u679c src \u7684\u6587\u4ef6\u672a\u80fd\u540c\u6b65\u5230 dest \u4e0a\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\u3002\u5982\u60a8\u7684 JindoDistCp \u4efb\u52a1\u5305\u542b\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u5219 --diff \u4e0d\u80fd\u663e\u793a\u6b63\u786e\u7684\u6587\u4ef6\u5dee\u5f02\uff0c\u56e0\u4e3a\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u4f1a\u6539\u53d8\u6587\u4ef6\u7684\u5927\u5c0f\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --diff \u5982\u679c\u6587\u4ef6\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter DIFF_FILES=1 \u5982\u679c\u60a8\u7684--dest\u4e3aHDFS\u8def\u5f84\uff0c\u73b0\u652f\u6301 /path \uff0c hdfs://hostname:ip/path \uff0c hdfs://headerIp:ip/path \u7684\u5199\u6cd5\uff0c\u6682\u4e0d\u652f\u6301 hdfs:///path \uff0c hdfs:/path \u548c\u5176\u4ed6\u81ea\u5b9a\u4e49\u5199\u6cd5\u3002 \u5982\u679c\u60f3\u67e5\u770b\u6587\u4ef6\u5143\u6570\u636e\u7684\u5dee\u5f02\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 --diff --preserveMeta \u547d\u4ee4\uff1a \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --diff --preserveMeta \u5982\u679c\u6587\u4ef6\u5143\u6570\u636e\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5143\u6570\u636e\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter META_FILE_DIFF=1 15\u3001\u4f7f\u7528 --update \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --update \u6307\u5b9a\u589e\u91cf\u540c\u6b65\u529f\u80fd\uff0c\u8df3\u8fc7\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u548c\u76ee\u5f55\uff0c\u76f4\u63a5\u5c06 src \u4e2d\u65b0\u589e\u6216\u53d1\u751f\u6539\u53d8\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u540c\u6b65\u5230 dest \u4e0a\u3002 \u5982\u679c JindoDistCp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53ea Copy \u5269\u4e0b\u672a Copy \u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 JindoDistCp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u6307\u5b9a\u8be5\u53c2\u6570\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --update 16\u3001\u4f7f\u7528\u51b7\u5f52\u6863/\u5f52\u6863/\u4f4e\u9891\u5199\u5165OSS \u00a7 \u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u4e0d\u652f\u6301 --update \u6307\u5b9a\u4ee5\u51b7\u5f52\u6863\u3001\u5f52\u6863\u548c\u4f4e\u9891\u7684\u6a21\u5f0f\u5199\u5165 OSS\uff0c\u8fdb\u884c\u6570\u636e\u5b58\u50a8\u3002 1\uff09\u4f7f\u7528\u51b7\u5f52\u6863\u5199\u5165OSS\uff08coldArchive\uff09 \u76ee\u524d\u53ea\u5728\u90e8\u5206region\u53ef\u7528\uff0c\u5177\u4f53\u53c2\u89c1 OSS\u5b58\u50a8\u7c7b\u578b\u4ecb\u7ecd \uff0c\u4f7f\u7528\u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-bucket/hourly_table --policy coldArchive --parallelism 20 2\uff09\u4f7f\u7528\u5f52\u6863\u5199\u5165OSS\uff08archive\uff09 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-bucket/hourly_table --policy archive --parallelism 20 3\uff09\u4f7f\u7528\u4f4e\u9891\u5199\u5165OSS\uff08ia\uff09 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-bucket/hourly_table --policy ia --parallelism 20 \u5982\u60a8\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4\u4ee5 standard \u6807\u51c6\u6a21\u5f0f\u5199\u5165\uff0c\u4e0d\u8fdb\u884c\u51b7\u5f52\u6863\u3001\u5f52\u6863\u548c\u4f4e\u9891\u64cd\u4f5c\u3002 17\u3001\u4f7f\u7528 --preserveMeta \u00a7 \u7248\u672c OSS OSS-HDFS 4.4.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u4e0d\u652f\u6301 \u652f\u6301 --preserveMeta \u6307\u5b9a\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --preserveMeta 18\u3001\u4f7f\u7528 --jobBatch \u00a7 \u7248\u672c OSS OSS-HDFS 4.5.1 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --jobBatch \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--jobBatch\u6765\u6307\u5b9a\u6bcf\u4e2adistcp job\u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10000\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --jobBatch 50000 19\u3001 \u4f7f\u7528 --enableCMS \u00a7 \u7248\u672c OSS OSS-HDFS 4.5.1 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --enableCMS \u5f00\u542f\u4e91\u76d1\u63a7\u544a\u8b66\u529f\u80fd\uff0c\u5177\u4f53\u53c2\u89c1 \u300aJindoDistcp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66\u300b \u3002 JindoDistCp Counters \u8bf4\u660e \u00a7 \u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf BYTES_EXPECTED \u9884\u671f\u7684copy\u5b57\u8282\u6570 FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 BYTES_COPIED copy\u6210\u529f\u7684\u5b57\u8282\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"JindoDistcp \u5feb\u901f\u5165\u95e8"},{"location":"jindotools/jindodistcp_quickstart/#jindodistcp","text":"","title":"JindoDistCp \u4f7f\u7528\u8bf4\u660e"},{"location":"jindotools/jindodistcp_quickstart/#jindodistcp_1","text":"JindoDistCp \u662f\u963f\u91cc\u4e91\u6570\u636e\u6e56\u5b58\u50a8\u56e2\u961f\u5f00\u53d1\u7684\u5927\u89c4\u6a21\u96c6\u7fa4\u5185\u90e8\u548c\u96c6\u7fa4\u4e4b\u95f4\u5206\u5e03\u5f0f\u6587\u4ef6\u62f7\u8d1d\u7684\u5de5\u5177\u3002\u5176\u4f7f\u7528 MapReduce \u5b9e\u73b0\u6587\u4ef6\u5206\u53d1\uff0c\u9519\u8bef\u5904\u7406\u548c\u6062\u590d\uff0c\u628a\u6587\u4ef6\u548c\u76ee\u5f55\u7684\u5217\u8868\u4f5c\u4e3a MapReduce \u4efb\u52a1\u7684\u8f93\u5165\uff0c\u6bcf\u4e2a\u4efb\u52a1\u4f1a\u5b8c\u6210\u6e90\u5217\u8868\u4e2d\u90e8\u5206\u6587\u4ef6\u7684\u62f7\u8d1d\u3002\u76ee\u524d\u5168\u91cf\u652f\u6301 HDFS/OSS-HDFS/OSS/S3 \u4e4b\u95f4\u7684\u7684\u6570\u636e\u62f7\u8d1d\u573a\u666f\uff0c\u63d0\u4f9b\u591a\u79cd\u4e2a\u6027\u5316\u62f7\u8d1d\u53c2\u6570\u548c\u591a\u79cd\u62f7\u8d1d\u7b56\u7565\u3002\u91cd\u70b9\u4f18\u5316 HDFS \u5230 OSS-HDFS \u7684\u6570\u636e\u62f7\u8d1d\uff0c\u901a\u8fc7\u5b9a\u5236\u5316 CopyCommitter\uff0c\u5b9e\u73b0 No-Rename \u62f7\u8d1d\uff0c\u5e76\u4fdd\u8bc1\u6570\u636e\u62f7\u8d1d\u843d\u5730\u7684\u4e00\u81f4\u6027\u3002\u529f\u80fd\u5168\u91cf\u5bf9\u9f50 S3 DistCp \u548c HDFS DistCp\uff0c\u6027\u80fd\u8f83 HDFS DistCp \u6709\u8f83\u5927\u63d0\u5347\uff0c\u81f4\u529b\u4e8e\u63d0\u4f9b\u9ad8\u6548\u3001\u7a33\u5b9a\u3001\u5b89\u5168\u7684\u6570\u636e\u62f7\u8d1d\u5de5\u5177\u3002","title":"JindoDistCp \u4ecb\u7ecd"},{"location":"jindotools/jindodistcp_quickstart/#_1","text":"JDK 1.8\u53ca\u4ee5\u4e0a Hadoop 2.3+ \u7248\u672c\uff0c\u8bf7\u4e0b\u8f7d\u6700\u65b0\u7248\u7684 jindo-distcp-tool-x.x.x.jar\uff0c\u8be5 jar \u5305\u542b\u5728 jindosdk-${version}.tar.gz \u5185\uff0c\u89e3\u538b\u7f29\u540e\u53ef\u5728 tools/ \u76ee\u5f55\u4e0b\u627e\u5230\uff0c \u4e0b\u8f7d\u9875\u9762 \u3002 \u6ce8\uff1aE-MapReduce EMR-5.6.0/EMR-3.40.0 \u6216\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u5728\u96c6\u7fa4\u4e2d\u5df2\u90e8\u7f72 JindoDistCp\uff0c\u53ef\u5728 /opt/apps/JINDOSDK/jindosdk-current/tools \u76ee\u5f55\u4e0b\u627e\u5230 jindo-distcp-tool-x.x.x.jar\u3002","title":"\u73af\u5883\u8981\u6c42"},{"location":"jindotools/jindodistcp_quickstart/#_2","text":"JindoDistCp \u63d0\u4f9b jar \u5305\u5f62\u5f0f\u4f7f\u7528\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 hadoop jar \u547d\u4ee4\u914d\u5408\u4e00\u7cfb\u5217\u53c2\u6570\u6765\u5b8c\u6210\u8fc1\u79fb\u64cd\u4f5c\u3002 \u53c2\u6570 \u53c2\u6570\u7c7b\u578b \u8bf4\u660e \u9ed8\u8ba4\u503c \u7248\u672c OSS OSS-HDFS --src \u5fc5\u9009 \u8bbe\u7f6e\u6e90\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ \u652f\u6301 \u652f\u6301 --dest \u5fc5\u9009 \u8bbe\u7f6e\u76ee\u6807\u76ee\u5f55\uff0c\u652f\u6301\u7684\u524d\u7f00\u6709 hdfs:// oss:// s3:// cos:// obs:// - 4.3.0+ \u652f\u6301 \u652f\u6301 --bandWidth \u53ef\u9009 \u8bbe\u7f6e\u5355\u4e2a\u8282\u70b9\u7684\u5e26\u5bbd\u9650\u5236\uff0c\u5355\u4f4d M -1 4.3.0+ \u652f\u6301 \u652f\u6301 --codec \u53ef\u9009 \u8bbe\u7f6e\u538b\u7f29\u7c7b\u578b\uff0c\u652f\u6301\u7f16\u89e3\u7801\u5668\u6709 gzip\u3001gz\u3001lzo\u3001lzop\u3001lzop\u3001snappy keep\uff08\u4e0d\u66f4\u6539\u538b\u7f29\u7c7b\u578b\uff09 4.3.0+ \u652f\u6301 \u652f\u6301 --policy \u53ef\u9009 \u8bbe\u7f6e\u76ee\u6807\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301 Standard\u3001IA\u3001Archive\u3001ColdArchive Standard 4.3.0+ \u652f\u6301 \u4e0d\u652f\u6301 --filters \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ \u652f\u6301 \u652f\u6301 --srcPrefixesFile \u53ef\u9009 \u8bbe\u7f6e\u5305\u542b\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6 - 4.3.0+ \u652f\u6301 \u652f\u6301 --parallelism \u53ef\u9009 \u8bbe\u7f6eDistCp\u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\uff0c\u5bf9\u5e94MR\u4efb\u52a1\u4e2d\u7684 mapreduce.job.maps 10 4.3.0+ \u652f\u6301 \u652f\u6301 --jobBatch \u53ef\u9009 \u8bbe\u7f6e\u6bcf\u4e2a distcp job \u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf 10000 4.5.1+ \u652f\u6301 \u652f\u6301 --taskBatch \u53ef\u9009 \u8bbe\u7f6e\u6bcf\u4e2a distcp task \u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf 1 4.3.0+ \u652f\u6301 \u652f\u6301 --tmp \u53ef\u9009 \u8bbe\u7f6e\u4e34\u65f6\u76ee\u5f55 /tmp 4.3.0+ \u652f\u6301 \u652f\u6301 --hadoopConf \u53ef\u9009 \u8bbe\u7f6e Configuration - 4.3.0+ \u652f\u6301 \u652f\u6301 --disableChecksum \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5173\u95ed Checksum \u68c0\u67e5 false 4.3.0+ \u652f\u6301 \u652f\u6301 --deleteOnSuccess \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5220\u9664\u6e90\u6587\u4ef6\uff0c\u7528\u4e8e\u79fb\u52a8\u6570\u636e false 4.3.0+ \u652f\u6301 \u652f\u6301 --enableTransaction \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4e8b\u52a1\uff0c\u4ee5\u4fdd\u8bc1 Job \u7ea7\u522b\u7684\u539f\u5b50\u6027 false 4.3.0+ \u652f\u6301 \u652f\u6301 --ignore \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5ffd\u7565\u62f7\u8d1d\u4efb\u52a1\u4e2d\u629b\u51fa\u7684\u5f02\u5e38\uff0c\u907f\u514d\u4e2d\u65ad\u4efb\u52a1 false 4.3.0+ \u652f\u6301 \u652f\u6301 --enableCMS \u53ef\u9009 \u662f\u5426\u5f00\u542f\u76d1\u63a7\u544a\u8b66 false 4.5.1+ \u652f\u6301 \u652f\u6301 --diff \u53ef\u9009 \u8bbe\u7f6e DistCp \u6a21\u5f0f\u4e3a DIFF, \u67e5\u770b src \u548c dest \u7684\u6587\u4ef6\u5dee\u5f02 DistCpMode.COPY 4.3.0+ \u652f\u6301 \u652f\u6301 --update \u53ef\u9009 \u8bbe\u7f6e DistCp \u6a21\u5f0f\u4e3a UPDATE, \u6307\u5b9a\u589e\u91cf\u540c\u6b65\u529f\u80fd\uff0c\u8df3\u8fc7\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u548c\u76ee\u5f55\uff0c\u76f4\u63a5\u5c06 src \u4e2d\u65b0\u589e\u6216\u53d1\u751f\u6539\u53d8\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u540c\u6b65\u5230 dest \u4e0a DistCpMode.COPY 4.3.0+ \u652f\u6301 \u652f\u6301 --preserveMeta \u53ef\u9009 \u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u4fdd\u5b58\u5143\u6570\u636e\u4fe1\u606f false 4.4.0+ \u4e0d\u652f\u6301 \u652f\u6301","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"jindotools/jindodistcp_quickstart/#1-src-dest","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --src \u8868\u793a\u6307\u5b9a\u6e90\u6587\u4ef6\u7684\u8def\u5f84\u3002 --dest \u8868\u793a\u76ee\u6807\u6587\u4ef6\u7684\u8def\u5f84\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table \u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9a dest \u8def\u5f84\u6765\u786e\u5b9a\u62f7\u8d1d\u540e\u7684\u6587\u4ef6\u5c42\u6b21\uff0c\u5982\u60a8\u9700\u8981\u5c06 /data/hourly_table \u4e0b\u7684\u6587\u4ef6\u62f7\u8d1d\u5230 example-oss-bucket \u8fd9\u4e2a bucket \u4e0b\u7684 hourly_table \u76ee\u5f55\u4e0b\uff0c\u5219\u53ef\u4ee5\u4f7f\u7528\u4e0a\u8ff0\u8bed\u53e5\u6765\u5b8c\u6210\u3002\u6b64\u5904\u548c Hadoop \u7684 DistCp \u884c\u4e3a\u6709\u6240\u4e0d\u540c\uff0cJindoDistCp \u4f1a\u9ed8\u8ba4\u5c06 src \u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6587\u4ef6\u62f7\u8d1d\u5230\u60a8\u6307\u5b9a dest \u8def\u5f84\u4e0b\uff0c\u5e76\u4e0d\u5305\u62ec\u5f53\u524d\u7684\u6839\u76ee\u5f55\u540d\u79f0\uff0c\u60a8\u53ef\u4ee5\u5728 dest \u4e2d\u6307\u5b9a\u62f7\u8d1d\u8def\u5f84\u7684\u6839\u76ee\u5f55\uff0c\u5982\u679c\u4e0d\u5b58\u5728\u4f1a\u81ea\u52a8\u521b\u5efa\u3002 \u6ce8\u610f\uff1a\u5982\u9700\u62f7\u8d1d\u5355\u4e2a\u6587\u4ef6, dest \u9700\u6307\u5b9a\u4e3a\u76ee\u5f55\u3002 hadoop jar jindo-distcp-tool-${version}.jar --src /test.txt --dest oss://example-oss-bucket/tmp","title":"1\u3001\u4f7f\u7528 --src \u548c --dest (\u5fc5\u9009)"},{"location":"jindotools/jindodistcp_quickstart/#2-bandwidth","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --bandWidth \u8868\u793a\u672c\u6b21distcp\u4efb\u52a1\u6240\u7528\u7684\u5355\u673a\u5e26\u5bbd(\u4ee5MB\u4e3a\u5355\u4f4d)\uff0c\u907f\u514d\u5355\u673a\u5360\u7528\u8fc7\u5927\u5e26\u5bbd\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --bandWidth 6","title":"2\u3001\u4f7f\u7528 --bandWidth"},{"location":"jindotools/jindodistcp_quickstart/#3-codec","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 \u539f\u59cb\u6587\u4ef6\u901a\u5e38\u4ee5\u672a\u538b\u7f29\u7684\u6587\u672c\u683c\u5f0f\u8fdb\u5165 OSS/OSS-HDFS\u3002\u65e0\u8bba\u662f\u5b58\u50a8\u6210\u672c\u8fd8\u662f\u5bf9\u8be5\u6570\u636e\u8fdb\u884c\u5206\u6790\uff0c\u6b64\u683c\u5f0f\u90fd\u4e0d\u751a\u7406\u60f3\u3002JindoDistCp \u53ef\u4ee5\u4f7f\u7528 --codec \u9009\u9879\u5e2e\u52a9\u60a8\u5728\u7ebf\u9ad8\u6548\u5730\u5b58\u50a8\u6570\u636e\u548c\u538b\u7f29\u6587\u4ef6\u3002 --codec \u6307\u5b9a\u6587\u4ef6\u538b\u7f29\u7f16\u89e3\u7801\u5668\u3002 \u652f\u6301 gzip\u3001gz\u3001lzo\u3001lzop \u548c snappy \u5173\u952e\u5b57\u3002 none \u4fdd\u5b58\u4e3a\u672a\u538b\u7f29\u7684\u6587\u4ef6\u3002\u5982\u679c\u6587\u4ef6\u5df2\u538b\u7f29\uff0c\u5219 JindoDistCp \u4f1a\u5c06\u5176\u89e3\u538b\u7f29\u3002 keep(\u9ed8\u8ba4\u503c) \u4e0d\u66f4\u6539\u6587\u4ef6\u538b\u7f29\u5f62\u6001\uff0c\u6309\u539f\u6837\u590d\u5236\u3002 \u547d\u4ee4\u793a\u4f8b\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --codec gz \u6211\u4eec\u68c0\u67e5\u4e00\u4e0b\u76ee\u6807\u6587\u4ef6\u5939\u4e2d\u7684\u6587\u4ef6\uff0c\u8fd9\u4e9b\u6587\u4ef6\u73b0\u5728\u5df2\u7ecf\u7528 gz \u7f16\u89e3\u7801\u5668\u538b\u7f29\u4e86\uff1a [root@emr-header-1 opt]# hdfs dfs -ls oss://example-oss-bucket/hourly_table/2017-02-01/03 Found 6 items -rw-rw-rw- 1 938 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/000151.sst.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/1.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/2.log.gz -rw-rw-rw- 1 1956 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/OPTIONS-000109.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/emp01.txt.gz -rw-rw-rw- 1 506 2020-04-17 20:58 oss://example-oss-bucket/hourly_table/2017-02-01/03/emp06.txt.gz \u5982\u60a8\u5728\u5f00\u6e90Hadoop\u96c6\u7fa4\u73af\u5883\u4e2d\u4f7f\u7528lzo\u538b\u7f29\u529f\u80fd\uff0c\u5219\u60a8\u9700\u8981\u5b89\u88c5 gplcompression \u7684 native \u5e93\u548chadoop-lzo\u5305\uff0c\u5982\u60a8\u7f3a\u5c11\u76f8\u5173\u73af\u5883\uff0c\u5efa\u8bae\u4f7f\u7528\u5176\u4ed6\u538b\u7f29\u65b9\u5f0f\u8fdb\u884c\u538b\u7f29\u3002","title":"3\u3001\u4f7f\u7528 --codec"},{"location":"jindotools/jindodistcp_quickstart/#4-filters","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --filters \u6307\u5b9a\u6709\u8fc7\u6ee4\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --filters filter.txt \u5047\u8bbe filter.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u4e0d\u4f1a\u5c06\u8def\u5f84\u4e2d\u5e26\u6709 test \u5b57\u7b26\u7684\u6587\u4ef6\u62f7\u8d1d\u5230OSS\u3002","title":"4\u3001\u4f7f\u7528 --filters"},{"location":"jindotools/jindodistcp_quickstart/#5-srcprefixesfile","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --srcPrefixesFile \u6307\u5b9a\u6709\u7b26\u5408\u89c4\u5219\u7684\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --srcPrefixesFile prefixes.txt \u5047\u8bbe prefixes.txt \u7684\u5185\u5bb9\u5982\u4e0b\uff1a .*test.* \u5219\u53ea\u6709\u8def\u5f84\u4e2d\u5e26\u6709 test \u5b57\u7b26\u7684\u6587\u4ef6\u624d\u4f1a\u62f7\u8d1d\u5230OSS\u3002","title":"5\u3001\u4f7f\u7528 --srcPrefixesFile"},{"location":"jindotools/jindodistcp_quickstart/#6-parallelism","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --parallelism \u6307\u5b9aMR\u4efb\u52a1\u91cc\u7684 mapreduce.job.reduces \u53c2\u6570\uff0c\u8be5\u53c2\u6570\u5728EMR\u73af\u5883\u4e2d\u9ed8\u8ba4\u4e3a7\uff0c\u60a8\u53ef\u4ee5\u6839\u636e\u96c6\u7fa4\u7684\u8d44\u6e90\u60c5\u51b5\u81ea\u5b9a\u4e49 parallelism \u7684\u5927\u5c0f\u6765\u63a7\u5236 distcp \u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /opt/tmp --dest oss://example-oss-bucket/tmp --parallelism 20","title":"6\u3001\u4f7f\u7528 --parallelism"},{"location":"jindotools/jindodistcp_quickstart/#7-taskbatch","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --taskBatch \u6307\u5b9a\u6bcf\u4e2a distcp task \u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --taskBatch 10","title":"7\u3001\u4f7f\u7528 --taskBatch"},{"location":"jindotools/jindodistcp_quickstart/#8-tmp","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --tmp \u6307\u5b9a\u4e00\u4e2a\u4e34\u65f6\u76ee\u5f55\u5728hdfs\u4e0a\u4f20\u5b58\u653e\u4e34\u65f6\u6570\u636e\uff0c\u9ed8\u8ba4\u503c\u4e3a/tmp\uff0c\u5373hdfs:///tmp/\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --tmp /tmp","title":"8\u3001\u4f7f\u7528 --tmp"},{"location":"jindotools/jindodistcp_quickstart/#9-ossoss-hdfs-ak","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 \u5728 EMR \u5916\u6216\u514d\u5bc6\u670d\u52a1\u51fa\u73b0\u95ee\u9898\u7684\u60c5\u51b5\u4e0b\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9aAK\u6765\u83b7\u5f97\u8bbf\u95ee OSS/OSS-HDFS \u670d\u52a1\u7684\u6743\u9650\u3002\u60a8\u53ef\u4ee5\u5728\u547d\u4ee4\u4e2d\u4f7f\u7528 --hadoopConf \u9009\u9879\u6765\u6307\u5b9a AK\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret \u60a8\u4e5f\u53ef\u4ee5\u5c06 OSS/OSS-HDFS \u670d\u52a1\u7684 ak\u3001secret \u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u907f\u514d\u6bcf\u6b21\u4f7f\u7528\u65f6\u4e34\u65f6\u586b\u5199 ak\u3002\u60a8\u53ef\u4ee5\u5728 E-MapReduce \u63a7\u5236\u53f0 Hadoop-Common \u670d\u52a1\u7684 core-site.xml \u9875\u9762\u914d\u7f6e\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration>","title":"9\u3001\u914d\u7f6e\u8bbf\u95ee OSS/OSS-HDFS \u7684 AK"},{"location":"jindotools/jindodistcp_quickstart/#10-disablechecksum","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --disableChecksum \u5173\u95ed\u68c0\u67e5\u6587\u4ef6 checksum\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --disableChecksum","title":"10\u3001\u4f7f\u7528 --disableChecksum"},{"location":"jindotools/jindodistcp_quickstart/#11-deleteonsuccess","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --deleteOnSuccess \u6307\u5b9a\u79fb\u52a8\u6570\u636e\u800c\u4e0d\u662f\u590d\u5236\u6570\u636e\u3002\u6b64\u9009\u9879\u7c7b\u4f3c\u4e8e mv \u64cd\u4f5c\uff0c\u9996\u5148\u590d\u5236\u6587\u4ef6\uff0c\u7136\u540e\u4ece\u6e90\u4f4d\u7f6e\u5220\u9664\u6587\u4ef6\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --deleteOnSuccess","title":"11\u3001\u4f7f\u7528 --deleteOnSuccess"},{"location":"jindotools/jindodistcp_quickstart/#12-enabletransaction","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u4e0d\u652f\u6301 \u652f\u6301 --enableTransaction JindoDistCp \u9ed8\u8ba4\u4f7f\u7528 Task \u7ea7\u522b\u5b8c\u6574\u6027\uff0c\u5982\u60a8\u9700\u8981\u4fdd\u8bc1 Job \u7ea7\u522b\u7684\u5b8c\u6574\u6027\u4ee5\u53ca\u4fdd\u8bc1 Job \u4e4b\u95f4\u7684\u4e8b\u52a1\u652f\u6301\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528\u8be5\u53c2\u6570\u6307\u5b9a\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --enableTransaction","title":"12\u3001\u4f7f\u7528 --enableTransaction"},{"location":"jindotools/jindodistcp_quickstart/#13-ignore","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --ignore \u5ffd\u7565\u6570\u636e\u8fc1\u79fb\u671f\u95f4\u53d1\u751f\u7684\u5f02\u5e38\uff0c\u76f8\u5173\u62a5\u9519\u4e0d\u4f1a\u4e2d\u65ad\u4efb\u52a1\uff0c\u5e76\u6700\u7ec8\u4ee5 JindoCounter \u7684\u5f62\u5f0f\u900f\u51fa\u3002\uff08\u5982\u679c\u5f00\u542fCMS\u7684\u8bdd\uff0c\u4e5f\u4f1a\u4ee5\u6307\u5b9a\u65b9\u5f0f\u8fdb\u884c\u901a\u77e5\u3002\uff09 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --ignore","title":"13\u3001\u4f7f\u7528 --ignore"},{"location":"jindotools/jindodistcp_quickstart/#14-diff","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --diff \u67e5\u770b src \u548c dest \u7684\u6587\u4ef6\u5dee\u5f02\u3002\u5982\u679c src \u7684\u6587\u4ef6\u672a\u80fd\u540c\u6b65\u5230 dest \u4e0a\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\u3002\u5982\u60a8\u7684 JindoDistCp \u4efb\u52a1\u5305\u542b\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u5219 --diff \u4e0d\u80fd\u663e\u793a\u6b63\u786e\u7684\u6587\u4ef6\u5dee\u5f02\uff0c\u56e0\u4e3a\u538b\u7f29\u6216\u8005\u89e3\u538b\u7f29\u4f1a\u6539\u53d8\u6587\u4ef6\u7684\u5927\u5c0f\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --diff \u5982\u679c\u6587\u4ef6\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5185\u5bb9\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter DIFF_FILES=1 \u5982\u679c\u60a8\u7684--dest\u4e3aHDFS\u8def\u5f84\uff0c\u73b0\u652f\u6301 /path \uff0c hdfs://hostname:ip/path \uff0c hdfs://headerIp:ip/path \u7684\u5199\u6cd5\uff0c\u6682\u4e0d\u652f\u6301 hdfs:///path \uff0c hdfs:/path \u548c\u5176\u4ed6\u81ea\u5b9a\u4e49\u5199\u6cd5\u3002 \u5982\u679c\u60f3\u67e5\u770b\u6587\u4ef6\u5143\u6570\u636e\u7684\u5dee\u5f02\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528 --diff --preserveMeta \u547d\u4ee4\uff1a \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --diff --preserveMeta \u5982\u679c\u6587\u4ef6\u5143\u6570\u636e\u6709\u5dee\u5f02\uff0c\u5219\u4f1a\u5728\u5f53\u524d\u76ee\u5f55\u4e0b\u751f\u6210\u4e00\u4e2a\u5305\u542b\u6587\u4ef6\u5143\u6570\u636e\u5dee\u5f02\u7684\u6587\u4ef6\uff0c\u5e76\u4f1a\u63d0\u793a\u5982\u4e0b\u4fe1\u606f\uff1a JindoCounter META_FILE_DIFF=1","title":"14\u3001\u4f7f\u7528 --diff"},{"location":"jindotools/jindodistcp_quickstart/#15-update","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --update \u6307\u5b9a\u589e\u91cf\u540c\u6b65\u529f\u80fd\uff0c\u8df3\u8fc7\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u548c\u76ee\u5f55\uff0c\u76f4\u63a5\u5c06 src \u4e2d\u65b0\u589e\u6216\u53d1\u751f\u6539\u53d8\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u540c\u6b65\u5230 dest \u4e0a\u3002 \u5982\u679c JindoDistCp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53ea Copy \u5269\u4e0b\u672a Copy \u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 JindoDistCp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u6307\u5b9a\u8be5\u53c2\u6570\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --update","title":"15\u3001\u4f7f\u7528 --update"},{"location":"jindotools/jindodistcp_quickstart/#16oss","text":"\u7248\u672c OSS OSS-HDFS 4.3.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u4e0d\u652f\u6301 --update \u6307\u5b9a\u4ee5\u51b7\u5f52\u6863\u3001\u5f52\u6863\u548c\u4f4e\u9891\u7684\u6a21\u5f0f\u5199\u5165 OSS\uff0c\u8fdb\u884c\u6570\u636e\u5b58\u50a8\u3002 1\uff09\u4f7f\u7528\u51b7\u5f52\u6863\u5199\u5165OSS\uff08coldArchive\uff09 \u76ee\u524d\u53ea\u5728\u90e8\u5206region\u53ef\u7528\uff0c\u5177\u4f53\u53c2\u89c1 OSS\u5b58\u50a8\u7c7b\u578b\u4ecb\u7ecd \uff0c\u4f7f\u7528\u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-bucket/hourly_table --policy coldArchive --parallelism 20 2\uff09\u4f7f\u7528\u5f52\u6863\u5199\u5165OSS\uff08archive\uff09 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-bucket/hourly_table --policy archive --parallelism 20 3\uff09\u4f7f\u7528\u4f4e\u9891\u5199\u5165OSS\uff08ia\uff09 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-bucket/hourly_table --policy ia --parallelism 20 \u5982\u60a8\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4\u4ee5 standard \u6807\u51c6\u6a21\u5f0f\u5199\u5165\uff0c\u4e0d\u8fdb\u884c\u51b7\u5f52\u6863\u3001\u5f52\u6863\u548c\u4f4e\u9891\u64cd\u4f5c\u3002","title":"16\u3001\u4f7f\u7528\u51b7\u5f52\u6863/\u5f52\u6863/\u4f4e\u9891\u5199\u5165OSS"},{"location":"jindotools/jindodistcp_quickstart/#17-preservemeta","text":"\u7248\u672c OSS OSS-HDFS 4.4.0 \u53ca\u4ee5\u4e0a\u7248\u672c \u4e0d\u652f\u6301 \u652f\u6301 --preserveMeta \u6307\u5b9a\u8fc1\u79fb\u6570\u636e\u7684\u540c\u65f6\u8fc1\u79fb\u5305\u62ec Owner, Group, Permission, Atime, Mtime, Replication, BlockSize, XAttrs, ACL \u5728\u5185\u7684\u5143\u6570\u636e\u4fe1\u606f\u3002 jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --preserveMeta","title":"17\u3001\u4f7f\u7528 --preserveMeta"},{"location":"jindotools/jindodistcp_quickstart/#18-jobbatch","text":"\u7248\u672c OSS OSS-HDFS 4.5.1 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --jobBatch \u5728\u60a8\u7684distcp\u4efb\u52a1\u5199\u5165OSS\u65f6\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7--jobBatch\u6765\u6307\u5b9a\u6bcf\u4e2adistcp job\u5904\u7406\u7684\u6587\u4ef6\u6570\u91cf\uff0c\u9ed8\u8ba4\u4e3a10000\u3002 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a jindo-distcp-tool-${version}.jar --src /data/hourly_table --dest oss://example-oss-bucket/hourly_table --jobBatch 50000","title":"18\u3001\u4f7f\u7528 --jobBatch"},{"location":"jindotools/jindodistcp_quickstart/#19-enablecms","text":"\u7248\u672c OSS OSS-HDFS 4.5.1 \u53ca\u4ee5\u4e0a\u7248\u672c \u652f\u6301 \u652f\u6301 --enableCMS \u5f00\u542f\u4e91\u76d1\u63a7\u544a\u8b66\u529f\u80fd\uff0c\u5177\u4f53\u53c2\u89c1 \u300aJindoDistcp\u4f7f\u7528CMS\u8fdb\u884c\u544a\u8b66\u300b \u3002","title":"19\u3001 \u4f7f\u7528 --enableCMS"},{"location":"jindotools/jindodistcp_quickstart/#jindodistcp-counters","text":"\u4efb\u52a1\u8ba1\u6570\u5668 \u8bf4\u660e COPY_FAILED copy\u5931\u8d25\u7684\u6587\u4ef6\u6570 CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165COPY_FAILED FILES_EXPECTED \u9884\u671f\u7684copy\u6587\u4ef6\u6570\u91cf BYTES_EXPECTED \u9884\u671f\u7684copy\u5b57\u8282\u6570 FILES_COPIED copy\u6210\u529f\u7684\u6587\u4ef6\u6570 BYTES_COPIED copy\u6210\u529f\u7684\u5b57\u8282\u6570 FILES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u6587\u4ef6\u6570 BYTES_SKIPPED update\u589e\u91cf\u66f4\u65b0\u65f6\u8df3\u8fc7\u7684\u5b57\u8282\u6570 DIFF_FILES \u4e0d\u76f8\u540c\u7684\u6587\u4ef6\u6570 SAME_FILES \u7ecf\u6821\u9a8c\u5b8c\u5168\u76f8\u540c\u7684\u6587\u4ef6\u6570 DST_MISS \u76ee\u6807\u8def\u5f84\u4e0d\u5b58\u5728\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES LENGTH_DIFF \u6e90\u6587\u4ef6\u548c\u76ee\u6807\u6587\u4ef6\u5927\u5c0f\u4e0d\u4e00\u81f4\u7684\u6570\u91cf\uff0c\u5e76\u8ba1\u5165DIFF_FILES CHECKSUM_DIFF checksum\u6821\u9a8c\u5931\u8d25\u7684\u6587\u4ef6\u6570\uff0c\u5e76\u8ba1\u5165DIFF_FILES DIFF_FAILED diff\u64cd\u4f5c\u5f02\u5e38\u7684\u6587\u4ef6\u6570\uff0c\u5177\u4f53\u62a5\u9519\u53c2\u89c1job\u65e5\u5fd7","title":"JindoDistCp Counters \u8bf4\u660e"},{"location":"jindotools/jindodistcp_s3_to_oss/","text":"\u6570\u636e\u4ece\u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u8fc1\u79fb\u5230 OSS/OSS-HDFS \u00a7 \u4f7f\u7528\u524d\u987b\u77e5 \u00a7 \u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d\u3002 \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\u3002 1\u3001\u62f7\u8d1d\u6570\u636e\u5230\u963f\u91cc\u4e91 OSS/OSS-HDFS \u4e0a \u00a7 \u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06\u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS/OSS-HDFS \u4e0a hadoop jar jindo-distcp-tool-${version}.jar s3://example-s3-bucket/data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src \u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u8def\u5f84\u3002 s3://example-s3-bucket/data/ --dest OSS/OSS-HDFS \u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/ --hadoopConf \u6307\u5b9a OSS/OSS-HDFS/S3/COS/OBS \u7684 Access Key ID , Access Key Secret * \u914d\u7f6e OSS/OSS-HDFS \u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS/OSS-HDFS \u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10 2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6 \u00a7 \u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53eaCopy\u5269\u4e0b\u672aCopy\u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a \u4f7f\u7528 --update \u547d\u4ee4\uff0c\u83b7\u5f97\u589e\u91cf\u7684\u6587\u4ef6\u5217\u8868 \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src s3://example-s3-bucket/data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --update --parallelism 20 3\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9 \u00a7 \u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src s3://example-s3-bucket/data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB 4\u3001\u914d\u7f6e\u8bbf\u95ee\u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u7684 AK \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9a AK \u6765\u83b7\u5f97\u8bbf\u95ee\u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u7684\u6743\u9650\u3002\u60a8\u53ef\u4ee5\u5728\u547d\u4ee4\u4e2d\u4f7f\u7528 --hadoopConf \u9009\u9879\u6765\u6307\u5b9a AK\u3002 S3 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src s3://example-s3-bucket/data/hourly_table --dest oss://example-oss-bucket/hourly_table --hadoopConf fs.s3.accessKeyId=yourkey --hadoopConf fs.s3.accessKeySecret=yoursecret COS \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src cos://example-cos-bucket/data/hourly_table --dest oss://example-oss-bucket/hourly_table --hadoopConf fs.cos.accessKeyId=yourkey --hadoopConf fs.cos.accessKeySecret=yoursecret OBS \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src obs://example-obs-bucket/data/hourly_table --dest oss://example-oss-bucket/hourly_table --hadoopConf fs.obs.accessKeyId=yourkey --hadoopConf fs.obs.accessKeySecret=yoursecret \u60a8\u4e5f\u53ef\u4ee5\u5c06\u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u7684 AK \u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u907f\u514d\u6bcf\u6b21\u4f7f\u7528\u65f6\u4e34\u65f6\u586b\u5199 AK, \u4ee5 S3 \u4e3a\u4f8b\u3002 <configuration> <property> <name>fs.s3.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.s3.accessKeySecret</name> <value>xxx</value> </property> </configuration> 5\u3001\u914d\u7f6e OSS/OSS-HDFS \u7684 AK \u00a7 \u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS/OSS-HDFS AccessKey/AccessSecret \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662f JindoDistCp \u53ef\u4ee5\u5c06\u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration> 6\u3001\u914d\u7f6e OSS-HDFS Endpoint \u00a7 \u8bbf\u95ee OSS-HDFS \u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS/OSS-HDFS Endpoint \u914d\u7f6e \u3002 \u6ce8\u610f \uff1a JindoSDK 4.4.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u9ed8\u8ba4\u4f7f\u7528\u57df\u540d\u5206\u79bb\u7684\u65b9\u5f0f\uff0c\u6570\u636e\u8bfb\u5199\u57df\u540d\u9ed8\u8ba4\u4f7f\u7528\u5185\u7f51\u6807\u51c6 OSS \u57df\u540d\uff0c\u5982\u679c\u5728\u975e\u963f\u91cc\u4e91\u5185\u7f51\u73af\u5883\u6267\u884c distcp \u547d\u4ee4\uff0c\u5219\u9700\u8981\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u914d\u7f6e\u6570\u636e\u6d41\u8bbf\u95ee\u7684\u516c\u7f51 OSS endpoint\u3002 <configuration> <property> <name>fs.oss.data.endpoint</name> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration> 7\u3001\u5176\u4ed6\u529f\u80fd \u00a7 \u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003: * JindoDistCp \u4f7f\u7528\u8bf4\u660e","title":"\u4ece S3 \u8fc1\u79fb\u5230 OSS \u4e0a"},{"location":"jindotools/jindodistcp_s3_to_oss/#ossoss-hdfs","text":"","title":"\u6570\u636e\u4ece\u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u8fc1\u79fb\u5230 OSS/OSS-HDFS"},{"location":"jindotools/jindodistcp_s3_to_oss/#_1","text":"\u8bf7\u53c2\u8003 JindoDistCp \u4ecb\u7ecd \u6587\u7ae0\u5185\u5bb9\u8fdb\u884c\u73af\u5883\u9002\u914d\u548c\u5de5\u5177\u5305\u4e0b\u8f7d\u3002 \u5982\u60a8\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u95ee\u9898\u53ef\u53c2\u8003 JindoDistCp \u95ee\u9898\u6392\u67e5\u6307\u5357 \u8fdb\u884c\u89e3\u51b3\u3002","title":"\u4f7f\u7528\u524d\u987b\u77e5"},{"location":"jindotools/jindodistcp_s3_to_oss/#1-ossoss-hdfs","text":"\u60a8\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\u5c06\u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u4e0a\u7684\u76ee\u5f55\u62f7\u8d1d\u5230 OSS/OSS-HDFS \u4e0a hadoop jar jindo-distcp-tool-${version}.jar s3://example-s3-bucket/data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism 10 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b --src \u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u8def\u5f84\u3002 s3://example-s3-bucket/data/ --dest OSS/OSS-HDFS \u7684\u76ee\u6807\u8def\u5f84\u3002 oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/ --hadoopConf \u6307\u5b9a OSS/OSS-HDFS/S3/COS/OBS \u7684 Access Key ID , Access Key Secret * \u914d\u7f6e OSS/OSS-HDFS \u7684 AccessKeyId: --hadoopConf fs.oss.accessKeyId=yourkey * \u914d\u7f6e OSS/OSS-HDFS \u7684 AccessKeySecret: --hadoopConf fs.oss.accessKeySecret=yoursecret --parallelism \u4efb\u52a1\u5e76\u53d1\u5927\u5c0f\uff0c\u6839\u636e\u96c6\u7fa4\u8d44\u6e90\u53ef\u8c03\u6574\u3002 10","title":"1\u3001\u62f7\u8d1d\u6570\u636e\u5230\u963f\u91cc\u4e91 OSS/OSS-HDFS \u4e0a"},{"location":"jindotools/jindodistcp_s3_to_oss/#2","text":"\u5982\u679c Distcp \u4efb\u52a1\u56e0\u4e3a\u5404\u79cd\u539f\u56e0\u4e2d\u95f4\u5931\u8d25\u4e86\uff0c\u800c\u6b64\u65f6\u60a8\u60f3\u8fdb\u884c\u65ad\u70b9\u7eed\u4f20\uff0c\u53eaCopy\u5269\u4e0b\u672aCopy\u6210\u529f\u7684\u6587\u4ef6\u3002\u6216\u8005\u6e90\u7aef\u6587\u4ef6\u65b0\u589e\u4e86\u90e8\u5206\u6587\u4ef6\uff0c\u6b64\u65f6\u9700\u8981\u60a8\u5728\u8fdb\u884c\u4e0a\u4e00\u6b21 Distcp \u4efb\u52a1\u5b8c\u6210\u540e\u8fdb\u884c\u5982\u4e0b\u64cd\u4f5c\uff1a","title":"2\u3001\u589e\u91cf\u62f7\u8d1d\u6587\u4ef6"},{"location":"jindotools/jindodistcp_s3_to_oss/#-update","text":"hadoop jar jindo-distcp-tool-${version}.jar --src s3://example-s3-bucket/data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --update --parallelism 20","title":"\u4f7f\u7528 --update \u547d\u4ee4\uff0c\u83b7\u5f97\u589e\u91cf\u7684\u6587\u4ef6\u5217\u8868"},{"location":"jindotools/jindodistcp_s3_to_oss/#3yarn","text":"\u5982\u60a8\u9700\u8981\u5bf9 DistCp \u4f5c\u4e1a\u4f7f\u7528\u7684 YARN \u961f\u5217\u548c\u5e26\u5bbd\u8fdb\u884c\u9650\u5b9a\uff0c\u53ef\u7528\u5982\u4e0b\u547d\u4ee4 hadoop jar jindo-distcp-tool-${version}.jar --src s3://example-s3-bucket/data --dest oss://destBucket.cn-xxx.oss-dls.aliyuncs.com/dir/ --hadoopConf fs.oss.accessKeyId=yourkey --hadoopConf fs.oss.accessKeySecret=yoursecret --hadoopConf mapreduce.job.queuename=yarnQueue --bandWidth 100 --parallelism 10 --hadoopConf mapreduce.job.queuename=yarnQueue\uff1a\u6307\u5b9a YARN \u961f\u5217\u7684\u540d\u79f0 --bandWidth\uff1a\u6307\u5b9a\u5355\u673a\u9650\u6d41\u5e26\u5bbd\u7684\u5927\u5c0f\uff0c\u5355\u4f4d MB","title":"3\u3001YARN \u961f\u5217\u53ca\u5e26\u5bbd\u9009\u62e9"},{"location":"jindotools/jindodistcp_s3_to_oss/#4-ak","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9a AK \u6765\u83b7\u5f97\u8bbf\u95ee\u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u7684\u6743\u9650\u3002\u60a8\u53ef\u4ee5\u5728\u547d\u4ee4\u4e2d\u4f7f\u7528 --hadoopConf \u9009\u9879\u6765\u6307\u5b9a AK\u3002 S3 \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src s3://example-s3-bucket/data/hourly_table --dest oss://example-oss-bucket/hourly_table --hadoopConf fs.s3.accessKeyId=yourkey --hadoopConf fs.s3.accessKeySecret=yoursecret COS \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src cos://example-cos-bucket/data/hourly_table --dest oss://example-oss-bucket/hourly_table --hadoopConf fs.cos.accessKeyId=yourkey --hadoopConf fs.cos.accessKeySecret=yoursecret OBS \u793a\u4f8b\u547d\u4ee4\u5982\u4e0b\uff1a hadoop jar jindo-distcp-tool-${version}.jar --src obs://example-obs-bucket/data/hourly_table --dest oss://example-oss-bucket/hourly_table --hadoopConf fs.obs.accessKeyId=yourkey --hadoopConf fs.obs.accessKeySecret=yoursecret \u60a8\u4e5f\u53ef\u4ee5\u5c06\u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u7684 AK \u9884\u5148\u914d\u7f6e\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u907f\u514d\u6bcf\u6b21\u4f7f\u7528\u65f6\u4e34\u65f6\u586b\u5199 AK, \u4ee5 S3 \u4e3a\u4f8b\u3002 <configuration> <property> <name>fs.s3.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.s3.accessKeySecret</name> <value>xxx</value> </property> </configuration>","title":"4\u3001\u914d\u7f6e\u8bbf\u95ee\u4e09\u65b9\u4e91\u5bf9\u8c61\u5b58\u50a8\u7684 AK"},{"location":"jindotools/jindodistcp_s3_to_oss/#5-ossoss-hdfs-ak","text":"\u901a\u5e38\u60a8\u9700\u8981\u5c06 OSS/OSS-HDFS AccessKey/AccessSecret \u4fe1\u606f\u5199\u5728\u53c2\u6570\u91cc\uff0c\u4f46\u662f JindoDistCp \u53ef\u4ee5\u5c06\u5176\u9884\u5148\u5199\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u91cc \uff0c\u4ee5\u907f\u514d\u4f7f\u7528\u65f6\u591a\u6b21\u586b\u5199\u7684\u95ee\u9898\u3002 <configuration> <property> <name>fs.oss.accessKeyId</name> <value>xxx</value> </property> <property> <name>fs.oss.accessKeySecret</name> <value>xxx</value> </property> </configuration>","title":"5\u3001\u914d\u7f6e OSS/OSS-HDFS \u7684 AK"},{"location":"jindotools/jindodistcp_s3_to_oss/#6-oss-hdfs-endpoint","text":"\u8bbf\u95ee OSS-HDFS \u9700\u8981\u914d\u7f6e Endpoint\uff08 cn-xxx.oss-dls.aliyuncs.com \uff09\uff0c\u4e0e OSS \u5bf9\u8c61\u63a5\u53e3\u7684 Endpoint\uff08 oss-cn-xxx-internal.aliyuncs.com \uff09\u4e0d\u540c\u3002JindoSDK \u4f1a\u6839\u636e\u914d\u7f6e\u7684 Endpoint \u8bbf\u95ee OSS-HDFS \u6216 OSS \u5bf9\u8c61\u63a5\u53e3\u3002 \u4f7f\u7528 OSS-HDFS \u65f6\uff0c\u63a8\u8350\u8bbf\u95ee\u8def\u5f84\u683c\u5f0f\u4e3a\uff1a oss://<Bucket>.<Endpoint>/<Object> \u5982: oss://mydlsbucket.cn-shanghai.oss-dls.aliyuncs.com/Test \u3002 \u8fd9\u79cd\u65b9\u5f0f\u5728\u8bbf\u95ee\u8def\u5f84\u4e2d\u5305\u542b OSS-HDFS \u7684 Endpoint\uff0cJindoSDK \u4f1a\u6839\u636e\u8def\u5f84\u4e2d\u7684 Endpoint \u8bbf\u95ee\u5bf9\u5e94\u7684 OSS-HDFS \u63a5\u53e3\u3002 JindoSDK \u8fd8\u652f\u6301\u66f4\u591a\u7684 Endpoint \u914d\u7f6e\u65b9\u5f0f\uff0c\u8be6\u60c5\u53c2\u8003 OSS/OSS-HDFS Endpoint \u914d\u7f6e \u3002 \u6ce8\u610f \uff1a JindoSDK 4.4.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u9ed8\u8ba4\u4f7f\u7528\u57df\u540d\u5206\u79bb\u7684\u65b9\u5f0f\uff0c\u6570\u636e\u8bfb\u5199\u57df\u540d\u9ed8\u8ba4\u4f7f\u7528\u5185\u7f51\u6807\u51c6 OSS \u57df\u540d\uff0c\u5982\u679c\u5728\u975e\u963f\u91cc\u4e91\u5185\u7f51\u73af\u5883\u6267\u884c distcp \u547d\u4ee4\uff0c\u5219\u9700\u8981\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u914d\u7f6e\u6570\u636e\u6d41\u8bbf\u95ee\u7684\u516c\u7f51 OSS endpoint\u3002 <configuration> <property> <name>fs.oss.data.endpoint</name> <value>oss-cn-xxx.aliyuncs.com</value> </property> </configuration>","title":"6\u3001\u914d\u7f6e OSS-HDFS Endpoint"},{"location":"jindotools/jindodistcp_s3_to_oss/#7","text":"\u5982\u60a8\u9700\u8981\u5176\u4ed6\u4f7f\u7528\u5176\u4ed6\u529f\u80fd\uff0c\u8bf7\u53c2\u8003: * JindoDistCp \u4f7f\u7528\u8bf4\u660e","title":"7\u3001\u5176\u4ed6\u529f\u80fd"},{"location":"jindotools/jindotable_moveto/","text":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS/OSS-HDFS \u00a7 HDFS \u53d7\u9650\u4e8e\u96c6\u7fa4\u89c4\u6a21\u548c\u6210\u672c\u56e0\u7d20\uff0c\u65e0\u6cd5\u65e0\u9650\u6269\u5c55\uff0c\u5bb9\u91cf\u5b58\u5728\u74f6\u9888\u3002\u4e91\u5382\u5546\u63d0\u4f9b\u7684 OSS/OSS-HDFS \u53ef\u4ee5\u4f5c\u4e3a HDFS \u7684\u66ff\u4ee3\u6216\u8865\u5145\uff0c\u6269\u5c55\u4e91\u4e0a Hadoop \u5e73\u53f0\u7684\u5b58\u50a8\u80fd\u529b\u3002 \u672c\u5de5\u5177\u7528\u4e8e\u628a Hive \u6570\u636e\u6839\u636e\u5206\u533a\u952e\u89c4\u5219\u7b5b\u9009\uff0c\u5728 HDFS \u548c OSS/OSS-HDFS \u4e4b\u95f4\u8f6c\u79fb\u5206\u533a\u3002 \u524d\u63d0\u6761\u4ef6 \u00a7 \u5df2\u90e8\u7f72 JindoSDK \u00a7 EMR \u73af\u5883\u4e2d\uff0c\u9ed8\u8ba4\u5df2\u5b89\u88c5 JindoSDK\uff0c\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528\u3002\u6ce8\u610f\uff1a \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u521b\u5efaEMR-3.42.0\u53ca\u4ee5\u4e0a\u7248\u672c\u6216EMR-5.8.0\u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u96c6\u7fa4\u3002 \u975e EMR \u73af\u5883\uff0c\u8bf7\u5148\u5b89\u88c5\u90e8\u7f72 JindoSDK\u3002\u90e8\u7f72\u65b9\u5f0f\u8bf7\u53c2\u8003 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u90e8\u7f72 JindoSDK 4.x \u53ca\u4ee5\u4e0a\u7248\u672c\u3002 \u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883 \u00a7 \u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e \u914d\u7f6e MoveTo \u5de5\u5177\u5728 HDFS \u4e0b\u7684\u9501\u76ee\u5f55\uff1a \u00a7 \u5728 Hadoop \u914d\u7f6e\u6587\u4ef6 core-site.xml \u6216 hdfs-site.xml \uff08\u4efb\u4e00\u5373\u53ef\uff0c\u5728 $HADOOP_CONF_DIR \u76ee\u5f55\u4e0b\uff09\u65b0\u589e\u914d\u7f6e\u9879\uff1a jindotable.moveto.tablelock.base.dir \u8be5\u914d\u7f6e\u7684\u503c\u5e94\u6307\u5411\u4e00\u4e2a HDFS \u76ee\u5f55\uff0c\u76ee\u7684\u662f\u5b58\u653e MoveTo \u5de5\u5177\u5728\u8fd0\u884c\u65f6\u81ea\u52a8\u521b\u5efa\u7684\u9501\u6587\u4ef6\u3002\u9700\u786e\u4fdd\u8be5\u76ee\u5f55\u53ea\u4f1a\u88ab MoveTo \u5de5\u5177\u8bbf\u95ee\uff0c\u5e76\u4e14\u6709\u8bbf\u95ee\u6743\u9650\u3002\u5982\u679c\u4e0d\u914d\u7f6e\uff0c\u5219\u4f7f\u7528\u7f3a\u7701\u503c hdfs:///tmp/jindotable-lock/ \uff0c\u65e0\u6743\u9650\u5219\u62a5\u9519\u3002 \u4f7f\u7528\u8bf4\u660e \u00a7 \u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f \u00a7 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help moveTo \u53c2\u6570\u8bf4\u660e \u00a7 jindotable -moveTo -t <dbName.tableName> -d <destination path> [-c \"<condition>\" | -fullTable] [-b/-before <before days>] [-p/-parallel <parallelism>] [-s/-storagePolicy <OSS storage policy>] [-o/-overWrite] [-r/-removeSource] [-skipTrash] [-e/-explain] [-q/-queue <yarn queue>] [-w/-workingDir <working directory>][-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u8981\u79fb\u52a8\u7684\u8868\u3002 \u662f -d <destination path> \u76ee\u6807\u8def\u5f84\uff0c\u4e3a\u8868\u7ea7\u522b\u7684\u8def\u5f84\uff0c\u5206\u533a\u8def\u5f84\u4f1a\u5728\u8fd9\u4e2a\u8def\u5f84\u4e0b\u81ea\u52a8\u521b\u5efa\u3002 \u662f -c \"<condition>\" / -fullTable \u5206\u533a\u8fc7\u6ee4\u6761\u4ef6\u8868\u8fbe\u5f0f\uff0c\u652f\u6301\u57fa\u672c\u8fd0\u7b97\u7b26\uff0c\u4e0d\u652f\u6301udf\u3002 \u5426 -b/before <before days> \u6839\u636e\u5206\u533a\u521b\u5efa\u65f6\u95f4\uff0c\u521b\u5efa\u65f6\u95f4\u8d85\u8fc7\u7ed9\u5b9a\u5929\u6570\u7684\u5206\u533a\u624d\u8fdb\u884c\u79fb\u52a8\u3002 \u5426 -p/-parallel <parallelism> \u6574\u4e2amoveTo\u4efb\u52a1\u7684\u6700\u5927task\u5e76\u53d1\u5ea6\uff0c\u9ed8\u8ba4\u4e3a1\u3002 \u5426 -s/-storagePolicy <OSS storage policy> \u62f7\u8d1d\u81f3 OSS \u65f6\u6570\u636e\u6587\u4ef6\u7684\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301 Standard\uff08\u9ed8\u8ba4\uff09\u3001IA\u3001Archive\u3001ColdArchive\u3002 \u5426 -o/-overWrite \u662f\u5426\u8986\u76d6\u6700\u7ec8\u76ee\u5f55\u3002\u5206\u533a\u7ea7\u522b\u8986\u76d6\uff0c\u4e0d\u4f1a\u8986\u76d6\u672c\u6b21\u79fb\u52a8\u4e0d\u6d89\u53ca\u7684\u5206\u533a\u3002 \u5426 -r/-removeSource \u662f\u5426\u5728\u79fb\u52a8\u5b8c\u6210\u540e\u5220\u9664\u6e90\u8def\u5f84\u3002 \u5426 -skipTrash \u5982\u679c\u5220\u9664\u6e90\u8def\u5f84\uff0c\u662f\u5426\u8df3\u8fc7Trash\u3002 \u5426 -e/-explain \u5982\u679c\u6307\u5b9aexplain\u6a21\u5f0f\uff0c\u4e0d\u4f1a\u89e6\u53d1\u5b9e\u9645\u64cd\u4f5c\uff0c\u4ec5\u6253\u5370\u4f1a\u540c\u6b65\u7684\u5206\u533a\u3002 \u5426 -q/-queue <yarn queue \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684yarn\u961f\u5217\u3002 \u5426 -w/-workingDir \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684\u5de5\u4f5c\u4e34\u65f6\u76ee\u5f55\u3002 \u5426 -iceberg \u6307\u5b9a\u6240\u8fc1\u79fb\u8868\u4e3aIceberg\u3002\u76ee\u524d\u4ec5\u652f\u6301Iceberg\u6574\u8868\u8fc1\u79fb\uff0c\u9700\u914d\u5408 -fullTable \u4f7f\u7528\u3002(6.3.0\u4ee5\u4e0a\u7248\u672c) \u5426 -l/-logDir <log directory> \u672c\u5730\u65e5\u5fd7\u76ee\u5f55\uff0c\u9ed8\u8ba4\u4e3a /tmp/<current user>/ \u5426 \u4f7f\u7528\u793a\u4f8b \u00a7 \u6709\u4e00\u4e2a HDFS \u4e0a\u7684 Hive \u5206\u533a\u8868\uff0c\u5982\u4e0b\u56fe\u6240\u793a\uff1a \u60f3\u628a bbb \u548c ccc \u5206\u533a\u79fb\u52a8\u5230 OSS\uff0c\u5148\u7528 explain \u6a21\u5f0f\u770b\u770b\u4f1a\u79fb\u52a8\u7684\u5206\u533a\u662f\u5426\u7b26\u5408\u9884\u671f\uff0c\u53c2\u6570\u4e3a -e \u6216 -explain\uff1a \u53bb\u6389\u53c2\u6570 -e\uff0c\u771f\u6b63\u79fb\u52a8\u5206\u533a\uff1a \u6267\u884c\u5b8c\u6210\u540e\uff0c\u68c0\u67e5\u6570\u636e\uff0c\u6570\u636e\u5df2\u7ecf\u5728 OSS\uff1a \u518d\u79fb\u56de HDFS\uff0c\u7ed3\u679c\u5931\u8d25\u4e86\uff0c\u4f1a\u663e\u793a\u5931\u8d25\u539f\u56e0\uff0c\u539f\u6765\u662f\u76ee\u6807\u76ee\u5f55\u975e\u7a7a\uff0c\u63d0\u793a\u53ef\u4ee5\u4f7f\u7528 -overWrite \u6e05\u7a7a\u76ee\u6807\u76ee\u5f55\uff1a \u4f7f\u7528 -overWrite \u5f3a\u5236\u8986\u76d6\uff1a \u5f02\u5e38\u72b6\u6001\u5904\u7406 \u00a7 \u4e3a\u4e86\u5c3d\u53ef\u80fd\u4fdd\u8bc1\u6570\u636e\u5b89\u5168\uff0c\u9632\u6b62\u6c61\u67d3\uff0c\u8be5\u547d\u4ee4\u4f1a\u81ea\u52a8\u68c0\u67e5\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u786e\u4fdd\u4e0d\u5b58\u5728\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u76ee\u5f55\u62f7\u8d1d\u3002 \u5982\u679c\u65e0\u6cd5\u6392\u9664\uff0c\u5219\u8be5\u8868\u6216\u5206\u533a\u7684\u62f7\u8d1d\u547d\u4ee4\u5c06\u6267\u884c\u5931\u8d25\u3002 \u6b64\u65f6\uff0c\u9700\u8981\u4e3b\u52a8\u5e72\u9884\uff0c\u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u6b63\u5728\u6267\u884c\u62f7\u8d1d\u4e4b\u540e\uff0c\u6e05\u7406\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u5219\u53ef\u4ee5\u91cd\u65b0\u6267\u884c\u8be5\u547d\u4ee4\u3002 \u5bf9\u4e8e\u975e\u5206\u533a\u8868\u7684\u62f7\u8d1d\u6216\u8fc1\u79fb\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u5373\u4e3a\u8868\u7ea7\u522b\u76ee\u5f55\uff1b\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u4e3a\u5f85\u62f7\u8d1d\u6216\u79fb\u52a8\u5206\u533a\u7684\u5206\u533a\u7ea7\u76ee\u5f55\uff0c \u53ea\u9700\u5bf9\u5f85\u62f7\u8d1d\u7684\u5206\u533a\u8fdb\u884c\u6e05\u7406\u3002 \u51fa\u73b0\u539f\u56e0 \u00a7 \u5982\u679c\u8be5\u547d\u4ee4\u88ab\u5f02\u5e38\u4e2d\u6b62\uff0c\u5219\u53ef\u80fd\u51fa\u73b0\u9700\u8981\u4e3b\u52a8\u5e72\u9884\u7684\u60c5\u51b5\u3002\u6b64\u65f6\u62f7\u8d1d\u8fd8\u672a\u5b8c\u6210\uff0c\u6e90\u6570\u636e\u4e0e\u8868\u7684\u5143\u4fe1\u606f\u5747\u672a\u6539\u53d8\uff0c \u6570\u636e\u4ecd\u5904\u4e8e\u5b89\u5168\u72b6\u6001\u3002\u5e38\u89c1\u7684\u5f02\u5e38\u4e2d\u6b62\u6709\u4e24\u79cd\u60c5\u51b5\uff1a * \u7528\u6237\u5728\u547d\u4ee4\u5c1a\u672a\u7ed3\u675f\u65f6\uff0c\u4e3b\u52a8\u6740\u6389\u547d\u4ee4\u8fdb\u7a0b * \u7531\u4e8e\u5185\u5b58\u6ea2\u51fa\u7b49\u5f02\u5e38\uff0c\u8fdb\u7a0b\u5f02\u5e38\u4e2d\u6b62 \u5982\u679c\u62f7\u8d1d\u8868\u6216\u5206\u533a\u65f6\u62f7\u8d1d\u5931\u8d25\u5e76\u663e\u793a Conflicts found \uff0c\u5219\u5c5e\u4e8e\u5f02\u5e38\u72b6\u6001\uff0c\u9700\u4e3b\u52a8\u5e72\u9884\u3002 \u5904\u7406\u65b9\u6cd5 \u00a7 \u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u7684\u76ee\u6807\u8def\u5f84\u62f7\u8d1d\u6570\u636e\u3002\u7279\u522b\u662f\uff0c\u786e\u4fdd\u6ca1\u6709 DistCp/JindoDistCp \u7b49\u5206\u5e03\u5f0f\u62f7\u8d1d\u547d\u4ee4\u3002 \u5220\u9664\u76ee\u7684\u5730\u76ee\u5f55\u3002\u5bf9\u4e8e\u975e\u5206\u533a\u8868\uff0c\u5220\u9664\u8868\u4e00\u7ea7\u76ee\u5f55\u3002\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u5220\u9664\u5b58\u5728\u51b2\u7a81\u7684\u5206\u533a\u7ea7\u76ee\u5f55\u3002\u5207\u52ff\u5220\u9664\u6e90\u76ee\u5f55\u3002","title":"\u4f7f\u7528 JindoTable \u5c06 Hive \u6570\u636e\u5206\u533a\u5f52\u6863\u548c\u5206\u5c42\u5230OSS"},{"location":"jindotools/jindotable_moveto/#jindotable-hive-ossoss-hdfs","text":"HDFS \u53d7\u9650\u4e8e\u96c6\u7fa4\u89c4\u6a21\u548c\u6210\u672c\u56e0\u7d20\uff0c\u65e0\u6cd5\u65e0\u9650\u6269\u5c55\uff0c\u5bb9\u91cf\u5b58\u5728\u74f6\u9888\u3002\u4e91\u5382\u5546\u63d0\u4f9b\u7684 OSS/OSS-HDFS \u53ef\u4ee5\u4f5c\u4e3a HDFS \u7684\u66ff\u4ee3\u6216\u8865\u5145\uff0c\u6269\u5c55\u4e91\u4e0a Hadoop \u5e73\u53f0\u7684\u5b58\u50a8\u80fd\u529b\u3002 \u672c\u5de5\u5177\u7528\u4e8e\u628a Hive \u6570\u636e\u6839\u636e\u5206\u533a\u952e\u89c4\u5219\u7b5b\u9009\uff0c\u5728 HDFS \u548c OSS/OSS-HDFS \u4e4b\u95f4\u8f6c\u79fb\u5206\u533a\u3002","title":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fc1\u79fb\u5230 OSS/OSS-HDFS"},{"location":"jindotools/jindotable_moveto/#_1","text":"","title":"\u524d\u63d0\u6761\u4ef6"},{"location":"jindotools/jindotable_moveto/#jindosdk","text":"EMR \u73af\u5883\u4e2d\uff0c\u9ed8\u8ba4\u5df2\u5b89\u88c5 JindoSDK\uff0c\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528\u3002\u6ce8\u610f\uff1a \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u521b\u5efaEMR-3.42.0\u53ca\u4ee5\u4e0a\u7248\u672c\u6216EMR-5.8.0\u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u96c6\u7fa4\u3002 \u975e EMR \u73af\u5883\uff0c\u8bf7\u5148\u5b89\u88c5\u90e8\u7f72 JindoSDK\u3002\u90e8\u7f72\u65b9\u5f0f\u8bf7\u53c2\u8003 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u90e8\u7f72 JindoSDK 4.x \u53ca\u4ee5\u4e0a\u7248\u672c\u3002","title":"\u5df2\u90e8\u7f72 JindoSDK"},{"location":"jindotools/jindotable_moveto/#hadoop-hive","text":"\u786e\u4fdd hadoop classpath \u80fd\u591f\u8fd4\u56de\u5408\u7406\u7ed3\u679c \u786e\u4fdd\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf $HIVE_HOME \u4e0e $HIVE_CONF_DIR \u6b63\u786e\u914d\u7f6e","title":"\u5df2\u90e8\u7f72 Hadoop \u4e0e Hive \u73af\u5883"},{"location":"jindotools/jindotable_moveto/#moveto-hdfs","text":"\u5728 Hadoop \u914d\u7f6e\u6587\u4ef6 core-site.xml \u6216 hdfs-site.xml \uff08\u4efb\u4e00\u5373\u53ef\uff0c\u5728 $HADOOP_CONF_DIR \u76ee\u5f55\u4e0b\uff09\u65b0\u589e\u914d\u7f6e\u9879\uff1a jindotable.moveto.tablelock.base.dir \u8be5\u914d\u7f6e\u7684\u503c\u5e94\u6307\u5411\u4e00\u4e2a HDFS \u76ee\u5f55\uff0c\u76ee\u7684\u662f\u5b58\u653e MoveTo \u5de5\u5177\u5728\u8fd0\u884c\u65f6\u81ea\u52a8\u521b\u5efa\u7684\u9501\u6587\u4ef6\u3002\u9700\u786e\u4fdd\u8be5\u76ee\u5f55\u53ea\u4f1a\u88ab MoveTo \u5de5\u5177\u8bbf\u95ee\uff0c\u5e76\u4e14\u6709\u8bbf\u95ee\u6743\u9650\u3002\u5982\u679c\u4e0d\u914d\u7f6e\uff0c\u5219\u4f7f\u7528\u7f3a\u7701\u503c hdfs:///tmp/jindotable-lock/ \uff0c\u65e0\u6743\u9650\u5219\u62a5\u9519\u3002","title":"\u914d\u7f6e MoveTo \u5de5\u5177\u5728 HDFS \u4e0b\u7684\u9501\u76ee\u5f55\uff1a"},{"location":"jindotools/jindotable_moveto/#_2","text":"","title":"\u4f7f\u7528\u8bf4\u660e"},{"location":"jindotools/jindotable_moveto/#_3","text":"\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable -help moveTo","title":"\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f"},{"location":"jindotools/jindotable_moveto/#_4","text":"jindotable -moveTo -t <dbName.tableName> -d <destination path> [-c \"<condition>\" | -fullTable] [-b/-before <before days>] [-p/-parallel <parallelism>] [-s/-storagePolicy <OSS storage policy>] [-o/-overWrite] [-r/-removeSource] [-skipTrash] [-e/-explain] [-q/-queue <yarn queue>] [-w/-workingDir <working directory>][-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u8981\u79fb\u52a8\u7684\u8868\u3002 \u662f -d <destination path> \u76ee\u6807\u8def\u5f84\uff0c\u4e3a\u8868\u7ea7\u522b\u7684\u8def\u5f84\uff0c\u5206\u533a\u8def\u5f84\u4f1a\u5728\u8fd9\u4e2a\u8def\u5f84\u4e0b\u81ea\u52a8\u521b\u5efa\u3002 \u662f -c \"<condition>\" / -fullTable \u5206\u533a\u8fc7\u6ee4\u6761\u4ef6\u8868\u8fbe\u5f0f\uff0c\u652f\u6301\u57fa\u672c\u8fd0\u7b97\u7b26\uff0c\u4e0d\u652f\u6301udf\u3002 \u5426 -b/before <before days> \u6839\u636e\u5206\u533a\u521b\u5efa\u65f6\u95f4\uff0c\u521b\u5efa\u65f6\u95f4\u8d85\u8fc7\u7ed9\u5b9a\u5929\u6570\u7684\u5206\u533a\u624d\u8fdb\u884c\u79fb\u52a8\u3002 \u5426 -p/-parallel <parallelism> \u6574\u4e2amoveTo\u4efb\u52a1\u7684\u6700\u5927task\u5e76\u53d1\u5ea6\uff0c\u9ed8\u8ba4\u4e3a1\u3002 \u5426 -s/-storagePolicy <OSS storage policy> \u62f7\u8d1d\u81f3 OSS \u65f6\u6570\u636e\u6587\u4ef6\u7684\u5b58\u50a8\u7b56\u7565\uff0c\u652f\u6301 Standard\uff08\u9ed8\u8ba4\uff09\u3001IA\u3001Archive\u3001ColdArchive\u3002 \u5426 -o/-overWrite \u662f\u5426\u8986\u76d6\u6700\u7ec8\u76ee\u5f55\u3002\u5206\u533a\u7ea7\u522b\u8986\u76d6\uff0c\u4e0d\u4f1a\u8986\u76d6\u672c\u6b21\u79fb\u52a8\u4e0d\u6d89\u53ca\u7684\u5206\u533a\u3002 \u5426 -r/-removeSource \u662f\u5426\u5728\u79fb\u52a8\u5b8c\u6210\u540e\u5220\u9664\u6e90\u8def\u5f84\u3002 \u5426 -skipTrash \u5982\u679c\u5220\u9664\u6e90\u8def\u5f84\uff0c\u662f\u5426\u8df3\u8fc7Trash\u3002 \u5426 -e/-explain \u5982\u679c\u6307\u5b9aexplain\u6a21\u5f0f\uff0c\u4e0d\u4f1a\u89e6\u53d1\u5b9e\u9645\u64cd\u4f5c\uff0c\u4ec5\u6253\u5370\u4f1a\u540c\u6b65\u7684\u5206\u533a\u3002 \u5426 -q/-queue <yarn queue \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684yarn\u961f\u5217\u3002 \u5426 -w/-workingDir \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684\u5de5\u4f5c\u4e34\u65f6\u76ee\u5f55\u3002 \u5426 -iceberg \u6307\u5b9a\u6240\u8fc1\u79fb\u8868\u4e3aIceberg\u3002\u76ee\u524d\u4ec5\u652f\u6301Iceberg\u6574\u8868\u8fc1\u79fb\uff0c\u9700\u914d\u5408 -fullTable \u4f7f\u7528\u3002(6.3.0\u4ee5\u4e0a\u7248\u672c) \u5426 -l/-logDir <log directory> \u672c\u5730\u65e5\u5fd7\u76ee\u5f55\uff0c\u9ed8\u8ba4\u4e3a /tmp/<current user>/ \u5426","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"jindotools/jindotable_moveto/#_5","text":"\u6709\u4e00\u4e2a HDFS \u4e0a\u7684 Hive \u5206\u533a\u8868\uff0c\u5982\u4e0b\u56fe\u6240\u793a\uff1a \u60f3\u628a bbb \u548c ccc \u5206\u533a\u79fb\u52a8\u5230 OSS\uff0c\u5148\u7528 explain \u6a21\u5f0f\u770b\u770b\u4f1a\u79fb\u52a8\u7684\u5206\u533a\u662f\u5426\u7b26\u5408\u9884\u671f\uff0c\u53c2\u6570\u4e3a -e \u6216 -explain\uff1a \u53bb\u6389\u53c2\u6570 -e\uff0c\u771f\u6b63\u79fb\u52a8\u5206\u533a\uff1a \u6267\u884c\u5b8c\u6210\u540e\uff0c\u68c0\u67e5\u6570\u636e\uff0c\u6570\u636e\u5df2\u7ecf\u5728 OSS\uff1a \u518d\u79fb\u56de HDFS\uff0c\u7ed3\u679c\u5931\u8d25\u4e86\uff0c\u4f1a\u663e\u793a\u5931\u8d25\u539f\u56e0\uff0c\u539f\u6765\u662f\u76ee\u6807\u76ee\u5f55\u975e\u7a7a\uff0c\u63d0\u793a\u53ef\u4ee5\u4f7f\u7528 -overWrite \u6e05\u7a7a\u76ee\u6807\u76ee\u5f55\uff1a \u4f7f\u7528 -overWrite \u5f3a\u5236\u8986\u76d6\uff1a","title":"\u4f7f\u7528\u793a\u4f8b"},{"location":"jindotools/jindotable_moveto/#_6","text":"\u4e3a\u4e86\u5c3d\u53ef\u80fd\u4fdd\u8bc1\u6570\u636e\u5b89\u5168\uff0c\u9632\u6b62\u6c61\u67d3\uff0c\u8be5\u547d\u4ee4\u4f1a\u81ea\u52a8\u68c0\u67e5\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u786e\u4fdd\u4e0d\u5b58\u5728\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u76ee\u5f55\u62f7\u8d1d\u3002 \u5982\u679c\u65e0\u6cd5\u6392\u9664\uff0c\u5219\u8be5\u8868\u6216\u5206\u533a\u7684\u62f7\u8d1d\u547d\u4ee4\u5c06\u6267\u884c\u5931\u8d25\u3002 \u6b64\u65f6\uff0c\u9700\u8981\u4e3b\u52a8\u5e72\u9884\uff0c\u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u6b63\u5728\u6267\u884c\u62f7\u8d1d\u4e4b\u540e\uff0c\u6e05\u7406\u76ee\u7684\u5730\u76ee\u5f55\uff0c\u5219\u53ef\u4ee5\u91cd\u65b0\u6267\u884c\u8be5\u547d\u4ee4\u3002 \u5bf9\u4e8e\u975e\u5206\u533a\u8868\u7684\u62f7\u8d1d\u6216\u8fc1\u79fb\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u5373\u4e3a\u8868\u7ea7\u522b\u76ee\u5f55\uff1b\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u76ee\u7684\u5730\u76ee\u5f55\u4e3a\u5f85\u62f7\u8d1d\u6216\u79fb\u52a8\u5206\u533a\u7684\u5206\u533a\u7ea7\u76ee\u5f55\uff0c \u53ea\u9700\u5bf9\u5f85\u62f7\u8d1d\u7684\u5206\u533a\u8fdb\u884c\u6e05\u7406\u3002","title":"\u5f02\u5e38\u72b6\u6001\u5904\u7406"},{"location":"jindotools/jindotable_moveto/#_7","text":"\u5982\u679c\u8be5\u547d\u4ee4\u88ab\u5f02\u5e38\u4e2d\u6b62\uff0c\u5219\u53ef\u80fd\u51fa\u73b0\u9700\u8981\u4e3b\u52a8\u5e72\u9884\u7684\u60c5\u51b5\u3002\u6b64\u65f6\u62f7\u8d1d\u8fd8\u672a\u5b8c\u6210\uff0c\u6e90\u6570\u636e\u4e0e\u8868\u7684\u5143\u4fe1\u606f\u5747\u672a\u6539\u53d8\uff0c \u6570\u636e\u4ecd\u5904\u4e8e\u5b89\u5168\u72b6\u6001\u3002\u5e38\u89c1\u7684\u5f02\u5e38\u4e2d\u6b62\u6709\u4e24\u79cd\u60c5\u51b5\uff1a * \u7528\u6237\u5728\u547d\u4ee4\u5c1a\u672a\u7ed3\u675f\u65f6\uff0c\u4e3b\u52a8\u6740\u6389\u547d\u4ee4\u8fdb\u7a0b * \u7531\u4e8e\u5185\u5b58\u6ea2\u51fa\u7b49\u5f02\u5e38\uff0c\u8fdb\u7a0b\u5f02\u5e38\u4e2d\u6b62 \u5982\u679c\u62f7\u8d1d\u8868\u6216\u5206\u533a\u65f6\u62f7\u8d1d\u5931\u8d25\u5e76\u663e\u793a Conflicts found \uff0c\u5219\u5c5e\u4e8e\u5f02\u5e38\u72b6\u6001\uff0c\u9700\u4e3b\u52a8\u5e72\u9884\u3002","title":"\u51fa\u73b0\u539f\u56e0"},{"location":"jindotools/jindotable_moveto/#_8","text":"\u786e\u4fdd\u6ca1\u6709\u53e6\u4e00\u547d\u4ee4\u5411\u76f8\u540c\u7684\u76ee\u6807\u8def\u5f84\u62f7\u8d1d\u6570\u636e\u3002\u7279\u522b\u662f\uff0c\u786e\u4fdd\u6ca1\u6709 DistCp/JindoDistCp \u7b49\u5206\u5e03\u5f0f\u62f7\u8d1d\u547d\u4ee4\u3002 \u5220\u9664\u76ee\u7684\u5730\u76ee\u5f55\u3002\u5bf9\u4e8e\u975e\u5206\u533a\u8868\uff0c\u5220\u9664\u8868\u4e00\u7ea7\u76ee\u5f55\u3002\u5bf9\u4e8e\u5206\u533a\u8868\uff0c\u5220\u9664\u5b58\u5728\u51b2\u7a81\u7684\u5206\u533a\u7ea7\u76ee\u5f55\u3002\u5207\u52ff\u5220\u9664\u6e90\u76ee\u5f55\u3002","title":"\u5904\u7406\u65b9\u6cd5"},{"location":"jindotools/jindotable_set_storage_class/","text":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fdb\u884c\u5f52\u6863 \u00a7 \u4f7f\u7528\u8bf4\u660e \u00a7 \u8be5\u547d\u4ee4\u53ea\u652f\u6301\u6807\u51c6 OSS \u5b58\u50a8\u7c7b\u578b\u7684\u8f6c\u6362\uff0c \u4e0d\u652f\u6301OSS-HDFS \u5b58\u50a8\u7c7b\u578b\u8f6c\u6362\uff0cOSS-HDFS \u5b58\u50a8\u7c7b\u578b\u8f6c\u6362\u53c2\u8003 \u94fe\u63a5 \u914d\u7f6eOSS \u751f\u547d\u5468\u671f \u00a7 \u914d\u7f6e\u4f4e\u9891\u751f\u547d\u5468\u671f\u89c4\u5219 \u00a7 \u914d\u7f6e\u5f52\u6863\u751f\u547d\u5468\u671f\u89c4\u5219 \u00a7 \u914d\u7f6e\u51b7\u5f52\u6863\u751f\u547d\u5468\u671f\u89c4\u5219 \u00a7 \u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f \u00a7 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable help -setStorageClass \u53c2\u6570\u8bf4\u660e \u00a7 [-setStorageClass -t <dbName.tableName> -i/-a/-ca[-c \"<condition>\" | -fullTable] [-b/-before <before days>] [-p/-parallel <parallelism>] [-mr/-mapReduce] [-e/-explain] [-w/-workingDir <working directory>][-l/-logDir <log directory>]] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u8981\u79fb\u52a8\u7684\u8868\u3002 \u662f -i/-a/-ca \u6307\u5b9a\u5b58\u50a8\u7c7b\u578b\uff0c-i\u8868\u793a\u4f4e\u9891\uff0c-a\u8868\u793a\u5f52\u6863 -ca\u8868\u793a\u51b7\u5f52\u6863\u3002 \u662f -c \"<condition>\" / -fullTable \u5206\u533a\u8fc7\u6ee4\u6761\u4ef6\u8868\u8fbe\u5f0f\uff0c\u652f\u6301\u57fa\u672c\u8fd0\u7b97\u7b26\uff0c\u4e0d\u652f\u6301udf\u3002 \u5426 -b/before <before days> \u6839\u636e\u5206\u533a\u521b\u5efa\u65f6\u95f4\uff0c\u521b\u5efa\u65f6\u95f4\u8d85\u8fc7\u7ed9\u5b9a\u5929\u6570\u7684\u5206\u533a\u624d\u8fdb\u884c\u79fb\u52a8\u3002 \u5426 -p/-parallel <parallelism> \u6574\u4e2amoveTo\u4efb\u52a1\u7684\u6700\u5927task\u5e76\u53d1\u5ea6\uff0c\u9ed8\u8ba4\u4e3a1\u3002 \u5426 -e/-explain \u5982\u679c\u6307\u5b9aexplain\u6a21\u5f0f\uff0c\u4e0d\u4f1a\u89e6\u53d1\u5b9e\u9645\u64cd\u4f5c\uff0c\u4ec5\u6253\u5370\u4f1a\u540c\u6b65\u7684\u5206\u533a\u3002 \u5426 -w/-workingDir \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684\u5de5\u4f5c\u4e34\u65f6\u76ee\u5f55\u3002 \u5426 -l/-logDir <log directory> \u672c\u5730\u65e5\u5fd7\u76ee\u5f55\uff0c\u9ed8\u8ba4\u4e3a /tmp/<current user>/ \u5426 \u4f7f\u7528\u793a\u4f8b \u00a7 jindotable -setStorageClass -t data_center.test_partition -ca -c \"date_id > '20240110'\"","title":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fdb\u884c\u5f52\u6863"},{"location":"jindotools/jindotable_set_storage_class/#jindotable-hive","text":"","title":"\u4f7f\u7528 JindoTable \u5c06 Hive \u8868\u548c\u5206\u533a\u6570\u636e\u8fdb\u884c\u5f52\u6863"},{"location":"jindotools/jindotable_set_storage_class/#_1","text":"\u8be5\u547d\u4ee4\u53ea\u652f\u6301\u6807\u51c6 OSS \u5b58\u50a8\u7c7b\u578b\u7684\u8f6c\u6362\uff0c \u4e0d\u652f\u6301OSS-HDFS \u5b58\u50a8\u7c7b\u578b\u8f6c\u6362\uff0cOSS-HDFS \u5b58\u50a8\u7c7b\u578b\u8f6c\u6362\u53c2\u8003 \u94fe\u63a5","title":"\u4f7f\u7528\u8bf4\u660e"},{"location":"jindotools/jindotable_set_storage_class/#oss","text":"","title":"\u914d\u7f6eOSS \u751f\u547d\u5468\u671f"},{"location":"jindotools/jindotable_set_storage_class/#_2","text":"","title":"\u914d\u7f6e\u4f4e\u9891\u751f\u547d\u5468\u671f\u89c4\u5219"},{"location":"jindotools/jindotable_set_storage_class/#_3","text":"","title":"\u914d\u7f6e\u5f52\u6863\u751f\u547d\u5468\u671f\u89c4\u5219"},{"location":"jindotools/jindotable_set_storage_class/#_4","text":"","title":"\u914d\u7f6e\u51b7\u5f52\u6863\u751f\u547d\u5468\u671f\u89c4\u5219"},{"location":"jindotools/jindotable_set_storage_class/#_5","text":"\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f\u3002 jindotable help -setStorageClass","title":"\u83b7\u53d6\u5e2e\u52a9\u4fe1\u606f"},{"location":"jindotools/jindotable_set_storage_class/#_6","text":"[-setStorageClass -t <dbName.tableName> -i/-a/-ca[-c \"<condition>\" | -fullTable] [-b/-before <before days>] [-p/-parallel <parallelism>] [-mr/-mapReduce] [-e/-explain] [-w/-workingDir <working directory>][-l/-logDir <log directory>]] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u8981\u79fb\u52a8\u7684\u8868\u3002 \u662f -i/-a/-ca \u6307\u5b9a\u5b58\u50a8\u7c7b\u578b\uff0c-i\u8868\u793a\u4f4e\u9891\uff0c-a\u8868\u793a\u5f52\u6863 -ca\u8868\u793a\u51b7\u5f52\u6863\u3002 \u662f -c \"<condition>\" / -fullTable \u5206\u533a\u8fc7\u6ee4\u6761\u4ef6\u8868\u8fbe\u5f0f\uff0c\u652f\u6301\u57fa\u672c\u8fd0\u7b97\u7b26\uff0c\u4e0d\u652f\u6301udf\u3002 \u5426 -b/before <before days> \u6839\u636e\u5206\u533a\u521b\u5efa\u65f6\u95f4\uff0c\u521b\u5efa\u65f6\u95f4\u8d85\u8fc7\u7ed9\u5b9a\u5929\u6570\u7684\u5206\u533a\u624d\u8fdb\u884c\u79fb\u52a8\u3002 \u5426 -p/-parallel <parallelism> \u6574\u4e2amoveTo\u4efb\u52a1\u7684\u6700\u5927task\u5e76\u53d1\u5ea6\uff0c\u9ed8\u8ba4\u4e3a1\u3002 \u5426 -e/-explain \u5982\u679c\u6307\u5b9aexplain\u6a21\u5f0f\uff0c\u4e0d\u4f1a\u89e6\u53d1\u5b9e\u9645\u64cd\u4f5c\uff0c\u4ec5\u6253\u5370\u4f1a\u540c\u6b65\u7684\u5206\u533a\u3002 \u5426 -w/-workingDir \u6307\u5b9a\u5206\u5e03\u5f0f\u62f7\u8d1d\u7684\u5de5\u4f5c\u4e34\u65f6\u76ee\u5f55\u3002 \u5426 -l/-logDir <log directory> \u672c\u5730\u65e5\u5fd7\u76ee\u5f55\uff0c\u9ed8\u8ba4\u4e3a /tmp/<current user>/ \u5426","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"jindotools/jindotable_set_storage_class/#_7","text":"jindotable -setStorageClass -t data_center.test_partition -ca -c \"date_id > '20240110'\"","title":"\u4f7f\u7528\u793a\u4f8b"},{"location":"jindotools/jindotable_storagepolicy_restore/","text":"\u4f7f\u7528 JindoTable \u5de5\u5177\u5b9e\u73b0 OSS-HDFS \u7ed3\u6784\u5316\u6570\u636e\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8 \u00a7 \u6982\u8ff0 \u00a7 JindoTable \u662f\u57fa\u4e8e JindoSDK \u7684\u7ed3\u6784\u5316\u6570\u636e\u6cbb\u7406\u5de5\u5177\uff0c\u76ee\u6807\u662f\u5728 \u201c\u8868\u201d \u6216\u8005 \u201c\u5206\u533a\u201d \u5c42\u9762\u63d0\u4f9b\u4fbf\u6377\u7684\u6570\u636e\u8bbf\u95ee\u548c\u64cd\u4f5c\u3002\u4f7f\u7528 JindoTable \u5de5\u5177\u53ef\u4ee5\u65b9\u4fbf\u5730\u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u548c\u5206\u5c42\u5b58\u50a8\u3002 \u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u4f7f\u7528 JindoTable \u5de5\u5177\u5bf9\u5b58\u50a8\u5728 OSS-HDFS \u4e0a\u7684\u7ed3\u6784\u5316\u6570\u636e\u8fdb\u884c\u51b7\u70ed\u5206\u5c42\u5b58\u50a8\u3002 \u83b7\u53d6 \u00a7 \u53ef\u4ee5\u4ece JindoData \u4e0b\u8f7d\u9875\u9762 \u83b7\u53d6 JindoTable \u5de5\u5177\uff1a 1. \u4e0b\u8f7d jindosdk-<version>-<platform>.tar.gz \uff0c\u6839\u636e\u7cfb\u7edf\u9009\u62e9\u5408\u9002\u7684 <platform> \u3002 2. \u672c\u6587\u4ecb\u7ecd\u7684\u529f\u80fd\u6700\u4f4e\u7248\u672c\u8981\u6c42\u4e3a <version> 6.1.2\u3002 \u6ce8\uff1a - JindoTable \u5de5\u5177\u5bf9 OSS-HDFS \u7684\u4e34\u65f6\u89e3\u51bb\uff08 -once \uff09\u652f\u6301\u8bbe\u7f6e\u89e3\u51bb\u53c2\u6570\uff08 -d/-restoreDays <restore days> \uff09\uff0c\u7248\u672c\u8981\u6c42\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0\u3002\u82e5\u5bf9\u8be5\u529f\u80fd\u6709\u9700\u8981\uff0c\u8bf7\u4f7f\u7528\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0 \u7684\u7248\u672c\u3002 \u57fa\u672c\u7528\u6cd5 \u00a7 \u6309\u7167\u4e0a\u9762\u7684\u6b65\u9aa4\u83b7\u53d6\u538b\u7f29\u5305\u540e\uff0c\u89e3\u538b\u7f29\uff0c\u4fbf\u53ef\u5728 bin/ \u76ee\u5f55\u4e0b\u627e\u5230\u540d\u4e3a jindotable \u7684\u811a\u672c\u3002\u6267\u884c\u811a\u672c\uff0c\u5373\u53ef\u4f7f\u7528 JindoTable \u63d0\u4f9b\u7684\u5404\u79cd\u547d\u4ee4\u3002 \u6267\u884c jindotable -usage \u53ef\u4ee5\u663e\u793a\u5de5\u5177\u652f\u6301\u7684\u5404\u79cd\u547d\u4ee4\u3002 \u4e00\u4e9b\u6ce8\u610f\u4e8b\u9879\uff1a - \u6267\u884c\u65f6\uff0c\u987b\u4fdd\u6301\u89e3\u538b\u7f29\u540e\u7684\u76ee\u5f55\u7ed3\u6784\u3002 - \u9700\u8981\u96c6\u7fa4\u4e0a\u6709 Hadoop \u548c Hive \u73af\u5883\uff0c\u76f8\u5173\u7684\u73af\u5883\u53d8\u91cf\u5df2\u7ecf\u88ab\u6b63\u786e\u914d\u7f6e\u3002 \u5206\u5c42\u5b58\u50a8\u547d\u4ee4 \u00a7 \u652f\u6301 OSS-HDFS \u7ed3\u6784\u5316\u6570\u636e\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8\u529f\u80fd\uff0c\u547d\u4ee4\u4e3a -setStoragePolicy \uff0c\u6267\u884c jindotable -help setStoragePolicy \u53ef\u4ee5\u663e\u793a\u5e2e\u52a9\u3002 \u53c2\u6570\u8bf4\u660e \u00a7 jindotable -setStoragePolicy -t <dbName.tableName> -std/-ia/-archive/-coldArchive/-once [-c \"<condition>\" | -fullTable] [-d/-restoreDays <restore days>] [-b/-before <before days>] [-p/-parallel <parallelism>] [-mr/-mapReduce] [-e/-explain] [-w/-workingDir <working directory>] [-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u76ee\u6807\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a\u6570\u636e\u5e93\u540d.\u8868\u540d\u3002\u6570\u636e\u5e93\u548c\u8868\u540d\u4e4b\u95f4\u4ee5\u534a\u89d2\u53e5\u53f7\uff08.\uff09\u5206\u9694\u3002\u8868\u53ef\u4ee5\u662f\u5206\u533a\u8868\u6216\u975e\u5206\u533a\u8868\u3002 \u662f -std/-ia/-archive/-coldArchive/-once \u76ee\u6807\u5b58\u50a8\u65b9\u5f0f\u3002\u652f\u6301\u5982\u4e0b\u7c7b\u578b\uff1a-std\uff1a\u6807\u51c6\u5b58\u50a8\uff0c-ia\uff1a\u4f4e\u9891\u5b58\u50a8\uff0c-archive\uff1a\u5f52\u6863\u5b58\u50a8\uff0c-coldArchive\uff1a\u51b7\u5f52\u6863\u5b58\u50a8\uff0c-once\uff1a\u4e34\u65f6\u89e3\u51bb\u3002 \u662f -c \"<condition>\" / -fullTable -fullTable \u548c -c \"<condition>\" \u53ea\u9700\u63d0\u4f9b\u4e00\u4e2a\uff0c\u5373\u8981\u4e48\u6307\u5b9a -c \"<condition>\" \uff0c\u8981\u4e48\u6307\u5b9a -fullTable \u3002\u6307\u5b9a -fullTable \u65f6\uff0c\u5219\u4e3a\u79fb\u52a8\u6574\u8868\uff0c\u65e2\u53ef\u4ee5\u662f\u975e\u5206\u533a\u8868\u4e5f\u53ef\u4ee5\u662f\u5206\u533a\u8868\u3002\u6307\u5b9a -c \"<condition>\" \u65f6\uff0c\u5219\u63d0\u4f9b\u4e86\u4e00\u4e2a\u8fc7\u6ee4\u6761\u4ef6\uff0c\u7528\u6765\u9009\u62e9\u5e0c\u671b\u79fb\u52a8\u7684\u5206\u533a\uff0c\u652f\u6301\u5e38\u89c1\u8fd0\u7b97\u7b26\uff0c\u4f8b\u5982\u5927\u4e8e\u53f7\uff08>\uff09\u3002\u4f8b\u5982\uff0c\u6570\u636e\u7c7b\u578b\u4e3aString\u7684\u5206\u533ads\uff0c\u5e0c\u671b\u5206\u533a\u540d\u5927\u4e8e 'd'\uff0c\u5219\u4ee3\u7801\u4e3a-c \" ds > 'd' \"\u3002 \u662f -d/-restoreDays <restore days> \u6307\u5b9a\u4e34\u65f6\u89e3\u51bb\u7684\u5929\u6570\uff0c\u4ec5\u5bf9\u4e34\u65f6\u89e3\u51bb -once \u751f\u6548\u3002 \u5426 -b/before <before days> \u53ea\u6709\u521b\u5efa\u65f6\u95f4\u8ddd\u79bb\u73b0\u5728\u8d85\u8fc7\u4e00\u5b9a\u5929\u6570\u7684\u8868\u6216\u5206\u533a\u624d\u4f1a\u88ab\u6267\u884c\u3002 \u5426 -p/-parallel <parallelism> \u547d\u4ee4\u6267\u884c\u7684\u5e76\u884c\u5ea6\u3002 \u5426 -mr/-mapReduce \u4f7f\u7528Hadoop MapReduce\u800c\u975e\u672c\u5730\u591a\u7ebf\u7a0b\u6765\u6267\u884c\u547d\u4ee4\u3002 \u5426 -e/-explain \u5982\u679c\u51fa\u73b0\u8be5\u9009\u9879\uff0c\u5219\u4e3a\u89e3\u91ca\uff08explain\uff09\u6a21\u5f0f\uff0c\u53ea\u4f1a\u663e\u793a\u5f85\u64cd\u4f5c\u7684\u5206\u533a\u5217\u8868\uff0c\u800c\u4e0d\u4f1a\u771f\u6b63\u6267\u884c\u547d\u4ee4\u3002 \u5426 -w/-workingDir <working directory> \u53ea\u5728MapReduce\u4f5c\u4e1a\u65f6\u4f7f\u7528\uff0c\u4e3aMapReduce\u4f5c\u4e1a\u7684\u5de5\u4f5c\u76ee\u5f55\u3002\u5fc5\u987b\u6709\u8bfb\u5199\u6743\u9650\uff0c\u5de5\u4f5c\u76ee\u5f55\u53ef\u4ee5\u975e\u7a7a\uff08\u4f5c\u4e1a\u6267\u884c\u8fc7\u7a0b\u4e2d\u4f1a\u521b\u5efa\u4e34\u65f6\u6587\u4ef6\uff0c\u6267\u884c\u5b8c\u6bd5\u4f1a\u6e05\u7406\u4e34\u65f6\u6587\u4ef6\uff09\u3002 \u5426 -l/-logDir <log directory> \u6307\u5b9aLog\u6587\u4ef6\u7684\u76ee\u5f55\u3002 \u5426 \u6ce8\u610f\u4e8b\u9879 \u00a7 JindoTable \u63d0\u4f9b\u7684\u4e0a\u8ff0\u529f\u80fd\u57fa\u672c\u5bf9\u5e94\u4e8e OSS-HDFS \u7ebf\u4e0a\u670d\u52a1\u7684\u5206\u5c42\u5b58\u50a8\u80fd\u529b\uff0c\u6709\u4e0b\u5217\u4e8b\u9879\u9700\u8981\u6ce8\u610f\uff1a 1. \u4ec5\u652f\u6301 OSS-HDFS \u5b58\u50a8\u7684\u8868\u6216\u8005\u5206\u533a\u3002\u5bf9\u666e\u901a OSS \u6570\u636e\u7684\u5206\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoTable \u63d0\u4f9b\u7684 archiveTable \u4e0e unarchiveTable \u547d\u4ee4\u3002 2. \u6210\u529f\u6267\u884c -std/-ia/-archive/-coldArchive/-once \u6307\u5b9a\u7684\u8f6c\u6362\u5b58\u50a8\u7c7b\u578b\u547d\u4ee4\u540e\uff0c\u5fc5\u987b\u7b49\u5f85 2 \u5929\u624d\u53ef\u518d\u6b21\u8f6c\u6362\u5b58\u50a8\u7c7b\u578b\u3002 3. \u6807\u51c6\u548c\u4f4e\u9891\u7c7b\u578b\u65e0\u6cd5\u4e34\u65f6\u89e3\u51bb\uff08 -once \uff09\u3002\u5f52\u6863\u3001\u51b7\u5f52\u6863\uff0c\u6216\u8005\u5df2\u7ecf\u5904\u4e8e\u89e3\u51bb\u72b6\u6001\u65f6\uff0c\u53ef\u4ee5\u89e3\u51bb\uff0c\u4f46\u5982\u4e0a\u6240\u8ff0\uff0c\u95f4\u9694\u987b\u8d85\u8fc7 2 \u5929\u3002 4. \u4e34\u65f6\u89e3\u51bb\u547d\u4ee4\u8fd4\u56de\u540e\uff0c\u6570\u636e\u4ecd\u4e0d\u80fd\u7acb\u5373\u53ef\u8bfb\uff0c\u9700\u8981\u7b49\u5f85\u8fdc\u7aef\u670d\u52a1\u5b8c\u6210\u6570\u636e\u7684\u89e3\u51bb\u3002\u5bf9\u4e8e\u5f52\u6863\u6570\u636e\uff0c\u901a\u5e38\u9700\u8981\u6570\u5206\u949f\uff1b\u5bf9\u4e8e\u51b7\u5f52\u6863\u6570\u636e\uff0c\u901a\u5e38\u9700\u8981\u6570\u5c0f\u65f6\u3002 5. \u4e34\u65f6\u89e3\u51bb\u6709\u671f\u9650\u9650\u5236\uff0c\u901a\u8fc7 -d/restoreDays <restore days> \u6307\u5b9a\uff0c\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4 1 \u5929\uff0c\u5230\u671f\u540e\u6570\u636e\u5c06\u56de\u5230\u4e0d\u53ef\u8bfb\u72b6\u6001\u3002\u8be5\u529f\u80fd\u8981\u6c42\u7248\u672c\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0\u3002","title":"\u4f7f\u7528 JindoTable \u5de5\u5177\u5b9e\u73b0 OSS-HDFS \u7ed3\u6784\u5316\u6570\u636e\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8"},{"location":"jindotools/jindotable_storagepolicy_restore/#jindotable-oss-hdfs","text":"","title":"\u4f7f\u7528 JindoTable \u5de5\u5177\u5b9e\u73b0 OSS-HDFS \u7ed3\u6784\u5316\u6570\u636e\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8"},{"location":"jindotools/jindotable_storagepolicy_restore/#_1","text":"JindoTable \u662f\u57fa\u4e8e JindoSDK \u7684\u7ed3\u6784\u5316\u6570\u636e\u6cbb\u7406\u5de5\u5177\uff0c\u76ee\u6807\u662f\u5728 \u201c\u8868\u201d \u6216\u8005 \u201c\u5206\u533a\u201d \u5c42\u9762\u63d0\u4f9b\u4fbf\u6377\u7684\u6570\u636e\u8bbf\u95ee\u548c\u64cd\u4f5c\u3002\u4f7f\u7528 JindoTable \u5de5\u5177\u53ef\u4ee5\u65b9\u4fbf\u5730\u8fdb\u884c\u6570\u636e\u8fc1\u79fb\u548c\u5206\u5c42\u5b58\u50a8\u3002 \u672c\u6587\u4ecb\u7ecd\u5982\u4f55\u4f7f\u7528 JindoTable \u5de5\u5177\u5bf9\u5b58\u50a8\u5728 OSS-HDFS \u4e0a\u7684\u7ed3\u6784\u5316\u6570\u636e\u8fdb\u884c\u51b7\u70ed\u5206\u5c42\u5b58\u50a8\u3002","title":"\u6982\u8ff0"},{"location":"jindotools/jindotable_storagepolicy_restore/#_2","text":"\u53ef\u4ee5\u4ece JindoData \u4e0b\u8f7d\u9875\u9762 \u83b7\u53d6 JindoTable \u5de5\u5177\uff1a 1. \u4e0b\u8f7d jindosdk-<version>-<platform>.tar.gz \uff0c\u6839\u636e\u7cfb\u7edf\u9009\u62e9\u5408\u9002\u7684 <platform> \u3002 2. \u672c\u6587\u4ecb\u7ecd\u7684\u529f\u80fd\u6700\u4f4e\u7248\u672c\u8981\u6c42\u4e3a <version> 6.1.2\u3002 \u6ce8\uff1a - JindoTable \u5de5\u5177\u5bf9 OSS-HDFS \u7684\u4e34\u65f6\u89e3\u51bb\uff08 -once \uff09\u652f\u6301\u8bbe\u7f6e\u89e3\u51bb\u53c2\u6570\uff08 -d/-restoreDays <restore days> \uff09\uff0c\u7248\u672c\u8981\u6c42\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0\u3002\u82e5\u5bf9\u8be5\u529f\u80fd\u6709\u9700\u8981\uff0c\u8bf7\u4f7f\u7528\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0 \u7684\u7248\u672c\u3002","title":"\u83b7\u53d6"},{"location":"jindotools/jindotable_storagepolicy_restore/#_3","text":"\u6309\u7167\u4e0a\u9762\u7684\u6b65\u9aa4\u83b7\u53d6\u538b\u7f29\u5305\u540e\uff0c\u89e3\u538b\u7f29\uff0c\u4fbf\u53ef\u5728 bin/ \u76ee\u5f55\u4e0b\u627e\u5230\u540d\u4e3a jindotable \u7684\u811a\u672c\u3002\u6267\u884c\u811a\u672c\uff0c\u5373\u53ef\u4f7f\u7528 JindoTable \u63d0\u4f9b\u7684\u5404\u79cd\u547d\u4ee4\u3002 \u6267\u884c jindotable -usage \u53ef\u4ee5\u663e\u793a\u5de5\u5177\u652f\u6301\u7684\u5404\u79cd\u547d\u4ee4\u3002 \u4e00\u4e9b\u6ce8\u610f\u4e8b\u9879\uff1a - \u6267\u884c\u65f6\uff0c\u987b\u4fdd\u6301\u89e3\u538b\u7f29\u540e\u7684\u76ee\u5f55\u7ed3\u6784\u3002 - \u9700\u8981\u96c6\u7fa4\u4e0a\u6709 Hadoop \u548c Hive \u73af\u5883\uff0c\u76f8\u5173\u7684\u73af\u5883\u53d8\u91cf\u5df2\u7ecf\u88ab\u6b63\u786e\u914d\u7f6e\u3002","title":"\u57fa\u672c\u7528\u6cd5"},{"location":"jindotools/jindotable_storagepolicy_restore/#_4","text":"\u652f\u6301 OSS-HDFS \u7ed3\u6784\u5316\u6570\u636e\u7684\u51b7\u70ed\u5206\u5c42\u5b58\u50a8\u529f\u80fd\uff0c\u547d\u4ee4\u4e3a -setStoragePolicy \uff0c\u6267\u884c jindotable -help setStoragePolicy \u53ef\u4ee5\u663e\u793a\u5e2e\u52a9\u3002","title":"\u5206\u5c42\u5b58\u50a8\u547d\u4ee4"},{"location":"jindotools/jindotable_storagepolicy_restore/#_5","text":"jindotable -setStoragePolicy -t <dbName.tableName> -std/-ia/-archive/-coldArchive/-once [-c \"<condition>\" | -fullTable] [-d/-restoreDays <restore days>] [-b/-before <before days>] [-p/-parallel <parallelism>] [-mr/-mapReduce] [-e/-explain] [-w/-workingDir <working directory>] [-l/-logDir <log directory>] \u53c2\u6570 \u63cf\u8ff0 \u662f\u5426\u5fc5\u9009 -t <dbName.tableName> \u76ee\u6807\u8868\u540d\u79f0\uff0c\u683c\u5f0f\u4e3a\u6570\u636e\u5e93\u540d.\u8868\u540d\u3002\u6570\u636e\u5e93\u548c\u8868\u540d\u4e4b\u95f4\u4ee5\u534a\u89d2\u53e5\u53f7\uff08.\uff09\u5206\u9694\u3002\u8868\u53ef\u4ee5\u662f\u5206\u533a\u8868\u6216\u975e\u5206\u533a\u8868\u3002 \u662f -std/-ia/-archive/-coldArchive/-once \u76ee\u6807\u5b58\u50a8\u65b9\u5f0f\u3002\u652f\u6301\u5982\u4e0b\u7c7b\u578b\uff1a-std\uff1a\u6807\u51c6\u5b58\u50a8\uff0c-ia\uff1a\u4f4e\u9891\u5b58\u50a8\uff0c-archive\uff1a\u5f52\u6863\u5b58\u50a8\uff0c-coldArchive\uff1a\u51b7\u5f52\u6863\u5b58\u50a8\uff0c-once\uff1a\u4e34\u65f6\u89e3\u51bb\u3002 \u662f -c \"<condition>\" / -fullTable -fullTable \u548c -c \"<condition>\" \u53ea\u9700\u63d0\u4f9b\u4e00\u4e2a\uff0c\u5373\u8981\u4e48\u6307\u5b9a -c \"<condition>\" \uff0c\u8981\u4e48\u6307\u5b9a -fullTable \u3002\u6307\u5b9a -fullTable \u65f6\uff0c\u5219\u4e3a\u79fb\u52a8\u6574\u8868\uff0c\u65e2\u53ef\u4ee5\u662f\u975e\u5206\u533a\u8868\u4e5f\u53ef\u4ee5\u662f\u5206\u533a\u8868\u3002\u6307\u5b9a -c \"<condition>\" \u65f6\uff0c\u5219\u63d0\u4f9b\u4e86\u4e00\u4e2a\u8fc7\u6ee4\u6761\u4ef6\uff0c\u7528\u6765\u9009\u62e9\u5e0c\u671b\u79fb\u52a8\u7684\u5206\u533a\uff0c\u652f\u6301\u5e38\u89c1\u8fd0\u7b97\u7b26\uff0c\u4f8b\u5982\u5927\u4e8e\u53f7\uff08>\uff09\u3002\u4f8b\u5982\uff0c\u6570\u636e\u7c7b\u578b\u4e3aString\u7684\u5206\u533ads\uff0c\u5e0c\u671b\u5206\u533a\u540d\u5927\u4e8e 'd'\uff0c\u5219\u4ee3\u7801\u4e3a-c \" ds > 'd' \"\u3002 \u662f -d/-restoreDays <restore days> \u6307\u5b9a\u4e34\u65f6\u89e3\u51bb\u7684\u5929\u6570\uff0c\u4ec5\u5bf9\u4e34\u65f6\u89e3\u51bb -once \u751f\u6548\u3002 \u5426 -b/before <before days> \u53ea\u6709\u521b\u5efa\u65f6\u95f4\u8ddd\u79bb\u73b0\u5728\u8d85\u8fc7\u4e00\u5b9a\u5929\u6570\u7684\u8868\u6216\u5206\u533a\u624d\u4f1a\u88ab\u6267\u884c\u3002 \u5426 -p/-parallel <parallelism> \u547d\u4ee4\u6267\u884c\u7684\u5e76\u884c\u5ea6\u3002 \u5426 -mr/-mapReduce \u4f7f\u7528Hadoop MapReduce\u800c\u975e\u672c\u5730\u591a\u7ebf\u7a0b\u6765\u6267\u884c\u547d\u4ee4\u3002 \u5426 -e/-explain \u5982\u679c\u51fa\u73b0\u8be5\u9009\u9879\uff0c\u5219\u4e3a\u89e3\u91ca\uff08explain\uff09\u6a21\u5f0f\uff0c\u53ea\u4f1a\u663e\u793a\u5f85\u64cd\u4f5c\u7684\u5206\u533a\u5217\u8868\uff0c\u800c\u4e0d\u4f1a\u771f\u6b63\u6267\u884c\u547d\u4ee4\u3002 \u5426 -w/-workingDir <working directory> \u53ea\u5728MapReduce\u4f5c\u4e1a\u65f6\u4f7f\u7528\uff0c\u4e3aMapReduce\u4f5c\u4e1a\u7684\u5de5\u4f5c\u76ee\u5f55\u3002\u5fc5\u987b\u6709\u8bfb\u5199\u6743\u9650\uff0c\u5de5\u4f5c\u76ee\u5f55\u53ef\u4ee5\u975e\u7a7a\uff08\u4f5c\u4e1a\u6267\u884c\u8fc7\u7a0b\u4e2d\u4f1a\u521b\u5efa\u4e34\u65f6\u6587\u4ef6\uff0c\u6267\u884c\u5b8c\u6bd5\u4f1a\u6e05\u7406\u4e34\u65f6\u6587\u4ef6\uff09\u3002 \u5426 -l/-logDir <log directory> \u6307\u5b9aLog\u6587\u4ef6\u7684\u76ee\u5f55\u3002 \u5426","title":"\u53c2\u6570\u8bf4\u660e"},{"location":"jindotools/jindotable_storagepolicy_restore/#_6","text":"JindoTable \u63d0\u4f9b\u7684\u4e0a\u8ff0\u529f\u80fd\u57fa\u672c\u5bf9\u5e94\u4e8e OSS-HDFS \u7ebf\u4e0a\u670d\u52a1\u7684\u5206\u5c42\u5b58\u50a8\u80fd\u529b\uff0c\u6709\u4e0b\u5217\u4e8b\u9879\u9700\u8981\u6ce8\u610f\uff1a 1. \u4ec5\u652f\u6301 OSS-HDFS \u5b58\u50a8\u7684\u8868\u6216\u8005\u5206\u533a\u3002\u5bf9\u666e\u901a OSS \u6570\u636e\u7684\u5206\u5c42\u5b58\u50a8\uff0c\u4f7f\u7528 JindoTable \u63d0\u4f9b\u7684 archiveTable \u4e0e unarchiveTable \u547d\u4ee4\u3002 2. \u6210\u529f\u6267\u884c -std/-ia/-archive/-coldArchive/-once \u6307\u5b9a\u7684\u8f6c\u6362\u5b58\u50a8\u7c7b\u578b\u547d\u4ee4\u540e\uff0c\u5fc5\u987b\u7b49\u5f85 2 \u5929\u624d\u53ef\u518d\u6b21\u8f6c\u6362\u5b58\u50a8\u7c7b\u578b\u3002 3. \u6807\u51c6\u548c\u4f4e\u9891\u7c7b\u578b\u65e0\u6cd5\u4e34\u65f6\u89e3\u51bb\uff08 -once \uff09\u3002\u5f52\u6863\u3001\u51b7\u5f52\u6863\uff0c\u6216\u8005\u5df2\u7ecf\u5904\u4e8e\u89e3\u51bb\u72b6\u6001\u65f6\uff0c\u53ef\u4ee5\u89e3\u51bb\uff0c\u4f46\u5982\u4e0a\u6240\u8ff0\uff0c\u95f4\u9694\u987b\u8d85\u8fc7 2 \u5929\u3002 4. \u4e34\u65f6\u89e3\u51bb\u547d\u4ee4\u8fd4\u56de\u540e\uff0c\u6570\u636e\u4ecd\u4e0d\u80fd\u7acb\u5373\u53ef\u8bfb\uff0c\u9700\u8981\u7b49\u5f85\u8fdc\u7aef\u670d\u52a1\u5b8c\u6210\u6570\u636e\u7684\u89e3\u51bb\u3002\u5bf9\u4e8e\u5f52\u6863\u6570\u636e\uff0c\u901a\u5e38\u9700\u8981\u6570\u5206\u949f\uff1b\u5bf9\u4e8e\u51b7\u5f52\u6863\u6570\u636e\uff0c\u901a\u5e38\u9700\u8981\u6570\u5c0f\u65f6\u3002 5. \u4e34\u65f6\u89e3\u51bb\u6709\u671f\u9650\u9650\u5236\uff0c\u901a\u8fc7 -d/restoreDays <restore days> \u6307\u5b9a\uff0c\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4 1 \u5929\uff0c\u5230\u671f\u540e\u6570\u636e\u5c06\u56de\u5230\u4e0d\u53ef\u8bfb\u72b6\u6001\u3002\u8be5\u529f\u80fd\u8981\u6c42\u7248\u672c\u4e0d\u4f4e\u4e8e 6.3.5 \u6216 6.4.0\u3002","title":"\u6ce8\u610f\u4e8b\u9879"},{"location":"oss/dls_atime/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09AccessTime\u4f7f\u7528\u8bf4\u660e \u00a7 \u4ecb\u7ecd \u00a7 OSS-HDFS \u652f\u6301\u6587\u4ef6 AccessTime\uff0c \u6587\u4ef6 AccessTime \u7cbe\u5ea6\u9ed8\u8ba4\u914d\u7f6e\u4e3a1\u5c0f\u65f6\uff0c\u5177\u4f53\u914d\u7f6e\u9879\u4e3a namespace.file.accesstime.precision.millisecond = 3600000 AccessTime \u7cbe\u5ea6\u8868\u793a\u5982\u679c\u5f53\u524d\u65f6\u95f4\u548c\u6587\u4ef6\u5f53\u524d\u7684 AccessTime \u4e4b\u524d\u7684\u5dee\u503c\u8d85\u8fc7\u914d\u7f6e\u7684\u7cbe\u5ea6\u503c\uff0c\u5219\u66f4\u65b0\u6587\u4ef6\u7684AccessTime\uff0c\u5426\u5219\u4fdd\u6301\u6587\u4ef6AccessTime\u4e0d\u53d8\u3002AccessTime\u7cbe\u5ea6\u8fc7\u5c0f\u4f1a\u5bfc\u81f4\u9891\u7e41\u66f4\u65b0\u6587\u4ef6\u7684AccessTime\uff0c\u5f71\u54cd\u6587\u4ef6\u7684\u8bfb\u53d6\u6027\u80fd\u3002 \u83b7\u53d6 AccessTime \u00a7 OSS-HDFS \u6587\u4ef6\u7684AccessTime \u53ea\u80fd\u901a\u8fc7\u5143\u6570\u636e\u5bfc\u51fa\u67e5\u770b\uff0c\u5143\u6570\u636e\u5bfc\u51fa\u529f\u80fd\u53c2\u8003 \u5143\u6570\u636e\u5bfc\u51fa .","title":"OSS-HDFS \u6587\u4ef6\u8bbf\u95ee\u65f6\u95f4\u6233"},{"location":"oss/dls_atime/#oss-hdfs-jindofs-accesstime","text":"","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09AccessTime\u4f7f\u7528\u8bf4\u660e"},{"location":"oss/dls_atime/#_1","text":"OSS-HDFS \u652f\u6301\u6587\u4ef6 AccessTime\uff0c \u6587\u4ef6 AccessTime \u7cbe\u5ea6\u9ed8\u8ba4\u914d\u7f6e\u4e3a1\u5c0f\u65f6\uff0c\u5177\u4f53\u914d\u7f6e\u9879\u4e3a namespace.file.accesstime.precision.millisecond = 3600000 AccessTime \u7cbe\u5ea6\u8868\u793a\u5982\u679c\u5f53\u524d\u65f6\u95f4\u548c\u6587\u4ef6\u5f53\u524d\u7684 AccessTime \u4e4b\u524d\u7684\u5dee\u503c\u8d85\u8fc7\u914d\u7f6e\u7684\u7cbe\u5ea6\u503c\uff0c\u5219\u66f4\u65b0\u6587\u4ef6\u7684AccessTime\uff0c\u5426\u5219\u4fdd\u6301\u6587\u4ef6AccessTime\u4e0d\u53d8\u3002AccessTime\u7cbe\u5ea6\u8fc7\u5c0f\u4f1a\u5bfc\u81f4\u9891\u7e41\u66f4\u65b0\u6587\u4ef6\u7684AccessTime\uff0c\u5f71\u54cd\u6587\u4ef6\u7684\u8bfb\u53d6\u6027\u80fd\u3002","title":"\u4ecb\u7ecd"},{"location":"oss/dls_atime/#accesstime","text":"OSS-HDFS \u6587\u4ef6\u7684AccessTime \u53ea\u80fd\u901a\u8fc7\u5143\u6570\u636e\u5bfc\u51fa\u67e5\u770b\uff0c\u5143\u6570\u636e\u5bfc\u51fa\u529f\u80fd\u53c2\u8003 \u5143\u6570\u636e\u5bfc\u51fa .","title":"\u83b7\u53d6 AccessTime"},{"location":"oss/dls_checksum/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c \u00a7 (\u4ece 4.6.2 \u5f00\u59cb\u652f\u6301) \u901a\u8fc7 JindoDistCp \u4ece HDFS \u8fc1\u79fb\u5230 OSS-HDFS \u7684\u6587\u4ef6\uff0c\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u6821\u9a8c\u6587\u4ef6\u5185\u5bb9\u3002 \u6b65\u9aa41. \u00a7 \u8ba1\u7b97\u8f93\u51fa\u901a\u8fc7 JindoDistCp \u8fc1\u79fb\u7684\u76ee\u6807\u76ee\u5f55\u7684\u6587\u4ef6 checksum\u3002 \u547d\u4ee4\u683c\u5f0f \u00a7 jindo distjob -checksum --src <src> --dest <dest> --blockSize <blockSize> --recalculate \u53c2\u6570\u8bf4\u660e\uff1a --src <src> \uff1a\u5f85\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c\u7684 OSS-HDFS \u7684\u8def\u5f84\uff08\u5fc5\u586b\uff09\u3002\u6ce8\uff1asrc \u53ea\u80fd\u4e3a OSS-HDFS \u6216 HDFS \u8def\u5f84\u3002 --dest <dest> \uff1a\u8f93\u51fa checksum \u6587\u4ef6\u7684\u76ee\u6807\u8def\u5f84\uff08\u5fc5\u586b\uff09\u3002 --blockSize <blockSize> \uff1a\u6e90\u6587\u4ef6\u5199\u5165\u7684 blockSize\uff08\u53ef\u9009)\uff0c\u9ed8\u8ba4\u4e3a 134217728\u3002 --recalculate \uff1a\u662f\u5426\u8bfb\u53d6\u6587\u4ef6\u91cd\u65b0\u8ba1\u7b97 checksum\uff08\u53ef\u9009)\uff0c\u9ed8\u8ba4\u4e0d\u4f1a\u91cd\u65b0\u8ba1\u7b97\uff0c\u4ece OSS-HDFS \u8bfb\u53d6\u5199\u5165\u65f6\u8ba1\u7b97\u7684 checksum\u3002\u6ce8\uff1a\u8be5\u53c2\u6570\u53ea\u652f\u6301 --src \u4e3a OSS-HDFS\u3002 \u793a\u4f8b \u00a7 \u5047\u8bbe\u5728 OSS-HDFS \u7684 bucket \u540d\u4e3a dlsbucket\uff0c\u9700\u8981\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c\u7684\u76ee\u5f55\u4e3a oss://dlsbucket/test\uff0c\u8f93\u51fa checksum \u6587\u4ef6\u5230 OSS \u76ee\u5f55 oss://ossbucket/test-dls\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u547d\u4ee4\u4e3a\uff1a jindo distjob -checksum --src oss://dlsbucket/test --dest oss://ossbucket/test-dls --recalculate \u6b65\u9aa42. \u00a7 \u8ba1\u7b97\u8f93\u51fa JindoDistCp \u8fc1\u79fb\u7684\u6e90\u76ee\u5f55\u7684\u6587\u4ef6 checksum\u3002 \u793a\u4f8b \u00a7 \u5047\u8bbe JindoDistCp \u7684\u6e90\u76ee\u5f55\u4e3a hdfs:///test\uff0c\u8f93\u51fa checksum \u6587\u4ef6\u5230 OSS \u76ee\u5f55 oss://ossbucket/test-hdfs\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u547d\u4ee4\u4e3a\uff1a jindo distjob -checksum --src hdfs:///test --dest oss://ossbucket/test-hdfs \u6b65\u9aa43. \u00a7 \u901a\u8fc7 JindoDistCp --diff \u5bf9\u6bd4\u6b65\u9aa41\u548c\u6b65\u9aa42\u8f93\u51fa\u7684\u6587\u4ef6 checksum \u76ee\u5f55 \u793a\u4f8b \u00a7 hadoop jar jindo-distcp-tool-${version}.jar --src oss://ossbucket/test-hdfs --dest oss://ossbucket/test-dls --diff","title":"OSS-HDFS \u6587\u4ef6\u6821\u9a8c"},{"location":"oss/dls_checksum/#oss-hdfs-jindofs","text":"(\u4ece 4.6.2 \u5f00\u59cb\u652f\u6301) \u901a\u8fc7 JindoDistCp \u4ece HDFS \u8fc1\u79fb\u5230 OSS-HDFS \u7684\u6587\u4ef6\uff0c\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u6821\u9a8c\u6587\u4ef6\u5185\u5bb9\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c"},{"location":"oss/dls_checksum/#1","text":"\u8ba1\u7b97\u8f93\u51fa\u901a\u8fc7 JindoDistCp \u8fc1\u79fb\u7684\u76ee\u6807\u76ee\u5f55\u7684\u6587\u4ef6 checksum\u3002","title":"\u6b65\u9aa41."},{"location":"oss/dls_checksum/#_1","text":"jindo distjob -checksum --src <src> --dest <dest> --blockSize <blockSize> --recalculate \u53c2\u6570\u8bf4\u660e\uff1a --src <src> \uff1a\u5f85\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c\u7684 OSS-HDFS \u7684\u8def\u5f84\uff08\u5fc5\u586b\uff09\u3002\u6ce8\uff1asrc \u53ea\u80fd\u4e3a OSS-HDFS \u6216 HDFS \u8def\u5f84\u3002 --dest <dest> \uff1a\u8f93\u51fa checksum \u6587\u4ef6\u7684\u76ee\u6807\u8def\u5f84\uff08\u5fc5\u586b\uff09\u3002 --blockSize <blockSize> \uff1a\u6e90\u6587\u4ef6\u5199\u5165\u7684 blockSize\uff08\u53ef\u9009)\uff0c\u9ed8\u8ba4\u4e3a 134217728\u3002 --recalculate \uff1a\u662f\u5426\u8bfb\u53d6\u6587\u4ef6\u91cd\u65b0\u8ba1\u7b97 checksum\uff08\u53ef\u9009)\uff0c\u9ed8\u8ba4\u4e0d\u4f1a\u91cd\u65b0\u8ba1\u7b97\uff0c\u4ece OSS-HDFS \u8bfb\u53d6\u5199\u5165\u65f6\u8ba1\u7b97\u7684 checksum\u3002\u6ce8\uff1a\u8be5\u53c2\u6570\u53ea\u652f\u6301 --src \u4e3a OSS-HDFS\u3002","title":"\u547d\u4ee4\u683c\u5f0f"},{"location":"oss/dls_checksum/#_2","text":"\u5047\u8bbe\u5728 OSS-HDFS \u7684 bucket \u540d\u4e3a dlsbucket\uff0c\u9700\u8981\u6587\u4ef6\u5185\u5bb9\u6821\u9a8c\u7684\u76ee\u5f55\u4e3a oss://dlsbucket/test\uff0c\u8f93\u51fa checksum \u6587\u4ef6\u5230 OSS \u76ee\u5f55 oss://ossbucket/test-dls\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u547d\u4ee4\u4e3a\uff1a jindo distjob -checksum --src oss://dlsbucket/test --dest oss://ossbucket/test-dls --recalculate","title":"\u793a\u4f8b"},{"location":"oss/dls_checksum/#2","text":"\u8ba1\u7b97\u8f93\u51fa JindoDistCp \u8fc1\u79fb\u7684\u6e90\u76ee\u5f55\u7684\u6587\u4ef6 checksum\u3002","title":"\u6b65\u9aa42."},{"location":"oss/dls_checksum/#_3","text":"\u5047\u8bbe JindoDistCp \u7684\u6e90\u76ee\u5f55\u4e3a hdfs:///test\uff0c\u8f93\u51fa checksum \u6587\u4ef6\u5230 OSS \u76ee\u5f55 oss://ossbucket/test-hdfs\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u547d\u4ee4\u4e3a\uff1a jindo distjob -checksum --src hdfs:///test --dest oss://ossbucket/test-hdfs","title":"\u793a\u4f8b"},{"location":"oss/dls_checksum/#3","text":"\u901a\u8fc7 JindoDistCp --diff \u5bf9\u6bd4\u6b65\u9aa41\u548c\u6b65\u9aa42\u8f93\u51fa\u7684\u6587\u4ef6 checksum \u76ee\u5f55","title":"\u6b65\u9aa43."},{"location":"oss/dls_checksum/#_4","text":"hadoop jar jindo-distcp-tool-${version}.jar --src oss://ossbucket/test-hdfs --dest oss://ossbucket/test-dls --diff","title":"\u793a\u4f8b"},{"location":"oss/dls_dump_inventory_howto/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5143\u6570\u636e\u5bfc\u51fa\u4f7f\u7528\u8bf4\u660e \u00a7 (\u4ece 4.6.0 \u5f00\u59cb\u652f\u6301) \u4ecb\u7ecd \u00a7 \u4f7f\u7528\u5143\u6570\u636e\u5bfc\u51fa\u529f\u80fd\uff0c\u53ef\u4ee5\u5c06\u5f53\u524d OSS-HDFS bucket \u4e0b\u7684\u6587\u4ef6\u5143\u6570\u636e\u6e05\u5355\u5bfc\u51fa\u5230 OSS \u4e0a\u7684\uff0c\u683c\u5f0f\u4e3a json \u6587\u4ef6\uff0c\u65b9\u4fbf\u7528\u6237\u5bf9\u5143\u6570\u636e\u8fdb\u884c\u7edf\u8ba1\u5206\u6790 \u914d\u7f6e JindoFS \u547d\u4ee4\u884c\u5de5\u5177\uff0c\u914d\u7f6e\u5bf9\u5e94 OSS-HDFS bucket \u7684\u8bbf\u95ee\u5bc6\u94a5\uff0c\u53c2\u8003 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u8bf4\u660e \u6267\u884c\u5bfc\u51fa\u547d\u4ee4 ./jindofs admin -dumpInventory oss://<hdfs_bucket>/ \u6b64\u65f6\u53ef\u4ee5\u89c2\u5bdf\u5230\u8f93\u51fa\u8def\u5f84 ============Dump Inventory============= Job Id: 2ce40fba-5704-45c4-8720-d92a891d5cfd Data Location: oss://<oss_bucket>/.dlsdata/.sysinfo/meta_analyze/inventory/1666584461201.2ce40fba-5704-45c4-8720-d92a891d5cfd ..................................................................................................................... FINISHED. \u8be5\u547d\u4ee4\u4e3a\u963b\u585e\u547d\u4ee4\uff0c\u8bf7\u8010\u5fc3\u7b49\u5f8510\u79d2\u949f~10\u5206\u949f\uff08\u6839\u636e\u5143\u6570\u636e\u91cf\u5927\u5c0f\uff09\uff0c\u77e5\u9053\u6700\u540e\u8f93\u51fa FINISHED \u8868\u793a\u5bfc\u51fa\u6210\u529f\u3002 \u4e0b\u8f7d\u7ed3\u679c\u6587\u4ef6 \u4f7f\u7528 jindo \u547d\u4ee4\u884c\u5de5\u5177 \uff08\u914d\u7f6e\u666e\u901aOSS\u7684key\u975eOSS-HDFS\uff09 \u3001\u6216 ossutil \u3001\u6216\u4f7f\u7528Hadoop fs \u547d\u4ee4\u3001\u6216\u5728OSS\u63a7\u5236\u53f0\u9875\u9762\uff0c\u4e0b\u8f7d\u7ed3\u679c\u6587\u4ef6\u3002 ossutil cp oss://<oss_bucket>/.dlsdata/.sysinfo/meta_analyze/inventory/1666584461201.2ce40fba-5704-45c4-8720-d92a891d5cfd ./ \u4e0b\u8f7d\u5230\u672c\u5730\uff0c\u4f7f\u7528vi/vim\u6253\u5f00\u5373\u53ef\u3002 \u793a\u4f8b\u7ed3\u679c\u53c2\u8003 {\"id\":16385,\"path\":\"/\",\"type\":\"directory\",\"size\":0,\"user\":\"admin\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1666581702933,\"permission\":511,\"state\":1} {\"id\":6246684106789500068,\"path\":\"/dls-1000326249\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500069,\"path\":\"/dls-1000326249/benchmark\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500070,\"path\":\"/dls-1000326249/benchmark/n1\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500071,\"path\":\"/dls-1000326249/benchmark/n1/490747449\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660895613953,\"permission\":511,\"state\":0}","title":"OSS-HDFS \u6e05\u5355\u5bfc\u51fa"},{"location":"oss/dls_dump_inventory_howto/#oss-hdfs-jindofs","text":"(\u4ece 4.6.0 \u5f00\u59cb\u652f\u6301)","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5143\u6570\u636e\u5bfc\u51fa\u4f7f\u7528\u8bf4\u660e"},{"location":"oss/dls_dump_inventory_howto/#_1","text":"\u4f7f\u7528\u5143\u6570\u636e\u5bfc\u51fa\u529f\u80fd\uff0c\u53ef\u4ee5\u5c06\u5f53\u524d OSS-HDFS bucket \u4e0b\u7684\u6587\u4ef6\u5143\u6570\u636e\u6e05\u5355\u5bfc\u51fa\u5230 OSS \u4e0a\u7684\uff0c\u683c\u5f0f\u4e3a json \u6587\u4ef6\uff0c\u65b9\u4fbf\u7528\u6237\u5bf9\u5143\u6570\u636e\u8fdb\u884c\u7edf\u8ba1\u5206\u6790 \u914d\u7f6e JindoFS \u547d\u4ee4\u884c\u5de5\u5177\uff0c\u914d\u7f6e\u5bf9\u5e94 OSS-HDFS bucket \u7684\u8bbf\u95ee\u5bc6\u94a5\uff0c\u53c2\u8003 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u8bf4\u660e \u6267\u884c\u5bfc\u51fa\u547d\u4ee4 ./jindofs admin -dumpInventory oss://<hdfs_bucket>/ \u6b64\u65f6\u53ef\u4ee5\u89c2\u5bdf\u5230\u8f93\u51fa\u8def\u5f84 ============Dump Inventory============= Job Id: 2ce40fba-5704-45c4-8720-d92a891d5cfd Data Location: oss://<oss_bucket>/.dlsdata/.sysinfo/meta_analyze/inventory/1666584461201.2ce40fba-5704-45c4-8720-d92a891d5cfd ..................................................................................................................... FINISHED. \u8be5\u547d\u4ee4\u4e3a\u963b\u585e\u547d\u4ee4\uff0c\u8bf7\u8010\u5fc3\u7b49\u5f8510\u79d2\u949f~10\u5206\u949f\uff08\u6839\u636e\u5143\u6570\u636e\u91cf\u5927\u5c0f\uff09\uff0c\u77e5\u9053\u6700\u540e\u8f93\u51fa FINISHED \u8868\u793a\u5bfc\u51fa\u6210\u529f\u3002 \u4e0b\u8f7d\u7ed3\u679c\u6587\u4ef6 \u4f7f\u7528 jindo \u547d\u4ee4\u884c\u5de5\u5177 \uff08\u914d\u7f6e\u666e\u901aOSS\u7684key\u975eOSS-HDFS\uff09 \u3001\u6216 ossutil \u3001\u6216\u4f7f\u7528Hadoop fs \u547d\u4ee4\u3001\u6216\u5728OSS\u63a7\u5236\u53f0\u9875\u9762\uff0c\u4e0b\u8f7d\u7ed3\u679c\u6587\u4ef6\u3002 ossutil cp oss://<oss_bucket>/.dlsdata/.sysinfo/meta_analyze/inventory/1666584461201.2ce40fba-5704-45c4-8720-d92a891d5cfd ./ \u4e0b\u8f7d\u5230\u672c\u5730\uff0c\u4f7f\u7528vi/vim\u6253\u5f00\u5373\u53ef\u3002 \u793a\u4f8b\u7ed3\u679c\u53c2\u8003 {\"id\":16385,\"path\":\"/\",\"type\":\"directory\",\"size\":0,\"user\":\"admin\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1666581702933,\"permission\":511,\"state\":1} {\"id\":6246684106789500068,\"path\":\"/dls-1000326249\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500069,\"path\":\"/dls-1000326249/benchmark\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500070,\"path\":\"/dls-1000326249/benchmark/n1\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660889124590,\"permission\":511,\"state\":0} {\"id\":6246684106789500071,\"path\":\"/dls-1000326249/benchmark/n1/490747449\",\"type\":\"directory\",\"size\":0,\"user\":\"hadoop\",\"group\":\"supergroup\",\"atime\":0,\"mtime\":1660895613953,\"permission\":511,\"state\":0}","title":"\u4ecb\u7ecd"},{"location":"oss/dls_protected_directories/","text":"\u80cc\u666f \u00a7 JindoFS \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u5f53\u524d\u53ef\u4ee5\u901a\u8fc7\u547d\u4ee4\u884c\u547d\u4ee4\u7684\u65b9\u5f0f\u8fdb\u884c\u64cd\u4f5c\uff0c\u53ef\u4ee5\u663e\u793a\u6b63\u5728\u4fdd\u62a4\u7684\u76ee\u5f55\u5217\u8868\uff0c\u6216\u8005\u4fee\u6539\u4fdd\u62a4\u7684\u76ee\u5f55\u3002\u5c06\u6765\u8fd8\u4f1a\u901a\u8fc7\u4ea7\u54c1\u754c\u9762\u7684\u65b9\u5f0f\uff0c\u63d0\u4f9b\u66f4\u52a0\u4fbf\u6377\u6613\u7528\u7684\u67e5\u770b\u548c\u8bbe\u7f6e\u65b9\u6cd5\u3002 \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u63cf\u8ff0 \u00a7 JindoFS \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\uff0c\u4e3b\u8981\u9632\u6b62\u7528\u6237\u8bef\u5220\u76ee\u5f55\uff0c\u88ab\u4fdd\u62a4\u7684\u76ee\u5f55\u53ea\u8981\u8be5\u76ee\u5f55\u4e0b\u5b58\u5728\u76ee\u5f55\u6216\u8005\u6587\u4ef6\uff0c\u4fdd\u62a4\u76ee\u5f55\u7684\u5220\u9664\u4ee5\u53ca\u91cd\u547d\u540d\u64cd\u4f5c\u4f1a\u62a5\u9519\uff0c\u53ea\u6709\u5f53\u88ab\u4fdd\u62a4\u7684\u76ee\u5f55\u4e0b\u4e0d\u5b58\u5728\u4efb\u4f55\u76ee\u5f55\u6216\u8005\u6587\u4ef6\uff0c\u4fdd\u62a4\u76ee\u5f55\u624d\u4f1a\u88ab\u5141\u8bb8\u5220\u9664\u6216\u8005\u91cd\u547d\u540d\u3002 JindoFS \u547d\u4ee4\u884c\u7b80\u4ecb \u00a7 \u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u9700\u8981\u7528\u5230\u7684\u662f -putConfig \u4e0e -getConfig \u547d\u4ee4\uff0c\u5177\u4f53\u7528\u6cd5\u5c06\u5728\u4e0b\u6587\u4ecb\u7ecd\u3002 \u914d\u7f6e\u8bbf\u95ee AK \u00a7 \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u9700\u8981\u914d\u7f6e\u8fde\u63a5 OSS-HDFS \u6240\u9700\u7684 AK \u4fe1\u606f\u3002\u6709\u4e24\u79cd\u65b9\u6cd5\u53ef\u4ee5\u8fdb\u884c\u914d\u7f6e\uff1a \u914d\u7f6e\u6587\u4ef6\uff08\u63a8\u8350\uff09 \u00a7 \u8bbe\u7f6e\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6307\u5411\u67d0\u4e2a\u76ee\u5f55\uff0c\u7136\u540e\u5728\u76ee\u5f55\u4e0b\u521b\u5efa\u540d\u4e3a jindofs.cfg \u7684\u6587\u4ef6\uff0c\u7136\u540e\u5728\u6587\u4ef6\u91cc\u5199\u5165\u5982\u4e0b\u5185\u5bb9\uff1a [client] fs.oss.accessKeyId = <key> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK key fs.oss.accessKeySecret = <secret> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK secret fs.oss.bucket.<bucket>.accessKeyId = <key> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK key\uff0c\u4f18\u5148\u7ea7\u9ad8\u4e8e\u9ed8\u8ba4 fs.oss.bucket.<bucket>.accessKeySecret = <secret> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK secret --extraConf \u00a7 \u547d\u4ee4\u884c\u5de5\u5177\u7684\u6240\u6709\u6307\u4ee4\u5747\u53ef\u901a\u8fc7 --extraConf \u9009\u9879\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u914d\u7f6e\u6587\u4ef6\uff0c\u4f8b\u5982\uff1a jindofs admin -getConfig -dlsUri <path> -name <keys> \\ --extraConf fs.oss.accessKeyId=<AK key> \\ --extraConf fs.oss.accessKeySecret=<AK secret> \u5c06\u7ed9\u547d\u4ee4\u63d0\u4f9b\u989d\u5916\u7684 AK key \u4e0e AK secret \u914d\u7f6e\u3002 \u8bbe\u7f6e\u4fdd\u62a4\u76ee\u5f55 \u00a7 \u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf fs.protected.directories=/path/to/dir1,/path/to/dir2 \u4e0a\u8ff0\u547d\u4ee4\u5bf9\u540d\u4e3a \uff08endpoint \u4e3a \uff09\u7684\u6876\u914d\u7f6e\u4e86\u4e24\u4e2a\u4fdd\u62a4\u76ee\u5f55\uff1a /path/to/dir1 /path/to/dir2 \u5982\u793a\u4f8b\u4e2d\u7684\u683c\u5f0f\uff0c\u8fd9\u4e9b\u4fdd\u62a4\u76ee\u5f55\u7684\u8def\u5f84\u9700\u8981\u5199\u5728\u4e00\u8d77\uff0c\u9017\u53f7\u9694\u5f00\u3002\u8def\u5f84\u4e0d\u4ee5 oss:// \u5f00\u5934\uff0c\u800c\u662f\u4f7f\u7528\u6876\u5185\u4ee5 / \u5f00\u5934\u7684\u7edd\u5bf9\u8def\u5f84\u3002\u5982\u679c\u547d\u4ee4\u5931\u8d25\uff0c\u5c06\u663e\u793a\u5931\u8d25\u4fe1\u606f\u3002\u5982\u679c\u6ca1\u6709\u4efb\u4f55\u4fe1\u606f\u663e\u793a\uff0c\u8868\u660e\u6267\u884c\u6210\u529f\u3002\u547d\u4ee4\u6267\u884c\u6210\u529f\u4e4b\u540e\uff0c\u5bf9\u8fd9\u4e9b\u76ee\u5f55\u7684\u4fdd\u62a4\u529f\u80fd\u5c06\u5728 30 \u79d2\u5185\u6b63\u5f0f\u751f\u6548\u3002 \u67e5\u770b\u4fdd\u62a4\u76ee\u5f55 \u00a7 \u547d\u4ee4\uff1a jindofs admin -getConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -name fs.protected.directories \u4e0a\u8ff0\u547d\u4ee4\u8fd4\u56de\u4e86\u540d\u4e3a \uff08endpoint \u4e3a \uff09\u7684\u6876\u7684\u4fdd\u62a4\u76ee\u5f55\u5217\u8868\uff0c\u8fd4\u56de\u503c\u793a\u4f8b\uff1a fs.protected.directories: /path/to/dir1,/path/to/dir2 \u8868\u660e\u8fd9\u4e9b\u76ee\u5f55\u76ee\u524d\u6b63\u5728\u4fdd\u62a4\u5f53\u4e2d\uff1a /path/to/dir1 /path/to/dir2 \u5220\u9664\u6240\u6709\u4fdd\u62a4\u76ee\u5f55 \u00a7 \u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf fs.protected.directories= \u4e0a\u8ff0\u547d\u4ee4\u5220\u9664\u4e86\u540d\u4e3a \uff08endpoint \u4e3a \uff09\u7684\u6876\u7684\u6240\u6709\u4fdd\u62a4\u76ee\u5f55\uff0c\u6b63\u5f0f\u751f\u6548\u540e\uff08\u547d\u4ee4\u8fd4\u56de\u540e 30 \u79d2\u5185\uff09\uff0c\u8be5\u6876\u5c06\u4e0d\u518d\u6709\u4efb\u4f55\u76ee\u5f55\u53d7\u5230\u4fdd\u62a4\u3002","title":"OSS-HDFS \u53d7\u4fdd\u62a4\u76ee\u5f55"},{"location":"oss/dls_protected_directories/#_1","text":"JindoFS \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u5f53\u524d\u53ef\u4ee5\u901a\u8fc7\u547d\u4ee4\u884c\u547d\u4ee4\u7684\u65b9\u5f0f\u8fdb\u884c\u64cd\u4f5c\uff0c\u53ef\u4ee5\u663e\u793a\u6b63\u5728\u4fdd\u62a4\u7684\u76ee\u5f55\u5217\u8868\uff0c\u6216\u8005\u4fee\u6539\u4fdd\u62a4\u7684\u76ee\u5f55\u3002\u5c06\u6765\u8fd8\u4f1a\u901a\u8fc7\u4ea7\u54c1\u754c\u9762\u7684\u65b9\u5f0f\uff0c\u63d0\u4f9b\u66f4\u52a0\u4fbf\u6377\u6613\u7528\u7684\u67e5\u770b\u548c\u8bbe\u7f6e\u65b9\u6cd5\u3002","title":"\u80cc\u666f"},{"location":"oss/dls_protected_directories/#_2","text":"JindoFS \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\uff0c\u4e3b\u8981\u9632\u6b62\u7528\u6237\u8bef\u5220\u76ee\u5f55\uff0c\u88ab\u4fdd\u62a4\u7684\u76ee\u5f55\u53ea\u8981\u8be5\u76ee\u5f55\u4e0b\u5b58\u5728\u76ee\u5f55\u6216\u8005\u6587\u4ef6\uff0c\u4fdd\u62a4\u76ee\u5f55\u7684\u5220\u9664\u4ee5\u53ca\u91cd\u547d\u540d\u64cd\u4f5c\u4f1a\u62a5\u9519\uff0c\u53ea\u6709\u5f53\u88ab\u4fdd\u62a4\u7684\u76ee\u5f55\u4e0b\u4e0d\u5b58\u5728\u4efb\u4f55\u76ee\u5f55\u6216\u8005\u6587\u4ef6\uff0c\u4fdd\u62a4\u76ee\u5f55\u624d\u4f1a\u88ab\u5141\u8bb8\u5220\u9664\u6216\u8005\u91cd\u547d\u540d\u3002","title":"\u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u63cf\u8ff0"},{"location":"oss/dls_protected_directories/#jindofs","text":"\u53c2\u89c1 JindoFS \u547d\u4ee4\u884c\u5de5\u5177\u4f7f\u7528\u6307\u5357 \u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u9700\u8981\u7528\u5230\u7684\u662f -putConfig \u4e0e -getConfig \u547d\u4ee4\uff0c\u5177\u4f53\u7528\u6cd5\u5c06\u5728\u4e0b\u6587\u4ecb\u7ecd\u3002","title":"JindoFS \u547d\u4ee4\u884c\u7b80\u4ecb"},{"location":"oss/dls_protected_directories/#ak","text":"\u76ee\u5f55\u4fdd\u62a4\u529f\u80fd\u9700\u8981\u914d\u7f6e\u8fde\u63a5 OSS-HDFS \u6240\u9700\u7684 AK \u4fe1\u606f\u3002\u6709\u4e24\u79cd\u65b9\u6cd5\u53ef\u4ee5\u8fdb\u884c\u914d\u7f6e\uff1a","title":"\u914d\u7f6e\u8bbf\u95ee AK"},{"location":"oss/dls_protected_directories/#_3","text":"\u8bbe\u7f6e\u73af\u5883\u53d8\u91cf JINDOSDK_CONF_DIR \u6307\u5411\u67d0\u4e2a\u76ee\u5f55\uff0c\u7136\u540e\u5728\u76ee\u5f55\u4e0b\u521b\u5efa\u540d\u4e3a jindofs.cfg \u7684\u6587\u4ef6\uff0c\u7136\u540e\u5728\u6587\u4ef6\u91cc\u5199\u5165\u5982\u4e0b\u5185\u5bb9\uff1a [client] fs.oss.accessKeyId = <key> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK key fs.oss.accessKeySecret = <secret> # \u5bf9\u6240\u6709 Bucket \u751f\u6548\u7684\u9ed8\u8ba4 AK secret fs.oss.bucket.<bucket>.accessKeyId = <key> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK key\uff0c\u4f18\u5148\u7ea7\u9ad8\u4e8e\u9ed8\u8ba4 fs.oss.bucket.<bucket>.accessKeySecret = <secret> # \u4ec5\u5bf9\u67d0 Bucket \u751f\u6548\u7684 AK secret","title":"\u914d\u7f6e\u6587\u4ef6\uff08\u63a8\u8350\uff09"},{"location":"oss/dls_protected_directories/#-extraconf","text":"\u547d\u4ee4\u884c\u5de5\u5177\u7684\u6240\u6709\u6307\u4ee4\u5747\u53ef\u901a\u8fc7 --extraConf \u9009\u9879\u63d0\u4f9b\u914d\u7f6e\u53c2\u6570\uff0c\u6548\u679c\u7b49\u540c\u4e8e\u914d\u7f6e\u6587\u4ef6\uff0c\u4f8b\u5982\uff1a jindofs admin -getConfig -dlsUri <path> -name <keys> \\ --extraConf fs.oss.accessKeyId=<AK key> \\ --extraConf fs.oss.accessKeySecret=<AK secret> \u5c06\u7ed9\u547d\u4ee4\u63d0\u4f9b\u989d\u5916\u7684 AK key \u4e0e AK secret \u914d\u7f6e\u3002","title":"--extraConf"},{"location":"oss/dls_protected_directories/#_4","text":"\u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf fs.protected.directories=/path/to/dir1,/path/to/dir2 \u4e0a\u8ff0\u547d\u4ee4\u5bf9\u540d\u4e3a \uff08endpoint \u4e3a \uff09\u7684\u6876\u914d\u7f6e\u4e86\u4e24\u4e2a\u4fdd\u62a4\u76ee\u5f55\uff1a /path/to/dir1 /path/to/dir2 \u5982\u793a\u4f8b\u4e2d\u7684\u683c\u5f0f\uff0c\u8fd9\u4e9b\u4fdd\u62a4\u76ee\u5f55\u7684\u8def\u5f84\u9700\u8981\u5199\u5728\u4e00\u8d77\uff0c\u9017\u53f7\u9694\u5f00\u3002\u8def\u5f84\u4e0d\u4ee5 oss:// \u5f00\u5934\uff0c\u800c\u662f\u4f7f\u7528\u6876\u5185\u4ee5 / \u5f00\u5934\u7684\u7edd\u5bf9\u8def\u5f84\u3002\u5982\u679c\u547d\u4ee4\u5931\u8d25\uff0c\u5c06\u663e\u793a\u5931\u8d25\u4fe1\u606f\u3002\u5982\u679c\u6ca1\u6709\u4efb\u4f55\u4fe1\u606f\u663e\u793a\uff0c\u8868\u660e\u6267\u884c\u6210\u529f\u3002\u547d\u4ee4\u6267\u884c\u6210\u529f\u4e4b\u540e\uff0c\u5bf9\u8fd9\u4e9b\u76ee\u5f55\u7684\u4fdd\u62a4\u529f\u80fd\u5c06\u5728 30 \u79d2\u5185\u6b63\u5f0f\u751f\u6548\u3002","title":"\u8bbe\u7f6e\u4fdd\u62a4\u76ee\u5f55"},{"location":"oss/dls_protected_directories/#_5","text":"\u547d\u4ee4\uff1a jindofs admin -getConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -name fs.protected.directories \u4e0a\u8ff0\u547d\u4ee4\u8fd4\u56de\u4e86\u540d\u4e3a \uff08endpoint \u4e3a \uff09\u7684\u6876\u7684\u4fdd\u62a4\u76ee\u5f55\u5217\u8868\uff0c\u8fd4\u56de\u503c\u793a\u4f8b\uff1a fs.protected.directories: /path/to/dir1,/path/to/dir2 \u8868\u660e\u8fd9\u4e9b\u76ee\u5f55\u76ee\u524d\u6b63\u5728\u4fdd\u62a4\u5f53\u4e2d\uff1a /path/to/dir1 /path/to/dir2","title":"\u67e5\u770b\u4fdd\u62a4\u76ee\u5f55"},{"location":"oss/dls_protected_directories/#_6","text":"\u547d\u4ee4\uff1a jindofs admin -putConfig -dlsUri oss://<bucket>.<oss-hdfs-endpoint>/ -conf fs.protected.directories= \u4e0a\u8ff0\u547d\u4ee4\u5220\u9664\u4e86\u540d\u4e3a \uff08endpoint \u4e3a \uff09\u7684\u6876\u7684\u6240\u6709\u4fdd\u62a4\u76ee\u5f55\uff0c\u6b63\u5f0f\u751f\u6548\u540e\uff08\u547d\u4ee4\u8fd4\u56de\u540e 30 \u79d2\u5185\uff09\uff0c\u8be5\u6876\u5c06\u4e0d\u518d\u6709\u4efb\u4f55\u76ee\u5f55\u53d7\u5230\u4fdd\u62a4\u3002","title":"\u5220\u9664\u6240\u6709\u4fdd\u62a4\u76ee\u5f55"},{"location":"oss/dls_root_policy/","text":"\u4f7f\u7528 RootPolicy \u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09 \u00a7 (\u4ece 4.6.0 \u5f00\u59cb\u652f\u6301) \u80cc\u666f \u00a7 JindoFS \u652f\u6301 RootPolicy \u89c4\u5219\uff0c\u901a\u8fc7 RootPolicy \u53ef\u4ee5\u4e3a OSS-HDFS\uff08JindoFS \u670d\u52a1\uff09\u8bbe\u7f6e\u81ea\u5b9a\u4e49\u524d\u7f00\uff0c\u8fd9\u6837\u7528\u6237\u539f\u6709\u7684\u8bbf\u95ee hdfs:// \u524d\u7f00\u7684\u4f5c\u4e1a\uff0c\u53ef\u4ee5\u4e0d\u7ecf\u4fee\u6539\u76f4\u63a5\u8fd0\u884c\u5728 JindoFS \u670d\u52a1\u4e0a\u3002 1\u3001 \u8bbe\u7f6e RootPolicy \u00a7 \u6211\u4eec\u63d0\u4f9b\u4e86\u4e00\u7ec4 Shell \u547d\u4ee4\uff0c\u5e2e\u52a9\u7528\u6237\u5411\u7279\u5b9a\u7684bucket\u6ce8\u518c\u81ea\u5b9a\u4e49\u5730\u5740\u3002\u901a\u8fc7 SetRootPolicy\u547d\u4ee4 \uff0c\u53ef\u4ee5\u4e3a\u7279\u5b9a bucket \u6ce8\u518c\u81ea\u5b9a\u4e49\u524d\u7f00\u7684\u8bbf\u95ee\u5730\u5740\u3002\u5982\uff1a ./jindofs admin -setRootPolicy oss://<bucket_name>.<dls_endpoint>/ hdfs://<your_ns_name>/ : OSS-HDFS \u670d\u52a1 bucket \u7684\u540d\u5b57\uff0c\u5f53\u524d\u7248\u672c\u4ec5\u652f\u6301\u6839\u76ee\u5f55\u3002 : OSS-HDFS \u670d\u52a1\u7684 endpoint\uff0c\u5982\u676d\u5dderegion\u4e3a cn-hangzhou.oss-dls.aliyuncs.com \u3002\u5982\u679c Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u6709\u5982\u4e0b\u914d\u7f6e\u9879 <configuration> <property> <name>fs.oss.endpoint</name> <value><dls_endpoint></value> </property> \u6216 <property> <name>fs.oss.bucket.<bucket_name>.endpoint</name> <value><dls_endpoint></value> </property> </configuration> \u4e0a\u9762\u4f8b\u5b50\u4e2d\u7684 oss://<bucket_name>.<dls_endpoint>/ \u53ef\u4ee5\u5199\u6210 oss://<bucket_name>/ \u7684\u5f62\u5f0f\uff0c\u4e0b\u6587\u4e2d\u9700\u8981 endpoint \u7684\u5730\u65b9\u90fd\u53ef\u7701\u7565\u4e0d\u586b\u3002 * : \u81ea\u5b9a\u4e49\u8bbf\u95ee hdfs \u670d\u52a1\u7684 nsname\uff0c\u4e3a\u4efb\u610f\u975e\u7a7a\u5b57\u7b26\u4e32\uff0c\u5982 test \uff0c\u5f53\u524d\u7248\u672c\u4ec5\u652f\u6301\u6839\u76ee\u5f55\u3002 2\u3001 \u914d\u7f6e access policy \u53d1\u73b0\u5730\u5740 \u548c scheme \u5b9e\u73b0\u7c7b \u00a7 \u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u6dfb\u52a0\u5982\u4e0b\u914d\u7f6e\u9879 <configuration> <property> <name>fs.accessPolicies.discovery</name> <value>oss://<bucket_name>.<dls_endpoint>/</value> </property> <property> <name>fs.AbstractFileSystem.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.HDFS</value> </property> <property> <name>fs.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.JindoHdfsFileSystem</value> </property> </configuration> \u5982\u679c\u662f\u591a\u4e2abucket\uff0c\u7528 , \u5206\u9694\u3002 3\u3001 \u9a8c\u8bc1\u548c\u4f7f\u7528 \u00a7 \u914d\u7f6e\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u5bf9\u914d\u7f6e\u8fdb\u884c\u7b80\u5355\u9a8c\u8bc1\u3002 hadoop fs -ls hdfs://<your_ns_name>/ \u9a8c\u8bc1\u901a\u8fc7\u540e\uff0c\u91cd\u542f\u76f8\u5173\u670d\u52a1\uff0c\u5982 Hive/Spark \u7b49\uff0c\u5373\u53ef\u4f7f\u7528\u81ea\u5b9a\u4e49\u524d\u7f00\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u3002 4\u3001\u5176\u4ed6\u529f\u80fd \u00a7 4.1 \u5220\u9664 RootPolicy \u89c4\u5219 \u00a7 \u901a\u8fc7 UnsetRootPolicy\u547d\u4ee4 \u53ef\u4ee5\u5220\u9664\u76f8\u5e94\u5730\u5740\u3002 ./jindofs admin -unsetRootPolicy oss://<bucket_name>.<dls_endpoint>/ hdfs://<your_ns_name>/ 4.2 \u5217\u4e3e\u6240\u6709 RootPolicy \u89c4\u5219 \u00a7 \u4f7f\u7528 ListAccessPolicies\u547d\u4ee4 \uff0c\u53ef\u4ee5\u5217\u51fa\u7279\u5b9a bucket \u5f53\u524d\u6ce8\u518c\u7684\u6240\u6709\u524d\u7f00\u5730\u5740\u3002 ./jindofs admin -listAccessPolicies oss://<bucket_name>.<dls_endpoint>/","title":"OSS-HDFS \u6839\u8def\u5f84\u6539\u5199"},{"location":"oss/dls_root_policy/#rootpolicy-oss-hdfs-jindofs","text":"(\u4ece 4.6.0 \u5f00\u59cb\u652f\u6301)","title":"\u4f7f\u7528 RootPolicy \u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09"},{"location":"oss/dls_root_policy/#_1","text":"JindoFS \u652f\u6301 RootPolicy \u89c4\u5219\uff0c\u901a\u8fc7 RootPolicy \u53ef\u4ee5\u4e3a OSS-HDFS\uff08JindoFS \u670d\u52a1\uff09\u8bbe\u7f6e\u81ea\u5b9a\u4e49\u524d\u7f00\uff0c\u8fd9\u6837\u7528\u6237\u539f\u6709\u7684\u8bbf\u95ee hdfs:// \u524d\u7f00\u7684\u4f5c\u4e1a\uff0c\u53ef\u4ee5\u4e0d\u7ecf\u4fee\u6539\u76f4\u63a5\u8fd0\u884c\u5728 JindoFS \u670d\u52a1\u4e0a\u3002","title":"\u80cc\u666f"},{"location":"oss/dls_root_policy/#1-rootpolicy","text":"\u6211\u4eec\u63d0\u4f9b\u4e86\u4e00\u7ec4 Shell \u547d\u4ee4\uff0c\u5e2e\u52a9\u7528\u6237\u5411\u7279\u5b9a\u7684bucket\u6ce8\u518c\u81ea\u5b9a\u4e49\u5730\u5740\u3002\u901a\u8fc7 SetRootPolicy\u547d\u4ee4 \uff0c\u53ef\u4ee5\u4e3a\u7279\u5b9a bucket \u6ce8\u518c\u81ea\u5b9a\u4e49\u524d\u7f00\u7684\u8bbf\u95ee\u5730\u5740\u3002\u5982\uff1a ./jindofs admin -setRootPolicy oss://<bucket_name>.<dls_endpoint>/ hdfs://<your_ns_name>/ : OSS-HDFS \u670d\u52a1 bucket \u7684\u540d\u5b57\uff0c\u5f53\u524d\u7248\u672c\u4ec5\u652f\u6301\u6839\u76ee\u5f55\u3002 : OSS-HDFS \u670d\u52a1\u7684 endpoint\uff0c\u5982\u676d\u5dderegion\u4e3a cn-hangzhou.oss-dls.aliyuncs.com \u3002\u5982\u679c Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u6709\u5982\u4e0b\u914d\u7f6e\u9879 <configuration> <property> <name>fs.oss.endpoint</name> <value><dls_endpoint></value> </property> \u6216 <property> <name>fs.oss.bucket.<bucket_name>.endpoint</name> <value><dls_endpoint></value> </property> </configuration> \u4e0a\u9762\u4f8b\u5b50\u4e2d\u7684 oss://<bucket_name>.<dls_endpoint>/ \u53ef\u4ee5\u5199\u6210 oss://<bucket_name>/ \u7684\u5f62\u5f0f\uff0c\u4e0b\u6587\u4e2d\u9700\u8981 endpoint \u7684\u5730\u65b9\u90fd\u53ef\u7701\u7565\u4e0d\u586b\u3002 * : \u81ea\u5b9a\u4e49\u8bbf\u95ee hdfs \u670d\u52a1\u7684 nsname\uff0c\u4e3a\u4efb\u610f\u975e\u7a7a\u5b57\u7b26\u4e32\uff0c\u5982 test \uff0c\u5f53\u524d\u7248\u672c\u4ec5\u652f\u6301\u6839\u76ee\u5f55\u3002","title":"1\u3001 \u8bbe\u7f6e RootPolicy"},{"location":"oss/dls_root_policy/#2-access-policy-scheme","text":"\u5728 Hadoop \u7684 core-site.xml \u6587\u4ef6\u4e2d\u6dfb\u52a0\u5982\u4e0b\u914d\u7f6e\u9879 <configuration> <property> <name>fs.accessPolicies.discovery</name> <value>oss://<bucket_name>.<dls_endpoint>/</value> </property> <property> <name>fs.AbstractFileSystem.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.HDFS</value> </property> <property> <name>fs.hdfs.impl</name> <value>com.aliyun.jindodata.hdfs.JindoHdfsFileSystem</value> </property> </configuration> \u5982\u679c\u662f\u591a\u4e2abucket\uff0c\u7528 , \u5206\u9694\u3002","title":"2\u3001 \u914d\u7f6e access policy \u53d1\u73b0\u5730\u5740 \u548c scheme \u5b9e\u73b0\u7c7b"},{"location":"oss/dls_root_policy/#3","text":"\u914d\u7f6e\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u5bf9\u914d\u7f6e\u8fdb\u884c\u7b80\u5355\u9a8c\u8bc1\u3002 hadoop fs -ls hdfs://<your_ns_name>/ \u9a8c\u8bc1\u901a\u8fc7\u540e\uff0c\u91cd\u542f\u76f8\u5173\u670d\u52a1\uff0c\u5982 Hive/Spark \u7b49\uff0c\u5373\u53ef\u4f7f\u7528\u81ea\u5b9a\u4e49\u524d\u7f00\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u3002","title":"3\u3001 \u9a8c\u8bc1\u548c\u4f7f\u7528"},{"location":"oss/dls_root_policy/#4","text":"","title":"4\u3001\u5176\u4ed6\u529f\u80fd"},{"location":"oss/dls_root_policy/#41-rootpolicy","text":"\u901a\u8fc7 UnsetRootPolicy\u547d\u4ee4 \u53ef\u4ee5\u5220\u9664\u76f8\u5e94\u5730\u5740\u3002 ./jindofs admin -unsetRootPolicy oss://<bucket_name>.<dls_endpoint>/ hdfs://<your_ns_name>/","title":"4.1 \u5220\u9664 RootPolicy \u89c4\u5219"},{"location":"oss/dls_root_policy/#42-rootpolicy","text":"\u4f7f\u7528 ListAccessPolicies\u547d\u4ee4 \uff0c\u53ef\u4ee5\u5217\u51fa\u7279\u5b9a bucket \u5f53\u524d\u6ce8\u518c\u7684\u6240\u6709\u524d\u7f00\u5730\u5740\u3002 ./jindofs admin -listAccessPolicies oss://<bucket_name>.<dls_endpoint>/","title":"4.2 \u5217\u4e3e\u6240\u6709 RootPolicy \u89c4\u5219"},{"location":"oss/dls_snapshot_quickstart/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u7167\u4f7f\u7528\u8bf4\u660e \u00a7 (\u4ece JindoData 4.0.0 \u5f00\u59cb\u652f\u6301) \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u662fOSS\u65b0\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9HDFS\u63a5\u53e3, \u652f\u6301\u76ee\u5f55\u4ee5\u53ca\u76ee\u5f55\u5c42\u7ea7\uff0c\u901a\u8fc7Jindo SDK 4.x \u53ef\u4ee5\u517c\u5bb9\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u3002\u5173\u4e8e\u7528\u6237\u5982\u4f55\u521b\u5efa\u548c\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u7684\u57fa\u672c\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 \u94fe\u63a5 \u3002 OSS-HDFS \u670d\u52a1\u7684\u5feb\u7167\u529f\u80fd\u5728\u4f7f\u7528\u4e0a\u4e0e HDFS \u7684\u5feb\u7167\u529f\u80fd\u5b8c\u5168\u517c\u5bb9\u3002\u672c\u6587\u4e3b\u8981\u4ecb\u7ecd\u5728 OSS-HDFS \u670d\u52a1\u4e2d\u4f7f\u7528\u5feb\u7167\u529f\u80fd\u7684\u4e00\u4e9b\u5e38\u89c1\u64cd\u4f5c\u3002 \u4e3a\u4e86\u65b9\u4fbf\u4e0b\u9762\u547d\u4ee4\u7684\u4ecb\u7ecd\uff0c\u6211\u4eec\u5047\u8bbe oss://oss-dfs-test \u8fd9\u4e2aBucket\u5df2\u7ecf\u5f00\u542f OSS-HDFS \u670d\u52a1\u3002\u4e0b\u9762\u6240\u6709\u7684\u4f8b\u5b50\u90fd\u4f1a\u9488\u5bf9\u8fd9\u4e2a Bucket \u8fdb\u884c\u64cd\u4f5c\u3002 \u5f00\u542f\u5feb\u7167\u529f\u80fd \u00a7 \u5047\u8bbe\u6211\u4eec\u5148\u5728 oss://oss-dfs-test \u4e0b\u521b\u5efa\u4e00\u4e2a\u76ee\u5f55TestSnapshot\u3002 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u662f\u5173\u95ed\u7684\u3002\u9700\u8981\u5f00\u542f\u548c\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\uff0c\u9700\u8981\u4f7f\u7528 jindofs \u547d\u4ee4\u884c\u5de5\u5177 \u3002\u5177\u4f53\u7684\u5f00\u542f\u5feb\u7167\u7684\u683c\u5f0f\u4e3a ./jindofs admin -allowSnapshot -dlsUri <path> \u5982\u679c\u6211\u4eec\u8981\u5f00\u542f\u524d\u9762\u521b\u5efa\u7684 TestSnapshot \u7684\u5feb\u7167\u529f\u80fd\u3002\u53ef\u4ee5\u8f93\u5165\u5982\u4e0b\u7684\u547d\u4ee4\uff1a ./jindofs admin -allowSnapshot -dlsUri oss://oss-dfs-test.<Endpoint>/TestSnapshot \u521b\u5efa\u5feb\u7167 \u00a7 \u5728\u4e00\u4e2a\u76ee\u5f55\u5f00\u542f\u5feb\u7167\u529f\u80fd\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7 HDFS \u7684 shell \u547d\u4ee4\u3002\u5177\u4f53\u683c\u5f0f\u4e3a\uff1a hdfs dfs -createSnapshot <path> [<snapshotName>] \u4f8b\u5982\u4e0a\u9762\u4f8b\u5b50\u7684\u4e2d\u7684 TestSnapshot\uff0c\u6211\u4eec\u8fdb\u884c\u4e86\u4e00\u4e9b\u76ee\u5f55\u548c\u6587\u4ef6\u7684\u64cd\u4f5c\u3002\u7136\u540e\u53ef\u4ee5\u521b\u5efa\u4e00\u4e2a\u5feb\u7167\u5bf9\u6b64\u523b\u7684\u5feb\u7167\u7684\u72b6\u6001\u8fdb\u884c\u4fdd\u5b58\u3002\u5047\u8bbe\u6211\u4eec\u547d\u540d\u5feb\u7167\u540d\u4e3a S1\u3002 #\u4e0b\u9762\u8fd9\u4e9b\u4e3a\u521b\u5efa\u6d4b\u8bd5\u7684\u6587\u4ef6\u548c\u76ee\u5f55 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir2 hdfs dfs -touchz oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1/file1 hdfs dfs -touchz oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir3/file2 #\u901a\u8fc7HDFS\u7684shell\u547d\u4ee4\u884c\u5de5\u5177\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u521b\u5efa\u5feb\u7167\uff0c\u5e76\u5c06\u5feb\u7167\u547d\u540d\u4e3aS1\uff1a hdfs dfs -createSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S1 \u8bbf\u95ee\u5feb\u7167\u4e2d\u7684\u76ee\u5f55\u548c\u6587\u4ef6 \u00a7 \u5feb\u7167\u76ee\u5f55\u7684\u683c\u5f0f \u00a7 \u4e3a\u4e86\u533a\u522b\u4e8e\u6587\u4ef6\u7cfb\u7edf\u73b0\u6709\u7684\u76ee\u5f55\u548c\u6587\u4ef6\uff0c\u8bbf\u95ee\u5feb\u7167\u4e2d\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u9700\u8981\u6dfb\u52a0\u5feb\u7167\u7684\u4fe1\u606f\u6765\u6307\u51fa\u81ea\u5df1\u8981\u8bbf\u95ee\u7684\u5feb\u7167\u540d\u3002\u5bf9\u4e8e\u4e00\u4e2a\u5f00\u542f\u4e86\u5feb\u7167\u7684\u76ee\u5f55\uff0c\u6211\u4eec\u8981\u8bbf\u95ee\u5b83\u4e0b\u9762\u67d0\u4e2a\u5feb\u7167\u7684\u76ee\u5f55\u548c\u6587\u4ef6\uff0c\u6211\u4eec\u53ef\u4ee5\u4e0b\u9762\u7684\u683c\u5f0f\u6765\u8bbf\u95ee\uff1a <snapshotRoot>/.snapshot/<snapshotName>/<actual subPath> \u5176\u4e2d snapshotRoot \u4e3a\u5feb\u7167\u7684\u6839\u76ee\u5f55\uff0c\u4e5f\u5c31\u662f\u5f00\u542f\u5feb\u7167\u547d\u4ee4\u4e2ddlsUri\u53c2\u6570\u5236\u5b9a\u7684\u8def\u5f84\u3002snapshotName\u662f\u4e4b\u524d\u521b\u5efa\u7684\u5feb\u7167\u540d\u3002\u5236\u5b9a\u4e86\u5feb\u7167\u540d\u4e4b\u540e\uff0c\u4e4b\u540e\u7684\u8def\u5f84\u5c31\u662f\u539f\u6765\u5feb\u7167\u6839\u76ee\u5f55\uff08snapshotRoot\uff09\u4e0b\u8981\u8bbf\u95ee\u7684\u8def\u5f84\u3002 \u6bd4\u5982\u4e0a\u9762\u7684\u4f8b\u5b50 TestSnapshot \u76ee\u5f55\uff0c\u6211\u4eec\u5982\u679c\u8981\u5217\u51fa oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u4e0b\u7684\u6587\u4ef6\u3002\u53ef\u4ee5\u7528\u5e38\u89c4\u7684 ls \u547d\u4ee4\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u7531\u4e8e\u6211\u4eec\u5728 TestSnapshot \u4e0b\u5f00\u542f\u7684\u5feb\u7167\uff0c\u5e76\u521b\u5efa\u4e86\u5feb\u7167 S1\u3002\u6211\u4eec\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u8def\u5f84\u6765\u5217\u51fa\u5feb\u7167 S1 \u4e0b\u7684\u76ee\u5f55\u548c\u6587\u4ef6\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/.snapshot/S1/dir1 \u5176\u4e2d .snapshot/S1 \u5c31\u662f\u7528\u6765\u8868\u660e\u76ee\u5f55\u7684\u5feb\u7167\u7684\u4fe1\u606f\u3002 \u5229\u7528\u5feb\u7167\u6062\u590d\u6570\u636e \u00a7 \u5feb\u7167\u7684\u4e00\u4e2a\u5e38\u89c1\u4f5c\u7528\u662f\u8fdb\u884c\u6570\u636e\u5907\u4efd\u548c\u6062\u590d\u3002\u5229\u7528\u5feb\u7167\u53ef\u4ee5\u5bf9\u91cd\u8981\u6570\u636e\u8fdb\u884c\u6062\u590d\uff0c\u9632\u6b62\u7528\u6237\u9519\u8bef\u6027\u7684\u64cd\u4f5c\u3002\u53ea\u8981\u6839\u636e\u4e0a\u9762\u8bb2\u7684 \u5feb\u7167\u76ee\u5f55\u7684\u683c\u5f0f \u5c31\u53ef\u4ee5\u8bbf\u95ee\u5230\u53ea\u8bfb\u526f\u672c\u4e2d\u7684\u6570\u636e\uff0c\u7136\u540e\u53ef\u4ee5\u6839\u636e\u9700\u6c42\u8fdb\u884c\u6062\u590d\u64cd\u4f5c\u3002 \u5728\u4e0a\u9762\u7684\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u5047\u8bbe\u5728\u540e\u9762\u7684\u67d0\u6b21\u4efb\u52a1\u4e2d\u8bef\u5220\u4e86 oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 hdfs dfs -rm -r oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u7531\u4e8e\u6211\u4eec\u5df2\u7ecf\u5bf9 /TestSnapshot \u6dfb\u52a0\u4e86\u5feb\u7167 S1\u3002\u6211\u4eec\u53ef\u4ee5\u7528\u5982\u4e0b\u7684\u547d\u4ee4\u6062\u590d\u6570\u636e\u3002 hdfs dfs -cp oss://oss-dfs-test.<Endpoint>/TestSnapshot/.snapshot/S1/dir1 oss://oss-dfs-test.<Endpoint>/TestSnapshot \u8fd9\u4e2a\u65f6\u5019\u6211\u4eec\u518d\u8bbf\u95ee /TestSnapshot/dir1\uff0c\u5c31\u4f1a\u53d1\u73b0\u539f\u6765\u7684\u8bef\u5220\u7684\u6587\u4ef6\u5939\u548c\u6587\u4ef6\u90fd\u6062\u590d\u4e86\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u91cd\u547d\u540d\u5feb\u7167 \u00a7 \u5bf9\u4e8e\u4e00\u4e2a\u5df2\u7ecf\u521b\u5efa\u7684\u5feb\u7167\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u91cd\u547d\u540d hdfs dfs -renameSnapshot <path> <oldName> <newName> \u4f8b\u5982\u4e0a\u9762\u4f8b\u5b50\u7684\u4e2d\u7684 TestSnapshot \u7684\u5feb\u7167 S1\uff0c\u6211\u4eec\u53ef\u4ee5\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u91cd\u547d\u540d\u4e3a S100 hdfs dfs -renameSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S1 S100 \u5220\u9664\u5feb\u7167 \u00a7 \u5f53\u4e0d\u9700\u8981\u5feb\u7167\u7684\u65f6\u5019\uff0c\u53ef\u4ee5\u8fd0\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5220\u9664\u5236\u5b9a\u7684\u5feb\u7167\u3002 hdfs dfs -deleteSnapshot <path> <snapshotName> \u5728\u4e0a\u9762\u7684\u4f8b\u5b50\u4e2d\uff0c\u6211\u4eec\u53ef\u4ee5\u8fd0\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5220\u9664\u4e4b\u524d\u521b\u5efa\u5e76\u91cd\u547d\u540d\u7684\u5feb\u7167 S100 hdfs dfs -deleteSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S100 \u5173\u95ed\u5feb\u7167\u529f\u80fd \u00a7 \u5f53\u6211\u4eec\u9700\u8981\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u7684\u65f6\u5019\uff0c\u6211\u4eec\u540c\u6837\u9700\u8981\u4f7f\u7528 JindoSDK \u7684 shell \u547d\u4ee4\u884c\u3002\u5177\u4f53\u7684\u5173\u95ed\u5feb\u7167\u7684\u683c\u5f0f\u4e3a ./jindofs admin -disallowSnapshot -dlsUri <path> \u67e5\u770b\u4e24\u4e2a\u5feb\u7167\u4e4b\u95f4\u7684\u5dee\u5f02 \u00a7 \u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u5f53\u547d\u4ee4\u67e5\u770b\u4e24\u4e2a\u5feb\u7167\u4e4b\u95f4\u7684\u5dee\u5f02 ./jindofs admin -snapshotDiff -dlsUri <uri> -fromSnapshot <fromSnapshot> -toSnapshot <toSnapshot> \u6ce8\u610f\u4e8b\u9879 \u00a7 \u5982\u679c\u9700\u8981\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\uff0c\u9996\u5148\u8981\u786e\u4fdd\u8be5\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u5feb\u7167\u5df2\u7ecf\u88ab\u5220\u9664\u3002\u5220\u9664\u5feb\u7167\u7684\u547d\u4ee4\u53ef\u4ee5\u53c2\u8003\u524d\u9762\u7684\u5220\u9664\u5feb\u7167\u547d\u4ee4\u3002\u5982\u679c\u8be5\u76ee\u5f55\u4e0b\u8fd8\u5b58\u5728\u5feb\u7167\uff0c\u5173\u95ed\u5feb\u7167\u529f\u80fd\u4f1a\u62a5\u9519\u3002 \u5bf9\u4e8e\u6211\u4eec\u524d\u9762\u7684\u4f8b\u5b50 TestSnapshot\uff0c\u5047\u8bbe\u6211\u4eec\u5df2\u7ecf\u5220\u9664\u4e86\u6240\u6709\u5feb\u7167\u3002\u6211\u4eec\u53ef\u4ee5\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u3002 ./jindofs admin -disallowSnapshot -dlsUri oss://oss-dfs-test.<Endpoint>/TestSnapshot","title":"OSS-HDFS \u5feb\u7167"},{"location":"oss/dls_snapshot_quickstart/#oss-hdfs-jindofs","text":"(\u4ece JindoData 4.0.0 \u5f00\u59cb\u652f\u6301) \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u662fOSS\u65b0\u63a8\u51fa\u65b0\u7684\u5b58\u50a8\u7a7a\u95f4\u7c7b\u578b\uff0c\u517c\u5bb9HDFS\u63a5\u53e3, \u652f\u6301\u76ee\u5f55\u4ee5\u53ca\u76ee\u5f55\u5c42\u7ea7\uff0c\u901a\u8fc7Jindo SDK 4.x \u53ef\u4ee5\u517c\u5bb9\u8bbf\u95ee OSS-HDFS \u670d\u52a1\u3002\u5173\u4e8e\u7528\u6237\u5982\u4f55\u521b\u5efa\u548c\u4f7f\u7528 OSS-HDFS \u670d\u52a1\u7684\u57fa\u672c\u529f\u80fd\uff0c\u8bf7\u53c2\u8003 \u94fe\u63a5 \u3002 OSS-HDFS \u670d\u52a1\u7684\u5feb\u7167\u529f\u80fd\u5728\u4f7f\u7528\u4e0a\u4e0e HDFS \u7684\u5feb\u7167\u529f\u80fd\u5b8c\u5168\u517c\u5bb9\u3002\u672c\u6587\u4e3b\u8981\u4ecb\u7ecd\u5728 OSS-HDFS \u670d\u52a1\u4e2d\u4f7f\u7528\u5feb\u7167\u529f\u80fd\u7684\u4e00\u4e9b\u5e38\u89c1\u64cd\u4f5c\u3002 \u4e3a\u4e86\u65b9\u4fbf\u4e0b\u9762\u547d\u4ee4\u7684\u4ecb\u7ecd\uff0c\u6211\u4eec\u5047\u8bbe oss://oss-dfs-test \u8fd9\u4e2aBucket\u5df2\u7ecf\u5f00\u542f OSS-HDFS \u670d\u52a1\u3002\u4e0b\u9762\u6240\u6709\u7684\u4f8b\u5b50\u90fd\u4f1a\u9488\u5bf9\u8fd9\u4e2a Bucket \u8fdb\u884c\u64cd\u4f5c\u3002","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u5feb\u7167\u4f7f\u7528\u8bf4\u660e"},{"location":"oss/dls_snapshot_quickstart/#_1","text":"\u5047\u8bbe\u6211\u4eec\u5148\u5728 oss://oss-dfs-test \u4e0b\u521b\u5efa\u4e00\u4e2a\u76ee\u5f55TestSnapshot\u3002 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u662f\u5173\u95ed\u7684\u3002\u9700\u8981\u5f00\u542f\u548c\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\uff0c\u9700\u8981\u4f7f\u7528 jindofs \u547d\u4ee4\u884c\u5de5\u5177 \u3002\u5177\u4f53\u7684\u5f00\u542f\u5feb\u7167\u7684\u683c\u5f0f\u4e3a ./jindofs admin -allowSnapshot -dlsUri <path> \u5982\u679c\u6211\u4eec\u8981\u5f00\u542f\u524d\u9762\u521b\u5efa\u7684 TestSnapshot \u7684\u5feb\u7167\u529f\u80fd\u3002\u53ef\u4ee5\u8f93\u5165\u5982\u4e0b\u7684\u547d\u4ee4\uff1a ./jindofs admin -allowSnapshot -dlsUri oss://oss-dfs-test.<Endpoint>/TestSnapshot","title":"\u5f00\u542f\u5feb\u7167\u529f\u80fd"},{"location":"oss/dls_snapshot_quickstart/#_2","text":"\u5728\u4e00\u4e2a\u76ee\u5f55\u5f00\u542f\u5feb\u7167\u529f\u80fd\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7 HDFS \u7684 shell \u547d\u4ee4\u3002\u5177\u4f53\u683c\u5f0f\u4e3a\uff1a hdfs dfs -createSnapshot <path> [<snapshotName>] \u4f8b\u5982\u4e0a\u9762\u4f8b\u5b50\u7684\u4e2d\u7684 TestSnapshot\uff0c\u6211\u4eec\u8fdb\u884c\u4e86\u4e00\u4e9b\u76ee\u5f55\u548c\u6587\u4ef6\u7684\u64cd\u4f5c\u3002\u7136\u540e\u53ef\u4ee5\u521b\u5efa\u4e00\u4e2a\u5feb\u7167\u5bf9\u6b64\u523b\u7684\u5feb\u7167\u7684\u72b6\u6001\u8fdb\u884c\u4fdd\u5b58\u3002\u5047\u8bbe\u6211\u4eec\u547d\u540d\u5feb\u7167\u540d\u4e3a S1\u3002 #\u4e0b\u9762\u8fd9\u4e9b\u4e3a\u521b\u5efa\u6d4b\u8bd5\u7684\u6587\u4ef6\u548c\u76ee\u5f55 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 hdfs dfs -mkdir oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir2 hdfs dfs -touchz oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1/file1 hdfs dfs -touchz oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir3/file2 #\u901a\u8fc7HDFS\u7684shell\u547d\u4ee4\u884c\u5de5\u5177\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u521b\u5efa\u5feb\u7167\uff0c\u5e76\u5c06\u5feb\u7167\u547d\u540d\u4e3aS1\uff1a hdfs dfs -createSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S1","title":"\u521b\u5efa\u5feb\u7167"},{"location":"oss/dls_snapshot_quickstart/#_3","text":"","title":"\u8bbf\u95ee\u5feb\u7167\u4e2d\u7684\u76ee\u5f55\u548c\u6587\u4ef6"},{"location":"oss/dls_snapshot_quickstart/#_4","text":"\u4e3a\u4e86\u533a\u522b\u4e8e\u6587\u4ef6\u7cfb\u7edf\u73b0\u6709\u7684\u76ee\u5f55\u548c\u6587\u4ef6\uff0c\u8bbf\u95ee\u5feb\u7167\u4e2d\u7684\u6587\u4ef6\u548c\u76ee\u5f55\u9700\u8981\u6dfb\u52a0\u5feb\u7167\u7684\u4fe1\u606f\u6765\u6307\u51fa\u81ea\u5df1\u8981\u8bbf\u95ee\u7684\u5feb\u7167\u540d\u3002\u5bf9\u4e8e\u4e00\u4e2a\u5f00\u542f\u4e86\u5feb\u7167\u7684\u76ee\u5f55\uff0c\u6211\u4eec\u8981\u8bbf\u95ee\u5b83\u4e0b\u9762\u67d0\u4e2a\u5feb\u7167\u7684\u76ee\u5f55\u548c\u6587\u4ef6\uff0c\u6211\u4eec\u53ef\u4ee5\u4e0b\u9762\u7684\u683c\u5f0f\u6765\u8bbf\u95ee\uff1a <snapshotRoot>/.snapshot/<snapshotName>/<actual subPath> \u5176\u4e2d snapshotRoot \u4e3a\u5feb\u7167\u7684\u6839\u76ee\u5f55\uff0c\u4e5f\u5c31\u662f\u5f00\u542f\u5feb\u7167\u547d\u4ee4\u4e2ddlsUri\u53c2\u6570\u5236\u5b9a\u7684\u8def\u5f84\u3002snapshotName\u662f\u4e4b\u524d\u521b\u5efa\u7684\u5feb\u7167\u540d\u3002\u5236\u5b9a\u4e86\u5feb\u7167\u540d\u4e4b\u540e\uff0c\u4e4b\u540e\u7684\u8def\u5f84\u5c31\u662f\u539f\u6765\u5feb\u7167\u6839\u76ee\u5f55\uff08snapshotRoot\uff09\u4e0b\u8981\u8bbf\u95ee\u7684\u8def\u5f84\u3002 \u6bd4\u5982\u4e0a\u9762\u7684\u4f8b\u5b50 TestSnapshot \u76ee\u5f55\uff0c\u6211\u4eec\u5982\u679c\u8981\u5217\u51fa oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u4e0b\u7684\u6587\u4ef6\u3002\u53ef\u4ee5\u7528\u5e38\u89c4\u7684 ls \u547d\u4ee4\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u7531\u4e8e\u6211\u4eec\u5728 TestSnapshot \u4e0b\u5f00\u542f\u7684\u5feb\u7167\uff0c\u5e76\u521b\u5efa\u4e86\u5feb\u7167 S1\u3002\u6211\u4eec\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u8def\u5f84\u6765\u5217\u51fa\u5feb\u7167 S1 \u4e0b\u7684\u76ee\u5f55\u548c\u6587\u4ef6\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/.snapshot/S1/dir1 \u5176\u4e2d .snapshot/S1 \u5c31\u662f\u7528\u6765\u8868\u660e\u76ee\u5f55\u7684\u5feb\u7167\u7684\u4fe1\u606f\u3002","title":"\u5feb\u7167\u76ee\u5f55\u7684\u683c\u5f0f"},{"location":"oss/dls_snapshot_quickstart/#_5","text":"\u5feb\u7167\u7684\u4e00\u4e2a\u5e38\u89c1\u4f5c\u7528\u662f\u8fdb\u884c\u6570\u636e\u5907\u4efd\u548c\u6062\u590d\u3002\u5229\u7528\u5feb\u7167\u53ef\u4ee5\u5bf9\u91cd\u8981\u6570\u636e\u8fdb\u884c\u6062\u590d\uff0c\u9632\u6b62\u7528\u6237\u9519\u8bef\u6027\u7684\u64cd\u4f5c\u3002\u53ea\u8981\u6839\u636e\u4e0a\u9762\u8bb2\u7684 \u5feb\u7167\u76ee\u5f55\u7684\u683c\u5f0f \u5c31\u53ef\u4ee5\u8bbf\u95ee\u5230\u53ea\u8bfb\u526f\u672c\u4e2d\u7684\u6570\u636e\uff0c\u7136\u540e\u53ef\u4ee5\u6839\u636e\u9700\u6c42\u8fdb\u884c\u6062\u590d\u64cd\u4f5c\u3002 \u5728\u4e0a\u9762\u7684\u6587\u4ef6\u7cfb\u7edf\u4e2d\uff0c\u5047\u8bbe\u5728\u540e\u9762\u7684\u67d0\u6b21\u4efb\u52a1\u4e2d\u8bef\u5220\u4e86 oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 hdfs dfs -rm -r oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1 \u7531\u4e8e\u6211\u4eec\u5df2\u7ecf\u5bf9 /TestSnapshot \u6dfb\u52a0\u4e86\u5feb\u7167 S1\u3002\u6211\u4eec\u53ef\u4ee5\u7528\u5982\u4e0b\u7684\u547d\u4ee4\u6062\u590d\u6570\u636e\u3002 hdfs dfs -cp oss://oss-dfs-test.<Endpoint>/TestSnapshot/.snapshot/S1/dir1 oss://oss-dfs-test.<Endpoint>/TestSnapshot \u8fd9\u4e2a\u65f6\u5019\u6211\u4eec\u518d\u8bbf\u95ee /TestSnapshot/dir1\uff0c\u5c31\u4f1a\u53d1\u73b0\u539f\u6765\u7684\u8bef\u5220\u7684\u6587\u4ef6\u5939\u548c\u6587\u4ef6\u90fd\u6062\u590d\u4e86\u3002 hdfs dfs -ls oss://oss-dfs-test.<Endpoint>/TestSnapshot/dir1","title":"\u5229\u7528\u5feb\u7167\u6062\u590d\u6570\u636e"},{"location":"oss/dls_snapshot_quickstart/#_6","text":"\u5bf9\u4e8e\u4e00\u4e2a\u5df2\u7ecf\u521b\u5efa\u7684\u5feb\u7167\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u91cd\u547d\u540d hdfs dfs -renameSnapshot <path> <oldName> <newName> \u4f8b\u5982\u4e0a\u9762\u4f8b\u5b50\u7684\u4e2d\u7684 TestSnapshot \u7684\u5feb\u7167 S1\uff0c\u6211\u4eec\u53ef\u4ee5\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u91cd\u547d\u540d\u4e3a S100 hdfs dfs -renameSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S1 S100","title":"\u91cd\u547d\u540d\u5feb\u7167"},{"location":"oss/dls_snapshot_quickstart/#_7","text":"\u5f53\u4e0d\u9700\u8981\u5feb\u7167\u7684\u65f6\u5019\uff0c\u53ef\u4ee5\u8fd0\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5220\u9664\u5236\u5b9a\u7684\u5feb\u7167\u3002 hdfs dfs -deleteSnapshot <path> <snapshotName> \u5728\u4e0a\u9762\u7684\u4f8b\u5b50\u4e2d\uff0c\u6211\u4eec\u53ef\u4ee5\u8fd0\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5220\u9664\u4e4b\u524d\u521b\u5efa\u5e76\u91cd\u547d\u540d\u7684\u5feb\u7167 S100 hdfs dfs -deleteSnapshot oss://oss-dfs-test.<Endpoint>/TestSnapshot S100","title":"\u5220\u9664\u5feb\u7167"},{"location":"oss/dls_snapshot_quickstart/#_8","text":"\u5f53\u6211\u4eec\u9700\u8981\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u7684\u65f6\u5019\uff0c\u6211\u4eec\u540c\u6837\u9700\u8981\u4f7f\u7528 JindoSDK \u7684 shell \u547d\u4ee4\u884c\u3002\u5177\u4f53\u7684\u5173\u95ed\u5feb\u7167\u7684\u683c\u5f0f\u4e3a ./jindofs admin -disallowSnapshot -dlsUri <path>","title":"\u5173\u95ed\u5feb\u7167\u529f\u80fd"},{"location":"oss/dls_snapshot_quickstart/#_9","text":"\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u5f53\u547d\u4ee4\u67e5\u770b\u4e24\u4e2a\u5feb\u7167\u4e4b\u95f4\u7684\u5dee\u5f02 ./jindofs admin -snapshotDiff -dlsUri <uri> -fromSnapshot <fromSnapshot> -toSnapshot <toSnapshot>","title":"\u67e5\u770b\u4e24\u4e2a\u5feb\u7167\u4e4b\u95f4\u7684\u5dee\u5f02"},{"location":"oss/dls_snapshot_quickstart/#_10","text":"\u5982\u679c\u9700\u8981\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\uff0c\u9996\u5148\u8981\u786e\u4fdd\u8be5\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u5feb\u7167\u5df2\u7ecf\u88ab\u5220\u9664\u3002\u5220\u9664\u5feb\u7167\u7684\u547d\u4ee4\u53ef\u4ee5\u53c2\u8003\u524d\u9762\u7684\u5220\u9664\u5feb\u7167\u547d\u4ee4\u3002\u5982\u679c\u8be5\u76ee\u5f55\u4e0b\u8fd8\u5b58\u5728\u5feb\u7167\uff0c\u5173\u95ed\u5feb\u7167\u529f\u80fd\u4f1a\u62a5\u9519\u3002 \u5bf9\u4e8e\u6211\u4eec\u524d\u9762\u7684\u4f8b\u5b50 TestSnapshot\uff0c\u5047\u8bbe\u6211\u4eec\u5df2\u7ecf\u5220\u9664\u4e86\u6240\u6709\u5feb\u7167\u3002\u6211\u4eec\u53ef\u4ee5\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u5173\u95ed\u76ee\u5f55\u7684\u5feb\u7167\u529f\u80fd\u3002 ./jindofs admin -disallowSnapshot -dlsUri oss://oss-dfs-test.<Endpoint>/TestSnapshot","title":"\u6ce8\u610f\u4e8b\u9879"},{"location":"oss/dls_trash_quickstart/","text":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u56de\u6536\u7ad9\u4f7f\u7528\u8bf4\u660e \u00a7 (\u4ece JindoData 4.5.1 \u5f00\u59cb\u652f\u6301) \u4ecb\u7ecd \u00a7 \u5f53\u4ece OSS-HDFS \u670d\u52a1\u5220\u9664\u6587\u4ef6\u65f6\uff0c\u6587\u4ef6\u4e0d\u662f\u7acb\u5373\u88ab\u7acb\u5373\u5220\u9664\u3002\u88ab\u5220\u9664\u7684\u6587\u4ef6\u88ab\u632a\u5230\u4e86 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\u3002\u5f53\u7ecf\u8fc730\u5206\u949f\u540e\uff0cCurrent\u76ee\u5f55\u4f1a\u88ab\u632a\u5230 ```bash /user/ /.Trash/ \u4e5f\u5c31\u662f\u8bf4\u67d0\u4e00\u65f6\u95f4\u6bb5\u5185\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u4f1a\u88ab\u5f52\u7c7b\u5230\u4e00\u4e2a\u5e26\u65f6\u95f4\u6233\u7684\u76ee\u5f55\u4e0b\uff0c\u8868\u793a\u662f\u5728\u8fd9\u4e2a\u65f6\u95f4\u5185\u88ab\u5220\u9664\u7684\uff0c\u76f8\u5f53\u4e8e\u4e00\u6b21checkpoint\u3002\u5f53\u7ecf\u8fc7**7\u5929**\u540e\uff0c\u8fd9\u4e2a<timestamp>\u76ee\u5f55\u5c06\u4f1a\u88ab\u6c38\u8fdc\u5220\u9664\u3002\u56e0\u6b64\u57287\u5929\u5185\uff0c\u60a8\u6709\u673a\u4f1a\u4ece.Trash\u76ee\u5f55\u4e0b\u627e\u5230\u5bf9\u5e94\u65f6\u523b\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u5c06\u5176\u4ece.Trash\u632a\u51fa\uff0c\u4ece\u800c\u6062\u590d\u5b83\u3002 \u6ce8\u610f\uff1a\u8fd9\u9879\u529f\u80fd\uff0c\u662f\u7531\u5ba2\u6237\u7aef\u548c\u670d\u52a1\u7aef\u914d\u5408\uff0c\u65b9\u80fd\u5f62\u6210\u56de\u6536\u7ad9\u7684\u529f\u80fd\u3002\u670d\u52a1\u7aef\u53ea\u8d1f\u8d23\u7ef4\u62a4 ```/user/<username>/.Trash``` \u7684\u5b9a\u65f6\u6e05\u7406\uff0c\u670d\u52a1\u7aef\u7684\u5b9a\u65f6\u6e05\u7406\u662f\u9ed8\u8ba4\u5f00\u542f\u7684\u3002\u5ba2\u6237\u7aef\u8d1f\u8d23\u5c06\u5f85\u5220\u9664\u6587\u4ef6\u632a\u5230.Trash\u76ee\u5f55\u4e0b\u3002 ## Hadoop FileSystem Shell \u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd ```bash hadoop fs -rm oss://bucket/a/b/c \u5ba2\u6237\u7aef\u7684 Hadoop Shell \u547d\u4ee4\u9ed8\u8ba4\u4e0d\u5f00\u542fTrash\u529f\u80fd\uff0c\u56e0\u6b64\u9700\u8981\u5728 core-site.xml \u91cc\u6dfb\u52a0\u4e00\u6761\u914d\u7f6e <property> <name>fs.trash.interval</name> <value>1440</value> </property> (\u8be5\u503c\u53ea\u9700\u5927\u4e8e0\u5373\u53ef) \u6b64\u65f6\u5728\u5ba2\u6237\u7aef\u4f1a\u81ea\u52a8\u5c06rm\u547d\u4ee4\u8f6c\u6362\u4e3a\u4e00\u6761 hadoop fs -mv oss://bucket/a/b/c /user/<username>/.Trash/Current/a/b/c \u547d\u4ee4\uff0c\u56e0\u6b64\uff0c\u60a8\u4e0d\u9700\u8981\u611f\u77e5\u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u670d\u52a1\u7aef\u4f1a\u8d1f\u8d23\u6e05\u7406\u3002 \u5982\u679c\u60a8\u60f3\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\uff0c\u91ca\u653e\u7a7a\u95f4\uff0c\u53ef\u4ee5\u6dfb\u52a0 -skipTrash \u53c2\u6570\uff0c\u6b64\u65f6\u5c06\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\u3002 Hadoop \u751f\u6001\u7ec4\u4ef6\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd \u00a7 Hive/Spark/Flink \u7b49\u7ec4\u4ef6\u5e76\u4e0d\u611f\u77e5 OSS-HDFS \u670d\u52a1\u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u4f7f\u7528 FileSystem(HCFS) \u7684 delete \u63a5\u53e3\u610f\u5473\u7740\u7acb\u5373\u5220\u9664\u3002\u56e0\u6b64\u5982\u679c\u60f3\u8981\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd\uff0c\u9700\u8981\u663e\u5f0f\u5730\u8c03\u7528 FileSystem \u7684 rename \u63a5\u53e3\uff0c\u5c06\u76ee\u6807\u6587\u4ef6\u624b\u52a8\u632a\u5230 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\uff0c\u7531 OSS-HDFS \u670d\u52a1\u7aef\u8d1f\u8d23\u5b9a\u671f\u5220\u9664\u3002\u8fd9\u4e00\u70b9\u4e0a OSS-HDFS \u91c7\u53d6\u4e86\u8ddf\u5f00\u6e90 Hadoop \u76f8\u4f3c\u7684\u7b56\u7565\u3002","title":"OSS-HDFS \u56de\u6536\u7ad9"},{"location":"oss/dls_trash_quickstart/#oss-hdfs-jindofs","text":"(\u4ece JindoData 4.5.1 \u5f00\u59cb\u652f\u6301)","title":"\u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1\uff09\u56de\u6536\u7ad9\u4f7f\u7528\u8bf4\u660e"},{"location":"oss/dls_trash_quickstart/#_1","text":"\u5f53\u4ece OSS-HDFS \u670d\u52a1\u5220\u9664\u6587\u4ef6\u65f6\uff0c\u6587\u4ef6\u4e0d\u662f\u7acb\u5373\u88ab\u7acb\u5373\u5220\u9664\u3002\u88ab\u5220\u9664\u7684\u6587\u4ef6\u88ab\u632a\u5230\u4e86 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\u3002\u5f53\u7ecf\u8fc730\u5206\u949f\u540e\uff0cCurrent\u76ee\u5f55\u4f1a\u88ab\u632a\u5230 ```bash /user/ /.Trash/ \u4e5f\u5c31\u662f\u8bf4\u67d0\u4e00\u65f6\u95f4\u6bb5\u5185\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u4f1a\u88ab\u5f52\u7c7b\u5230\u4e00\u4e2a\u5e26\u65f6\u95f4\u6233\u7684\u76ee\u5f55\u4e0b\uff0c\u8868\u793a\u662f\u5728\u8fd9\u4e2a\u65f6\u95f4\u5185\u88ab\u5220\u9664\u7684\uff0c\u76f8\u5f53\u4e8e\u4e00\u6b21checkpoint\u3002\u5f53\u7ecf\u8fc7**7\u5929**\u540e\uff0c\u8fd9\u4e2a<timestamp>\u76ee\u5f55\u5c06\u4f1a\u88ab\u6c38\u8fdc\u5220\u9664\u3002\u56e0\u6b64\u57287\u5929\u5185\uff0c\u60a8\u6709\u673a\u4f1a\u4ece.Trash\u76ee\u5f55\u4e0b\u627e\u5230\u5bf9\u5e94\u65f6\u523b\u88ab\u5220\u9664\u7684\u6587\u4ef6\uff0c\u5c06\u5176\u4ece.Trash\u632a\u51fa\uff0c\u4ece\u800c\u6062\u590d\u5b83\u3002 \u6ce8\u610f\uff1a\u8fd9\u9879\u529f\u80fd\uff0c\u662f\u7531\u5ba2\u6237\u7aef\u548c\u670d\u52a1\u7aef\u914d\u5408\uff0c\u65b9\u80fd\u5f62\u6210\u56de\u6536\u7ad9\u7684\u529f\u80fd\u3002\u670d\u52a1\u7aef\u53ea\u8d1f\u8d23\u7ef4\u62a4 ```/user/<username>/.Trash``` \u7684\u5b9a\u65f6\u6e05\u7406\uff0c\u670d\u52a1\u7aef\u7684\u5b9a\u65f6\u6e05\u7406\u662f\u9ed8\u8ba4\u5f00\u542f\u7684\u3002\u5ba2\u6237\u7aef\u8d1f\u8d23\u5c06\u5f85\u5220\u9664\u6587\u4ef6\u632a\u5230.Trash\u76ee\u5f55\u4e0b\u3002 ## Hadoop FileSystem Shell \u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd ```bash hadoop fs -rm oss://bucket/a/b/c \u5ba2\u6237\u7aef\u7684 Hadoop Shell \u547d\u4ee4\u9ed8\u8ba4\u4e0d\u5f00\u542fTrash\u529f\u80fd\uff0c\u56e0\u6b64\u9700\u8981\u5728 core-site.xml \u91cc\u6dfb\u52a0\u4e00\u6761\u914d\u7f6e <property> <name>fs.trash.interval</name> <value>1440</value> </property> (\u8be5\u503c\u53ea\u9700\u5927\u4e8e0\u5373\u53ef) \u6b64\u65f6\u5728\u5ba2\u6237\u7aef\u4f1a\u81ea\u52a8\u5c06rm\u547d\u4ee4\u8f6c\u6362\u4e3a\u4e00\u6761 hadoop fs -mv oss://bucket/a/b/c /user/<username>/.Trash/Current/a/b/c \u547d\u4ee4\uff0c\u56e0\u6b64\uff0c\u60a8\u4e0d\u9700\u8981\u611f\u77e5\u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u670d\u52a1\u7aef\u4f1a\u8d1f\u8d23\u6e05\u7406\u3002 \u5982\u679c\u60a8\u60f3\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\uff0c\u91ca\u653e\u7a7a\u95f4\uff0c\u53ef\u4ee5\u6dfb\u52a0 -skipTrash \u53c2\u6570\uff0c\u6b64\u65f6\u5c06\u7acb\u5373\u5220\u9664\u8be5\u6587\u4ef6\u3002","title":"\u4ecb\u7ecd"},{"location":"oss/dls_trash_quickstart/#hadoop","text":"Hive/Spark/Flink \u7b49\u7ec4\u4ef6\u5e76\u4e0d\u611f\u77e5 OSS-HDFS \u670d\u52a1\u56de\u6536\u7ad9\u529f\u80fd\u7684\u5b58\u5728\uff0c\u4f7f\u7528 FileSystem(HCFS) \u7684 delete \u63a5\u53e3\u610f\u5473\u7740\u7acb\u5373\u5220\u9664\u3002\u56e0\u6b64\u5982\u679c\u60f3\u8981\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd\uff0c\u9700\u8981\u663e\u5f0f\u5730\u8c03\u7528 FileSystem \u7684 rename \u63a5\u53e3\uff0c\u5c06\u76ee\u6807\u6587\u4ef6\u624b\u52a8\u632a\u5230 /user/<username>/.Trash/Current \u76ee\u5f55\u4e0b\uff0c\u7531 OSS-HDFS \u670d\u52a1\u7aef\u8d1f\u8d23\u5b9a\u671f\u5220\u9664\u3002\u8fd9\u4e00\u70b9\u4e0a OSS-HDFS \u91c7\u53d6\u4e86\u8ddf\u5f00\u6e90 Hadoop \u76f8\u4f3c\u7684\u7b56\u7565\u3002","title":"Hadoop \u751f\u6001\u7ec4\u4ef6\u4f7f\u7528\u56de\u6536\u7ad9\u529f\u80fd"},{"location":"oss/oss_introduction/","text":"OSS/OSS-HDFS \u6982\u8ff0 \u00a7 \u6982\u8ff0 \u00a7 \u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8OSS\uff08Object Storage Service\uff09\u662f\u4e00\u6b3e\u6d77\u91cf\u3001\u5b89\u5168\u3001\u4f4e\u6210\u672c\u3001\u9ad8\u53ef\u9760\u7684\u4e91\u5b58\u50a8\u670d\u52a1\uff0c\u53ef\u63d0\u4f9b99.9999999999%\uff0812\u4e2a9\uff09\u7684\u6570\u636e\u6301\u4e45\u6027\uff0c99.995%\u7684\u6570\u636e\u53ef\u7528\u6027\u3002\u591a\u79cd\u5b58\u50a8\u7c7b\u578b\u4f9b\u9009\u62e9\uff0c\u5168\u9762\u4f18\u5316\u5b58\u50a8\u6210\u672c\u3002\u8be6\u60c5\u8bf7\u89c1 \u4ec0\u4e48\u662f\u5bf9\u8c61\u5b58\u50a8OSS \u3002 OSS-HDFS\u670d\u52a1\uff08JindoFS\u670d\u52a1\uff09\u662f\u4e00\u6b3e\u4e91\u539f\u751f\u6570\u636e\u6e56\u5b58\u50a8\u4ea7\u54c1\u3002\u57fa\u4e8e\u7edf\u4e00\u7684\u5143\u6570\u636e\u7ba1\u7406\u80fd\u529b\uff0c\u5728\u5b8c\u5168\u517c\u5bb9HDFS\u6587\u4ef6\u7cfb\u7edf\u63a5\u53e3\u7684\u540c\u65f6\uff0c\u63d0\u4f9b\u5145\u5206\u7684POSIX\u80fd\u529b\u652f\u6301\uff0c\u80fd\u66f4\u597d\u5730\u6ee1\u8db3\u5927\u6570\u636e\u548cAI\u7b49\u9886\u57df\u7684\u6570\u636e\u6e56\u8ba1\u7b97\u573a\u666f\u3002\u8be6\u60c5\u8bf7\u89c1 OSS-HDFS\u670d\u52a1\u6982\u8ff0 \u3002 JindoData \u662f\u963f\u91cc\u4e91\u5f00\u6e90\u5927\u6570\u636e\u56e2\u961f\u81ea\u7814\u7684\u6570\u636e\u6e56\u5b58\u50a8\u52a0\u901f\u5957\u4ef6\uff0c\u9762\u5411\u5927\u6570\u636e\u548c AI \u751f\u6001\uff0c\u4e3a\u963f\u91cc\u4e91\u548c\u4e1a\u754c\u4e3b\u8981\u6570\u636e\u6e56\u5b58\u50a8\u7cfb\u7edf\u63d0\u4f9b\u5168\u65b9\u4f4d\u8bbf\u95ee\u52a0\u901f\u89e3\u51b3\u65b9\u6848\u3002JindoData \u5957\u4ef6\u57fa\u4e8e\u7edf\u4e00\u67b6\u6784\u548c\u5185\u6838\u5b9e\u73b0\uff0c\u4e3b\u8981\u5305\u62ec JindoFS \u5b58\u50a8\u7cfb\u7edf\uff08\u539f JindoFS Block \u6a21\u5f0f\uff09\uff0cJindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff08\u539f JindoFS Cache \u6a21\u5f0f\uff09\uff0cJindoSDK \u5927\u6570\u636e\u4e07\u80fd SDK \u548c\u5168\u9762\u517c\u5bb9\u7684\u751f\u6001\u5de5\u5177\uff08JindoFuse\u3001JindoDistCp\uff09\u3001\u63d2\u4ef6\u652f\u6301\u3002\u8be6\u60c5\u8bf7\u89c1 \u300aJindoData \u6982\u8ff0\u300b \u3002 \u4f7f\u7528 \u00a7 \u5728 EMR \u73af\u5883\u4e2d\uff0c\u5df2\u7ecf\u9ed8\u8ba4\u90e8\u7f72 JindoSDK\uff0c\u60a8\u53ef\u4ee5\u76f4\u63a5\u901a\u8fc7 JindoSDK \u6765\u8bbf\u95ee OSS/OSS-HDFS\u3002 \u5728 \u975e EMR \u73af\u5883\u4e2d\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4e0b\u8f7d\u6700\u65b0\u7248\u672c\u7684 JindoSDK \uff0c\u81ea\u884c\u90e8\u7f72\u4f7f\u7528\uff0cHadoop \u573a\u666f\u90e8\u7f72\u65b9\u5f0f\u53c2\u89c1 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b , AI\u573a\u666f\u90e8\u7f72\u65b9\u5f0f\u53c2\u89c1 \u300a\u5728 AI \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u3002 \u4f18\u70b9 \u00a7 \u4f7f\u7528 OSS/OSS-HDFS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\u6709\u4ee5\u4e0b\u4f18\u52bf\uff1a \u5373\u63d2\u5373\u7528\u3002OSS/OSS-HDFS \u90fd\u662f\u4e91\u539f\u751f\u5b58\u50a8\u670d\u52a1\uff0c\u901a\u8fc7Restful API\u63d0\u4f9b\u670d\u52a1\uff0c\u672c\u8eab\u4e0d\u9700\u90e8\u7f72\u3002\u5728\u963f\u91cc\u4e91 EMR \u96c6\u7fa4\u4e2d\uff0c\u5df2\u7ecf\u9ed8\u8ba4\u5b89\u88c5 JindoSDK\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7 JindoSDK \u76f4\u63a5\u8bbf\u95ee\u3002 \u8282\u7701\u6210\u672c\u3002\u4f7f\u7528 OSS/OSS-HDFS \u5b58\u50a8\u6570\u636e\u53ef\u4ee5\u6709\u6548\u8282\u7701\u6210\u672c\uff0c\u5e76\u7ed3\u5408\u4f4e\u9891/\u5f52\u6863/\u51b7\u5f52\u6863\u7b49\u65b9\u5f0f\uff0c\u53ef\u4ee5\u8fdb\u4e00\u6b65\u4f18\u5316\u51b7\u6570\u636e\u7684\u5b58\u50a8\u6210\u672c\u3002 \u53ef\u6269\u5c55\u6027\u3002OSS/OSS-HDFS \u6709\u7740\u66f4\u597d\u7684\u53ef\u6269\u5c55\u6027\uff0c\u4e0d\u53d7\u786c\u76d8\u5bb9\u91cf\u9650\u5236\uff0c\u65e0\u9700\u4eba\u5de5\u6269\u5bb9\u3002 \u7279\u6027 \u00a7 \u901a\u8fc7 JindoSDK \u4f7f\u7528 OSS/OSS-HDFS \u7684\u7279\u6027\u5bf9\u6bd4\u5982\u4e0b\uff1a \u573a\u666f \u652f\u6301\u7279\u6027 OSS OSS-HDFS \u5927\u6570\u636e\u573a\u666f \uff08Hadoop\uff09 \u652f\u6301\u76ee\u5f55\u3001\u6587\u4ef6\u8bed\u4e49\u548c\u64cd\u4f5c \u652f\u6301 \u652f\u6301 \u6dfb\u52a0\u76ee\u5f55\u3001\u6587\u4ef6\u6743\u9650 \u4e0d\u652f\u6301 \u652f\u6301 \u76ee\u5f55\u539f\u5b50\u6027\u3001rename\u6027\u80fd \u652f\u6301\uff0c\u4f46\u6027\u80fd\u4e0d\u4f73 \u652f\u6301\uff0c\u6beb\u79d2\u7ea7 \u901a\u8fc7 setTimes \u8bbe\u7f6e\u65f6\u95f4 \u4e0d\u652f\u6301 \u652f\u6301 \u6269\u5c55\u5c5e\u6027 XAttrs \u4e0d\u652f\u6301 \u652f\u6301 ACL \u4e0d\u652f\u6301 \u652f\u6301 \u672c\u5730\u8bfb\u7f13\u5b58\u52a0\u901f \u652f\u6301 \u652f\u6301 \u5feb\u7167 Snapshot \u4e0d\u652f\u6301 \u652f\u6301 \u6587\u4ef6 append/flush/sync \u64cd\u4f5c \u4e0d\u652f\u6301 \u652f\u6301 \u6587\u4ef6 truncate \u4e0d\u652f\u6301 \u652f\u6301 \u6821\u9a8c\u548c Checksum \u652f\u6301 \u652f\u6301 HDFS\u56de\u6536\u7ad9\u81ea\u52a8\u6e05\u7406 \u4e0d\u652f\u6301 \u652f\u6301 AI \u573a\u666f\uff08POSIX\uff09 \u5143\u6570\u636e\u4e00\u81f4\u6027 \u5f31 \u5f3a \u6587\u4ef6 append/flush/sync \u64cd\u4f5c \u652f\u6301\uff0c\u4f46\u6709 \u4f7f\u7528\u9650\u5236 \u652f\u6301 \u6587\u4ef6 truncate \u64cd\u4f5c \u4e0d\u652f\u6301 \u652f\u6301 \u968f\u673a\u5199 \u4e0d\u652f\u6301 \u652f\u6301","title":"OSS/OSS-HDFS \u4ecb\u7ecd"},{"location":"oss/oss_introduction/#ossoss-hdfs","text":"","title":"OSS/OSS-HDFS \u6982\u8ff0"},{"location":"oss/oss_introduction/#_1","text":"\u963f\u91cc\u4e91\u5bf9\u8c61\u5b58\u50a8OSS\uff08Object Storage Service\uff09\u662f\u4e00\u6b3e\u6d77\u91cf\u3001\u5b89\u5168\u3001\u4f4e\u6210\u672c\u3001\u9ad8\u53ef\u9760\u7684\u4e91\u5b58\u50a8\u670d\u52a1\uff0c\u53ef\u63d0\u4f9b99.9999999999%\uff0812\u4e2a9\uff09\u7684\u6570\u636e\u6301\u4e45\u6027\uff0c99.995%\u7684\u6570\u636e\u53ef\u7528\u6027\u3002\u591a\u79cd\u5b58\u50a8\u7c7b\u578b\u4f9b\u9009\u62e9\uff0c\u5168\u9762\u4f18\u5316\u5b58\u50a8\u6210\u672c\u3002\u8be6\u60c5\u8bf7\u89c1 \u4ec0\u4e48\u662f\u5bf9\u8c61\u5b58\u50a8OSS \u3002 OSS-HDFS\u670d\u52a1\uff08JindoFS\u670d\u52a1\uff09\u662f\u4e00\u6b3e\u4e91\u539f\u751f\u6570\u636e\u6e56\u5b58\u50a8\u4ea7\u54c1\u3002\u57fa\u4e8e\u7edf\u4e00\u7684\u5143\u6570\u636e\u7ba1\u7406\u80fd\u529b\uff0c\u5728\u5b8c\u5168\u517c\u5bb9HDFS\u6587\u4ef6\u7cfb\u7edf\u63a5\u53e3\u7684\u540c\u65f6\uff0c\u63d0\u4f9b\u5145\u5206\u7684POSIX\u80fd\u529b\u652f\u6301\uff0c\u80fd\u66f4\u597d\u5730\u6ee1\u8db3\u5927\u6570\u636e\u548cAI\u7b49\u9886\u57df\u7684\u6570\u636e\u6e56\u8ba1\u7b97\u573a\u666f\u3002\u8be6\u60c5\u8bf7\u89c1 OSS-HDFS\u670d\u52a1\u6982\u8ff0 \u3002 JindoData \u662f\u963f\u91cc\u4e91\u5f00\u6e90\u5927\u6570\u636e\u56e2\u961f\u81ea\u7814\u7684\u6570\u636e\u6e56\u5b58\u50a8\u52a0\u901f\u5957\u4ef6\uff0c\u9762\u5411\u5927\u6570\u636e\u548c AI \u751f\u6001\uff0c\u4e3a\u963f\u91cc\u4e91\u548c\u4e1a\u754c\u4e3b\u8981\u6570\u636e\u6e56\u5b58\u50a8\u7cfb\u7edf\u63d0\u4f9b\u5168\u65b9\u4f4d\u8bbf\u95ee\u52a0\u901f\u89e3\u51b3\u65b9\u6848\u3002JindoData \u5957\u4ef6\u57fa\u4e8e\u7edf\u4e00\u67b6\u6784\u548c\u5185\u6838\u5b9e\u73b0\uff0c\u4e3b\u8981\u5305\u62ec JindoFS \u5b58\u50a8\u7cfb\u7edf\uff08\u539f JindoFS Block \u6a21\u5f0f\uff09\uff0cJindoFSx \u5b58\u50a8\u52a0\u901f\u7cfb\u7edf\uff08\u539f JindoFS Cache \u6a21\u5f0f\uff09\uff0cJindoSDK \u5927\u6570\u636e\u4e07\u80fd SDK \u548c\u5168\u9762\u517c\u5bb9\u7684\u751f\u6001\u5de5\u5177\uff08JindoFuse\u3001JindoDistCp\uff09\u3001\u63d2\u4ef6\u652f\u6301\u3002\u8be6\u60c5\u8bf7\u89c1 \u300aJindoData \u6982\u8ff0\u300b \u3002","title":"\u6982\u8ff0"},{"location":"oss/oss_introduction/#_2","text":"\u5728 EMR \u73af\u5883\u4e2d\uff0c\u5df2\u7ecf\u9ed8\u8ba4\u90e8\u7f72 JindoSDK\uff0c\u60a8\u53ef\u4ee5\u76f4\u63a5\u901a\u8fc7 JindoSDK \u6765\u8bbf\u95ee OSS/OSS-HDFS\u3002 \u5728 \u975e EMR \u73af\u5883\u4e2d\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4e0b\u8f7d\u6700\u65b0\u7248\u672c\u7684 JindoSDK \uff0c\u81ea\u884c\u90e8\u7f72\u4f7f\u7528\uff0cHadoop \u573a\u666f\u90e8\u7f72\u65b9\u5f0f\u53c2\u89c1 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b , AI\u573a\u666f\u90e8\u7f72\u65b9\u5f0f\u53c2\u89c1 \u300a\u5728 AI \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u3002","title":"\u4f7f\u7528"},{"location":"oss/oss_introduction/#_3","text":"\u4f7f\u7528 OSS/OSS-HDFS \u4f5c\u4e3a\u5e95\u5c42\u5b58\u50a8\u6709\u4ee5\u4e0b\u4f18\u52bf\uff1a \u5373\u63d2\u5373\u7528\u3002OSS/OSS-HDFS \u90fd\u662f\u4e91\u539f\u751f\u5b58\u50a8\u670d\u52a1\uff0c\u901a\u8fc7Restful API\u63d0\u4f9b\u670d\u52a1\uff0c\u672c\u8eab\u4e0d\u9700\u90e8\u7f72\u3002\u5728\u963f\u91cc\u4e91 EMR \u96c6\u7fa4\u4e2d\uff0c\u5df2\u7ecf\u9ed8\u8ba4\u5b89\u88c5 JindoSDK\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7 JindoSDK \u76f4\u63a5\u8bbf\u95ee\u3002 \u8282\u7701\u6210\u672c\u3002\u4f7f\u7528 OSS/OSS-HDFS \u5b58\u50a8\u6570\u636e\u53ef\u4ee5\u6709\u6548\u8282\u7701\u6210\u672c\uff0c\u5e76\u7ed3\u5408\u4f4e\u9891/\u5f52\u6863/\u51b7\u5f52\u6863\u7b49\u65b9\u5f0f\uff0c\u53ef\u4ee5\u8fdb\u4e00\u6b65\u4f18\u5316\u51b7\u6570\u636e\u7684\u5b58\u50a8\u6210\u672c\u3002 \u53ef\u6269\u5c55\u6027\u3002OSS/OSS-HDFS \u6709\u7740\u66f4\u597d\u7684\u53ef\u6269\u5c55\u6027\uff0c\u4e0d\u53d7\u786c\u76d8\u5bb9\u91cf\u9650\u5236\uff0c\u65e0\u9700\u4eba\u5de5\u6269\u5bb9\u3002","title":"\u4f18\u70b9"},{"location":"oss/oss_introduction/#_4","text":"\u901a\u8fc7 JindoSDK \u4f7f\u7528 OSS/OSS-HDFS \u7684\u7279\u6027\u5bf9\u6bd4\u5982\u4e0b\uff1a \u573a\u666f \u652f\u6301\u7279\u6027 OSS OSS-HDFS \u5927\u6570\u636e\u573a\u666f \uff08Hadoop\uff09 \u652f\u6301\u76ee\u5f55\u3001\u6587\u4ef6\u8bed\u4e49\u548c\u64cd\u4f5c \u652f\u6301 \u652f\u6301 \u6dfb\u52a0\u76ee\u5f55\u3001\u6587\u4ef6\u6743\u9650 \u4e0d\u652f\u6301 \u652f\u6301 \u76ee\u5f55\u539f\u5b50\u6027\u3001rename\u6027\u80fd \u652f\u6301\uff0c\u4f46\u6027\u80fd\u4e0d\u4f73 \u652f\u6301\uff0c\u6beb\u79d2\u7ea7 \u901a\u8fc7 setTimes \u8bbe\u7f6e\u65f6\u95f4 \u4e0d\u652f\u6301 \u652f\u6301 \u6269\u5c55\u5c5e\u6027 XAttrs \u4e0d\u652f\u6301 \u652f\u6301 ACL \u4e0d\u652f\u6301 \u652f\u6301 \u672c\u5730\u8bfb\u7f13\u5b58\u52a0\u901f \u652f\u6301 \u652f\u6301 \u5feb\u7167 Snapshot \u4e0d\u652f\u6301 \u652f\u6301 \u6587\u4ef6 append/flush/sync \u64cd\u4f5c \u4e0d\u652f\u6301 \u652f\u6301 \u6587\u4ef6 truncate \u4e0d\u652f\u6301 \u652f\u6301 \u6821\u9a8c\u548c Checksum \u652f\u6301 \u652f\u6301 HDFS\u56de\u6536\u7ad9\u81ea\u52a8\u6e05\u7406 \u4e0d\u652f\u6301 \u652f\u6301 AI \u573a\u666f\uff08POSIX\uff09 \u5143\u6570\u636e\u4e00\u81f4\u6027 \u5f31 \u5f3a \u6587\u4ef6 append/flush/sync \u64cd\u4f5c \u652f\u6301\uff0c\u4f46\u6709 \u4f7f\u7528\u9650\u5236 \u652f\u6301 \u6587\u4ef6 truncate \u64cd\u4f5c \u4e0d\u652f\u6301 \u652f\u6301 \u968f\u673a\u5199 \u4e0d\u652f\u6301 \u652f\u6301","title":"\u7279\u6027"},{"location":"oss/oss_quickstart/","text":"OSS/OSS-HDFS \u5feb\u901f\u5165\u95e8 \u00a7 \u524d\u7f6e\u6761\u4ef6 \u00a7 \u8bbf\u95ee OSS/OSS-HDFS \u524d\uff0c\u9700\u8981\u521b\u5efa\u5bf9\u5e94\u7684\u5b58\u50a8\u7a7a\u95f4\uff08Bucket\uff09\u3002 \u5f00\u901a OSS \u670d\u52a1 \u3002 \u521b\u5efa OSS Bucket \u3002 \u786e\u8ba4 OSS \u6388\u6743\u3002 \u5728 EMR \u96c6\u7fa4\uff08\u65b0\u7248\u63a7\u5236\u53f0\uff09\u4e2d\u4f7f\u7528\uff0c \u5df2\u9ed8\u8ba4\u5f00\u901a\u3002\u5982\u9047\u95ee\u9898\uff0c\u8be6\u89c1 \u89d2\u8272\u6388\u6743 \u3002 \u5728 EMR \u96c6\u7fa4\uff08\u65e7\u7248\u63a7\u5236\u53f0\uff09\u4e2d\u4f7f\u7528\uff0c \u5df2\u9ed8\u8ba4\u5f00\u901a\u3002\u5982\u9047\u95ee\u9898\uff0c\u8be6\u89c1 \u89d2\u8272\u6388\u6743 \u3002 \u5728\u975e EMR \u73af\u5883\u4e2d\u4f7f\u7528\uff0c\u6388\u6743\u65b9\u5f0f\u8bf7\u53c2\u8003 \u300aOSS/OSS-HDFS \u6388\u6743\u300b \u5f00\u901a\u5e76\u6388\u6743\u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08\u53ef\u9009\u9879\uff0c\u63a8\u8350\u5f00\u901a\uff09\uff1a OSS-HDFS \u670d\u52a1\u4f7f\u7528\u524d\u987b\u77e5 \u3002 \u5f00\u901a\u5e76\u6388\u6743\u8bbf\u95ee OSS-HDFS \u3002 \u786e\u8ba4 JindoSDK \u90e8\u7f72\u7248\u672c\uff1a \u5728 EMR \u96c6\u7fa4\u4e2d\uff0c\u5df2\u9ed8\u8ba4\u90e8\u7f72 JindoSDK\u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u521b\u5efa EMR-3.42.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u6216 EMR-5.8.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u96c6\u7fa4\u3002 \u5728 \u975e EMR \u73af\u5883\u4e2d\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4e0b\u8f7d\u6700\u65b0\u7248\u672c\u7684 JindoSDK \uff0c\u81ea\u884c\u90e8\u7f72\u4f7f\u7528\uff0cHadoop \u573a\u666f\u90e8\u7f72\u65b9\u5f0f\u53c2\u89c1 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b , AI\u573a\u666f\u90e8\u7f72\u65b9\u5f0f\u53c2\u89c1 \u300a\u5728 AI \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u90e8\u7f72 JindoSDK 4.x \u53ca\u4ee5\u4e0a\u7248\u672c\u3002 \u8def\u5f84\u8bf4\u660e \u00a7 \u5b58\u50a8\u7cfb\u7edf \u6839\u8def\u5f84\u793a\u4f8b \u63cf\u8ff0 OSS oss://examplebucket.oss-cn-shanghai-internal.aliyuncs.com/ \u4ee5\u5728\u4e0a\u6d77\u533a\u57df\u5df2\u521b\u5efa\u4e86\u540d\u4e3aexamplebucket\u7684 OSS Bucket\uff0c\u5e76\u4f7f\u7528\u5185\u7f51 endpoint \u8bbf\u95ee\u4e3a\u4f8b\u3002 \u6ce8\uff1a\u5728 EMR \u96c6\u7fa4\u4e2d\u672a\u6302\u8f7d\u516c\u7f51\u7684\u8282\u70b9\uff0c\u9ed8\u8ba4\u4e0d\u652f\u6301\u8bbf\u95ee OSS \u516c\u7f51 endpoint\uff0c\u5373\u9ed8\u8ba4\u4e0d\u652f\u6301\u8de8\u533a\u57df\u8bbf\u95ee\u3002 OSS-HDFS oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u4ee5\u5728\u4e0a\u6d77\u533a\u57df\u5df2\u521b\u5efa\u4e86\u540d\u4e3aexamplebucket\u7684 OSS-HDFS Bucket \u4e3a\u4f8b\u3002 \u6ce8\uff1aOSS-HDFS \u76ee\u524d\u53ea\u652f\u6301\u5185\u7f51\u8bbf\u95ee\uff0c\u5373\u9ed8\u8ba4\u4e0d\u652f\u6301\u8de8\u533a\u57df\u8bbf\u95ee\u3002 \u8bbf\u95ee OSS \u548c OSS-HDFS \u7684\u65b9\u5f0f\uff0c\u9664\u8def\u5f84\u4e2d\u7684 endpoint \u5916\uff0c\u4f7f\u7528\u65b9\u5f0f\u5747\u76f8\u540c\u3002 \u8bbf\u95ee\u65b9\u5f0f \u00a7 \u8bbf\u95ee\u65b9\u5f0f \u793a\u4f8b \u63cf\u8ff0 Hadoop Shell \u547d\u4ee4 hadoop fs -ls oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ JindoSDK \u4e2d\u7684JindoOssFileSystem \u662f Hadoop FileSystem \u7684\u4e00\u79cd\u5b9e\u73b0\u3002 \u5728\u6267\u884c Hadoop Shell \u53ef\u4ee5\u901a\u8fc7\u8bc6\u522b\u8def\u5f84\u4e2d\u7684 endpoint \u6765\u8bbf\u95ee OSS/OSS-HDFS\u3002 \u66f4\u591a\u4f7f\u7528\u65b9\u5f0f\uff0c\u8be6\u89c1 \u300a\u901a\u8fc7 Hadoop Shell \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS\u300b Jindo CLI \u547d\u4ee4 jindo fs -ls oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ Jindo CLI \u5b9e\u73b0\u4e86\u7c7b\u4f3c Hadoop Shell \u7684\u65b9\u5f0f\u6765\u8bbf\u95ee OSS/OSS-HDFS\u3002 \u6b64\u5916\uff0c\u8fd8\u652f\u6301\u4e86\u66f4\u591a\u7684\u529f\u80fd\uff0c\u6bd4\u5982\u5f52\u6863/\u7f13\u5b58/\u9519\u8bef\u5206\u6790\u7b49\u3002\u8be6\u89c1 \u300a\u901a\u8fc7 Jindo CLI \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS\u300b POSIX \u547d\u4ee4 mkdir -p /mnt/oss jindo-fuse /mnt/oss -ouri=oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ ls /mnt/oss \u901a\u8fc7\u5b9e\u73b0 FUSE API\uff0cjindo-fuse \u53ef\u4ee5\u901a\u8fc7\u6302\u8f7d OSS/OSS-HDFS \u8def\u5f84\u5230\u672c\u5730\u8def\u5f84\u7684\u65b9\u5f0f\uff0c\u4f7f\u7528\u6237\u53ef\u4ee5\u50cf\u8bbf\u95ee\u672c\u5730\u6587\u4ef6\u4e00\u6837\u8bbf\u95ee OSS/OSS-HDFS\u3002 \u8be6\u89c1 \u300a\u901a\u8fc7 POSIX \u8bbf\u95ee OSS/OSS-HDFS\u300b OSS \u63a7\u5236\u53f0 \u53ef\u4ee5\u901a\u8fc7\u6253\u5f00 OSS \u63a7\u5236\u53f0 -> \u6587\u4ef6\u5217\u8868 -> OSS\u6587\u4ef6/HDFS \u6765\u8bbf\u95ee OSS/OSS-HDFS","title":"OSS/OSS-HDFS \u5feb\u901f\u5165\u95e8"},{"location":"oss/oss_quickstart/#ossoss-hdfs","text":"","title":"OSS/OSS-HDFS \u5feb\u901f\u5165\u95e8"},{"location":"oss/oss_quickstart/#_1","text":"\u8bbf\u95ee OSS/OSS-HDFS \u524d\uff0c\u9700\u8981\u521b\u5efa\u5bf9\u5e94\u7684\u5b58\u50a8\u7a7a\u95f4\uff08Bucket\uff09\u3002 \u5f00\u901a OSS \u670d\u52a1 \u3002 \u521b\u5efa OSS Bucket \u3002 \u786e\u8ba4 OSS \u6388\u6743\u3002 \u5728 EMR \u96c6\u7fa4\uff08\u65b0\u7248\u63a7\u5236\u53f0\uff09\u4e2d\u4f7f\u7528\uff0c \u5df2\u9ed8\u8ba4\u5f00\u901a\u3002\u5982\u9047\u95ee\u9898\uff0c\u8be6\u89c1 \u89d2\u8272\u6388\u6743 \u3002 \u5728 EMR \u96c6\u7fa4\uff08\u65e7\u7248\u63a7\u5236\u53f0\uff09\u4e2d\u4f7f\u7528\uff0c \u5df2\u9ed8\u8ba4\u5f00\u901a\u3002\u5982\u9047\u95ee\u9898\uff0c\u8be6\u89c1 \u89d2\u8272\u6388\u6743 \u3002 \u5728\u975e EMR \u73af\u5883\u4e2d\u4f7f\u7528\uff0c\u6388\u6743\u65b9\u5f0f\u8bf7\u53c2\u8003 \u300aOSS/OSS-HDFS \u6388\u6743\u300b \u5f00\u901a\u5e76\u6388\u6743\u8bbf\u95ee OSS-HDFS \u670d\u52a1\uff08\u53ef\u9009\u9879\uff0c\u63a8\u8350\u5f00\u901a\uff09\uff1a OSS-HDFS \u670d\u52a1\u4f7f\u7528\u524d\u987b\u77e5 \u3002 \u5f00\u901a\u5e76\u6388\u6743\u8bbf\u95ee OSS-HDFS \u3002 \u786e\u8ba4 JindoSDK \u90e8\u7f72\u7248\u672c\uff1a \u5728 EMR \u96c6\u7fa4\u4e2d\uff0c\u5df2\u9ed8\u8ba4\u90e8\u7f72 JindoSDK\u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u521b\u5efa EMR-3.42.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u6216 EMR-5.8.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u96c6\u7fa4\u3002 \u5728 \u975e EMR \u73af\u5883\u4e2d\uff0c\u60a8\u4e5f\u53ef\u4ee5\u901a\u8fc7\u4e0b\u8f7d\u6700\u65b0\u7248\u672c\u7684 JindoSDK \uff0c\u81ea\u884c\u90e8\u7f72\u4f7f\u7528\uff0cHadoop \u573a\u666f\u90e8\u7f72\u65b9\u5f0f\u53c2\u89c1 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b , AI\u573a\u666f\u90e8\u7f72\u65b9\u5f0f\u53c2\u89c1 \u300a\u5728 AI \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u90e8\u7f72 JindoSDK 4.x \u53ca\u4ee5\u4e0a\u7248\u672c\u3002","title":"\u524d\u7f6e\u6761\u4ef6"},{"location":"oss/oss_quickstart/#_2","text":"\u5b58\u50a8\u7cfb\u7edf \u6839\u8def\u5f84\u793a\u4f8b \u63cf\u8ff0 OSS oss://examplebucket.oss-cn-shanghai-internal.aliyuncs.com/ \u4ee5\u5728\u4e0a\u6d77\u533a\u57df\u5df2\u521b\u5efa\u4e86\u540d\u4e3aexamplebucket\u7684 OSS Bucket\uff0c\u5e76\u4f7f\u7528\u5185\u7f51 endpoint \u8bbf\u95ee\u4e3a\u4f8b\u3002 \u6ce8\uff1a\u5728 EMR \u96c6\u7fa4\u4e2d\u672a\u6302\u8f7d\u516c\u7f51\u7684\u8282\u70b9\uff0c\u9ed8\u8ba4\u4e0d\u652f\u6301\u8bbf\u95ee OSS \u516c\u7f51 endpoint\uff0c\u5373\u9ed8\u8ba4\u4e0d\u652f\u6301\u8de8\u533a\u57df\u8bbf\u95ee\u3002 OSS-HDFS oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u4ee5\u5728\u4e0a\u6d77\u533a\u57df\u5df2\u521b\u5efa\u4e86\u540d\u4e3aexamplebucket\u7684 OSS-HDFS Bucket \u4e3a\u4f8b\u3002 \u6ce8\uff1aOSS-HDFS \u76ee\u524d\u53ea\u652f\u6301\u5185\u7f51\u8bbf\u95ee\uff0c\u5373\u9ed8\u8ba4\u4e0d\u652f\u6301\u8de8\u533a\u57df\u8bbf\u95ee\u3002 \u8bbf\u95ee OSS \u548c OSS-HDFS \u7684\u65b9\u5f0f\uff0c\u9664\u8def\u5f84\u4e2d\u7684 endpoint \u5916\uff0c\u4f7f\u7528\u65b9\u5f0f\u5747\u76f8\u540c\u3002","title":"\u8def\u5f84\u8bf4\u660e"},{"location":"oss/oss_quickstart/#_3","text":"\u8bbf\u95ee\u65b9\u5f0f \u793a\u4f8b \u63cf\u8ff0 Hadoop Shell \u547d\u4ee4 hadoop fs -ls oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ JindoSDK \u4e2d\u7684JindoOssFileSystem \u662f Hadoop FileSystem \u7684\u4e00\u79cd\u5b9e\u73b0\u3002 \u5728\u6267\u884c Hadoop Shell \u53ef\u4ee5\u901a\u8fc7\u8bc6\u522b\u8def\u5f84\u4e2d\u7684 endpoint \u6765\u8bbf\u95ee OSS/OSS-HDFS\u3002 \u66f4\u591a\u4f7f\u7528\u65b9\u5f0f\uff0c\u8be6\u89c1 \u300a\u901a\u8fc7 Hadoop Shell \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS\u300b Jindo CLI \u547d\u4ee4 jindo fs -ls oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ Jindo CLI \u5b9e\u73b0\u4e86\u7c7b\u4f3c Hadoop Shell \u7684\u65b9\u5f0f\u6765\u8bbf\u95ee OSS/OSS-HDFS\u3002 \u6b64\u5916\uff0c\u8fd8\u652f\u6301\u4e86\u66f4\u591a\u7684\u529f\u80fd\uff0c\u6bd4\u5982\u5f52\u6863/\u7f13\u5b58/\u9519\u8bef\u5206\u6790\u7b49\u3002\u8be6\u89c1 \u300a\u901a\u8fc7 Jindo CLI \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS\u300b POSIX \u547d\u4ee4 mkdir -p /mnt/oss jindo-fuse /mnt/oss -ouri=oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ ls /mnt/oss \u901a\u8fc7\u5b9e\u73b0 FUSE API\uff0cjindo-fuse \u53ef\u4ee5\u901a\u8fc7\u6302\u8f7d OSS/OSS-HDFS \u8def\u5f84\u5230\u672c\u5730\u8def\u5f84\u7684\u65b9\u5f0f\uff0c\u4f7f\u7528\u6237\u53ef\u4ee5\u50cf\u8bbf\u95ee\u672c\u5730\u6587\u4ef6\u4e00\u6837\u8bbf\u95ee OSS/OSS-HDFS\u3002 \u8be6\u89c1 \u300a\u901a\u8fc7 POSIX \u8bbf\u95ee OSS/OSS-HDFS\u300b OSS \u63a7\u5236\u53f0 \u53ef\u4ee5\u901a\u8fc7\u6253\u5f00 OSS \u63a7\u5236\u53f0 -> \u6587\u4ef6\u5217\u8868 -> OSS\u6587\u4ef6/HDFS \u6765\u8bbf\u95ee OSS/OSS-HDFS","title":"\u8bbf\u95ee\u65b9\u5f0f"},{"location":"oss/oss_ram_policy/","text":"OSS/OSS-HDFS \u6388\u6743 \u00a7 \u6388\u6743\u8bbf\u95ee OSS \u00a7 \u6388\u6743\u64cd\u4f5c\u8d26\u53f7\u5bf9OSS\u7684\u64cd\u4f5c\u6743\u9650\u65f6\uff0c\u652f\u6301\u901a\u8fc7\u81ea\u5b9a\u4e49\u7b56\u7565\u7075\u6d3b\u5b9a\u4e49RAM\u7528\u6237\u5bf9OSS\u6570\u636e\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u8be6\u7ec6\u6b65\u9aa4\u5982\u4e0b\u3002 \u767b\u5f55 RAM\u63a7\u5236\u53f0 \u3002 \u5728\u5de6\u4fa7\u5bfc\u822a\u680f\uff0c\u9009\u62e9 \u6743\u9650\u7ba1\u7406 > \u6743\u9650\u7b56\u7565 \u3002 \u5728 \u6743\u9650\u7b56\u7565 \u9875\u9762\uff0c\u5355\u51fb \u521b\u5efa\u6743\u9650\u7b56\u7565 \u3002 \u5728 \u521b\u5efa\u6743\u9650\u7b56\u7565 \u9875\u9762\uff0c\u5355\u51fb \u811a\u672c\u7f16\u8f91 \u9875\u7b7e\u3002 \u8f93\u5165\u6743\u9650\u7b56\u7565\u5185\u5bb9\uff0c\u7136\u540e\u5355\u51fb \u4e0b\u4e00\u6b65\uff1a\u7f16\u8f91\u57fa\u672c\u4fe1\u606f \u3002 OSS \u63d0\u4f9b\u4e86\u5b8c\u6574\u7684\u6570\u636e\u6743\u9650\u7ba1\u63a7\u4f53\u7cfb\uff0c\u5b8c\u6574\u7684OSS\u6388\u6743\u7b56\u7565\u8bf7\u53c2\u89c1 RAM Policy\u6982\u8ff0 \u3002 \u6ce8\u610f \u8bf7\u6839\u636eRAM\u7528\u6237\u9700\u8981\u4f7f\u7528\u7684\u6743\u9650\uff0c\u8c28\u614e\u5b9a\u4e49\u6743\u9650\u7b56\u7565\u3002\u5bf9\u4e8e\u4f7f\u7528hive\u3001spark\u7684\u7528\u6237\uff0c\u65e0\u8bba\u662f\u5426\u5f00\u542f\u591a\u7248\u672c\u90fd\u9700\u8981\u914d\u7f6e oss:ListObjectVersions \u3001 oss:DeleteObjectVersion \uff0c\u6211\u4eec\u5bf9\u4e8e\u4e00\u4e9b\u5e38\u89c1\u7684\u4e34\u65f6\u76ee\u5f55\uff0c\u6bd4\u5982\"_temporary\"\u3001\".staging\"\u3001\".hive-staging\"\u3001\"__magic\" \u4f1a\u89e6\u53d1\u591a\u7248\u672c\u64cd\u4f5c\u3002 { \"Version\": \"1\", \"Statement\": [ { \"Action\": [ \"oss:ListObjects\", \"oss:GetBucketInfo\", \"oss:PutObject\", \"oss:GetObject\", \"oss:DeleteObject\", \"oss:AbortMultipartUpload\", \"oss:ListParts\", \"oss:RestoreObject\", \"oss:ListObjectVersions\", \"oss:DeleteObjectVersion\" ], \"Resource\": [ \"acs:oss:*:*:<example-bucket>\", \"acs:oss:*:*:<example-bucket>/*\" ], \"Effect\": \"Allow\" }, { \"Action\": [ \"oss:ListBuckets\" ], \"Resource\": \"*\", \"Effect\": \"Allow\" } ] } \u4e0a\u8ff0\u81ea\u5b9a\u4e49\u7b56\u7565\u4e2d\u7684 <example-bucket> \u9700\u8981\u66ff\u6362\u4e3a\u88ab\u6388\u6743\u7684Bucket\u540d\u79f0\u3002 \u8f93\u5165\u6743\u9650\u7b56\u7565 \u540d\u79f0 \u548c \u5907\u6ce8 \uff0c\u5e76\u5355\u51fb \u786e\u5b9a \u3002 \u5176\u4ed6\u81ea\u5b9a\u4e49\u573a\u666f\u793a\u4f8b \u00a7 \u53ea\u8bfb\u8bbf\u95ee OSS\uff1a \u6ce8\u610f \u8bf7\u6839\u636eRAM\u7528\u6237\u9700\u8981\u4f7f\u7528\u7684\u6743\u9650\uff0c\u8c28\u614e\u5b9a\u4e49\u6743\u9650\u7b56\u7565\u3002\u5bf9\u4e8e\u4f7f\u7528hive\u3001spark\u7684\u7528\u6237\uff0c\u65e0\u8bba\u662f\u5426\u5f00\u542f\u591a\u7248\u672c\u90fd\u9700\u8981\u914d\u7f6e oss:ListObjectVersions , \u6211\u4eec\u5bf9\u4e8e\u4e00\u4e9b\u5e38\u89c1\u7684\u4e34\u65f6\u76ee\u5f55\uff0c\u6bd4\u5982\"_temporary\"\u3001\".staging\"\u3001\".hive-staging\"\u3001\"__magic\" \u4f1a\u89e6\u53d1\u591a\u7248\u672c\u64cd\u4f5c\u3002 { \"Version\": \"1\", \"Statement\": [ { \"Action\": [ \"oss:ListObjects\", \"oss:GetBucketInfo\", \"oss:GetObject\", \"oss:RestoreObject\", \"oss:ListObjectVersions\" ], \"Resource\": [ \"acs:oss:*:*:<yourBucketName>\", \"acs:oss:*:*:<yourBucketName>/*\" ], \"Effect\": \"Allow\" }, { \"Action\": [ \"oss:ListBuckets\" ], \"Resource\": \"*\", \"Effect\": \"Allow\" } ] } \u8bbf\u95ee\u5f00\u542f\u4e86\u591a\u7248\u672c\u529f\u80fd\u7684 OSS\uff08\u4e0d\u5efa\u8bae\u5f00\u542f\u591a\u7248\u672c\uff0c\u5bb9\u6613\u4ea7\u751f\u6027\u80fd\u95ee\u9898\uff09\uff1a { \"Version\": \"1\", \"Statement\": [ { \"Action\": [ \"oss:ListObjects\", \"oss:GetBucketInfo\", \"oss:PutObject\", \"oss:GetObject\", \"oss:DeleteObject\", \"oss:AbortMultipartUpload\", \"oss:ListParts\", \"oss:RestoreObject\", \"oss:ListObjectVersions\", \"oss:GetObjectVersion\", \"oss:DeleteObjectVersion\", \"oss:RestoreObjectVersion\" ], \"Resource\": [ \"acs:oss:*:*:<yourBucketName>\", \"acs:oss:*:*:<yourBucketName>/*\" ], \"Effect\": \"Allow\" }, { \"Action\": [ \"oss:ListBuckets\" ], \"Resource\": \"*\", \"Effect\": \"Allow\" } ] } Bucket Policy \u6388\u6743 \u00a7 \u5efa\u8bae\u4f7f\u7528\u201c\u5b8c\u5168\u63a7\u5236\u201d\uff0c\u907f\u514d\u51fa\u73b0\u4ee5\u4e0b\u95ee\u9898\uff1a \u300a\u4f7f\u7528\u5b50\u8d26\u53f7\u8bbf\u95ee OSS \u51fa\u73b0 AccessDenied \u95ee\u9898\u6392\u67e5\u300b \u6388\u6743\u8bbf\u95ee OSS-HDFS \u00a7 { \"Statement\": [ { \"Effect\": \"Allow\", \"Action\": \"oss:ListObjects\", \"Resource\": [ \"acs:oss:*:*:*\" ] }, { \"Effect\": \"Allow\", \"Action\": [ \"oss:GetBucketInfo\", \"oss:PostDataLakeStorageFileOperation\", \"oss:PostDataLakeStorageAdminOperation\" ], \"Resource\": \"*\" }, { \"Effect\": \"Allow\", \"Action\": \"oss:*\", \"Resource\": [ \"acs:oss:*:*:*/.dlsdata\", \"acs:oss:*:*:*/.dlsdata*\" ] } ], \"Version\": \"1\" }","title":"OSS/OSS-HDFS RAM \u914d\u7f6e"},{"location":"oss/oss_ram_policy/#ossoss-hdfs","text":"","title":"OSS/OSS-HDFS \u6388\u6743"},{"location":"oss/oss_ram_policy/#oss","text":"\u6388\u6743\u64cd\u4f5c\u8d26\u53f7\u5bf9OSS\u7684\u64cd\u4f5c\u6743\u9650\u65f6\uff0c\u652f\u6301\u901a\u8fc7\u81ea\u5b9a\u4e49\u7b56\u7565\u7075\u6d3b\u5b9a\u4e49RAM\u7528\u6237\u5bf9OSS\u6570\u636e\u7684\u8bbf\u95ee\u6743\u9650\uff0c\u8be6\u7ec6\u6b65\u9aa4\u5982\u4e0b\u3002 \u767b\u5f55 RAM\u63a7\u5236\u53f0 \u3002 \u5728\u5de6\u4fa7\u5bfc\u822a\u680f\uff0c\u9009\u62e9 \u6743\u9650\u7ba1\u7406 > \u6743\u9650\u7b56\u7565 \u3002 \u5728 \u6743\u9650\u7b56\u7565 \u9875\u9762\uff0c\u5355\u51fb \u521b\u5efa\u6743\u9650\u7b56\u7565 \u3002 \u5728 \u521b\u5efa\u6743\u9650\u7b56\u7565 \u9875\u9762\uff0c\u5355\u51fb \u811a\u672c\u7f16\u8f91 \u9875\u7b7e\u3002 \u8f93\u5165\u6743\u9650\u7b56\u7565\u5185\u5bb9\uff0c\u7136\u540e\u5355\u51fb \u4e0b\u4e00\u6b65\uff1a\u7f16\u8f91\u57fa\u672c\u4fe1\u606f \u3002 OSS \u63d0\u4f9b\u4e86\u5b8c\u6574\u7684\u6570\u636e\u6743\u9650\u7ba1\u63a7\u4f53\u7cfb\uff0c\u5b8c\u6574\u7684OSS\u6388\u6743\u7b56\u7565\u8bf7\u53c2\u89c1 RAM Policy\u6982\u8ff0 \u3002 \u6ce8\u610f \u8bf7\u6839\u636eRAM\u7528\u6237\u9700\u8981\u4f7f\u7528\u7684\u6743\u9650\uff0c\u8c28\u614e\u5b9a\u4e49\u6743\u9650\u7b56\u7565\u3002\u5bf9\u4e8e\u4f7f\u7528hive\u3001spark\u7684\u7528\u6237\uff0c\u65e0\u8bba\u662f\u5426\u5f00\u542f\u591a\u7248\u672c\u90fd\u9700\u8981\u914d\u7f6e oss:ListObjectVersions \u3001 oss:DeleteObjectVersion \uff0c\u6211\u4eec\u5bf9\u4e8e\u4e00\u4e9b\u5e38\u89c1\u7684\u4e34\u65f6\u76ee\u5f55\uff0c\u6bd4\u5982\"_temporary\"\u3001\".staging\"\u3001\".hive-staging\"\u3001\"__magic\" \u4f1a\u89e6\u53d1\u591a\u7248\u672c\u64cd\u4f5c\u3002 { \"Version\": \"1\", \"Statement\": [ { \"Action\": [ \"oss:ListObjects\", \"oss:GetBucketInfo\", \"oss:PutObject\", \"oss:GetObject\", \"oss:DeleteObject\", \"oss:AbortMultipartUpload\", \"oss:ListParts\", \"oss:RestoreObject\", \"oss:ListObjectVersions\", \"oss:DeleteObjectVersion\" ], \"Resource\": [ \"acs:oss:*:*:<example-bucket>\", \"acs:oss:*:*:<example-bucket>/*\" ], \"Effect\": \"Allow\" }, { \"Action\": [ \"oss:ListBuckets\" ], \"Resource\": \"*\", \"Effect\": \"Allow\" } ] } \u4e0a\u8ff0\u81ea\u5b9a\u4e49\u7b56\u7565\u4e2d\u7684 <example-bucket> \u9700\u8981\u66ff\u6362\u4e3a\u88ab\u6388\u6743\u7684Bucket\u540d\u79f0\u3002 \u8f93\u5165\u6743\u9650\u7b56\u7565 \u540d\u79f0 \u548c \u5907\u6ce8 \uff0c\u5e76\u5355\u51fb \u786e\u5b9a \u3002","title":"\u6388\u6743\u8bbf\u95ee OSS"},{"location":"oss/oss_ram_policy/#_1","text":"\u53ea\u8bfb\u8bbf\u95ee OSS\uff1a \u6ce8\u610f \u8bf7\u6839\u636eRAM\u7528\u6237\u9700\u8981\u4f7f\u7528\u7684\u6743\u9650\uff0c\u8c28\u614e\u5b9a\u4e49\u6743\u9650\u7b56\u7565\u3002\u5bf9\u4e8e\u4f7f\u7528hive\u3001spark\u7684\u7528\u6237\uff0c\u65e0\u8bba\u662f\u5426\u5f00\u542f\u591a\u7248\u672c\u90fd\u9700\u8981\u914d\u7f6e oss:ListObjectVersions , \u6211\u4eec\u5bf9\u4e8e\u4e00\u4e9b\u5e38\u89c1\u7684\u4e34\u65f6\u76ee\u5f55\uff0c\u6bd4\u5982\"_temporary\"\u3001\".staging\"\u3001\".hive-staging\"\u3001\"__magic\" \u4f1a\u89e6\u53d1\u591a\u7248\u672c\u64cd\u4f5c\u3002 { \"Version\": \"1\", \"Statement\": [ { \"Action\": [ \"oss:ListObjects\", \"oss:GetBucketInfo\", \"oss:GetObject\", \"oss:RestoreObject\", \"oss:ListObjectVersions\" ], \"Resource\": [ \"acs:oss:*:*:<yourBucketName>\", \"acs:oss:*:*:<yourBucketName>/*\" ], \"Effect\": \"Allow\" }, { \"Action\": [ \"oss:ListBuckets\" ], \"Resource\": \"*\", \"Effect\": \"Allow\" } ] } \u8bbf\u95ee\u5f00\u542f\u4e86\u591a\u7248\u672c\u529f\u80fd\u7684 OSS\uff08\u4e0d\u5efa\u8bae\u5f00\u542f\u591a\u7248\u672c\uff0c\u5bb9\u6613\u4ea7\u751f\u6027\u80fd\u95ee\u9898\uff09\uff1a { \"Version\": \"1\", \"Statement\": [ { \"Action\": [ \"oss:ListObjects\", \"oss:GetBucketInfo\", \"oss:PutObject\", \"oss:GetObject\", \"oss:DeleteObject\", \"oss:AbortMultipartUpload\", \"oss:ListParts\", \"oss:RestoreObject\", \"oss:ListObjectVersions\", \"oss:GetObjectVersion\", \"oss:DeleteObjectVersion\", \"oss:RestoreObjectVersion\" ], \"Resource\": [ \"acs:oss:*:*:<yourBucketName>\", \"acs:oss:*:*:<yourBucketName>/*\" ], \"Effect\": \"Allow\" }, { \"Action\": [ \"oss:ListBuckets\" ], \"Resource\": \"*\", \"Effect\": \"Allow\" } ] }","title":"\u5176\u4ed6\u81ea\u5b9a\u4e49\u573a\u666f\u793a\u4f8b"},{"location":"oss/oss_ram_policy/#bucket-policy","text":"\u5efa\u8bae\u4f7f\u7528\u201c\u5b8c\u5168\u63a7\u5236\u201d\uff0c\u907f\u514d\u51fa\u73b0\u4ee5\u4e0b\u95ee\u9898\uff1a \u300a\u4f7f\u7528\u5b50\u8d26\u53f7\u8bbf\u95ee OSS \u51fa\u73b0 AccessDenied \u95ee\u9898\u6392\u67e5\u300b","title":"Bucket Policy \u6388\u6743"},{"location":"oss/oss_ram_policy/#oss-hdfs","text":"{ \"Statement\": [ { \"Effect\": \"Allow\", \"Action\": \"oss:ListObjects\", \"Resource\": [ \"acs:oss:*:*:*\" ] }, { \"Effect\": \"Allow\", \"Action\": [ \"oss:GetBucketInfo\", \"oss:PostDataLakeStorageFileOperation\", \"oss:PostDataLakeStorageAdminOperation\" ], \"Resource\": \"*\" }, { \"Effect\": \"Allow\", \"Action\": \"oss:*\", \"Resource\": [ \"acs:oss:*:*:*/.dlsdata\", \"acs:oss:*:*:*/.dlsdata*\" ] } ], \"Version\": \"1\" }","title":"\u6388\u6743\u8bbf\u95ee OSS-HDFS"},{"location":"oss/issues/oss_access_denied/","text":"","title":"Oss access denied"},{"location":"oss/practices/oss_performance/","text":"OSS/OSS-HDFS \u7684\u6027\u80fd\u4f18\u5316\u6700\u4f73\u5b9e\u8df5 \u00a7 \u8d8a\u6765\u8d8a\u591a\u7684\u7528\u6237\u6784\u5efa\u6570\u636e\u6e56\uff0c\u5e76\u501f\u52a9 OSS \u6765\u5b58\u50a8\u6570\u636e\u3002OSS \u53ef\u63d0\u4f9b99.9999999999%\uff0812\u4e2a9\uff09\u7684\u6570\u636e\u6301\u4e45\u6027\uff0c99.995%\u7684\u6570\u636e\u53ef\u7528\u6027\u3002\u591a\u79cd\u5b58\u50a8\u7c7b\u578b\u4f9b\u9009\u62e9\uff0c\u5168\u9762\u4f18\u5316\u5b58\u50a8\u6210\u672c\u3002 \u4ece OSS \u4e0a\u4f20\u548c\u4e0b\u8f7d\u6570\u636e\u65f6\uff0c\u60a8\u7684\u5e94\u7528\u7a0b\u5e8f\u53ef\u4ee5\u8f7b\u677e\u5b9e\u73b0\u6bcf\u79d2\u6570\u5343\u4e2a\u4e8b\u52a1\u7684\u8bf7\u6c42\u6027\u80fd\u3002\u60a8\u7684\u5e94\u7528\u7a0b\u5e8f\u53ef\u4ee5\u5b9e\u73b0\u5b58\u50a8\u7a7a\u95f4\uff08Bucket\uff09\u4e2d\u6bcf\u4e2a\u5206\u533a\uff08OSS \u6309\u5bf9\u8c61\u540d\u7684\u5b57\u6bcd\u5e8f\u6392\u5217\u5bf9\u8c61\uff0c\u5e76\u6309\u7167\u6570\u636e\u91cf\u548c\u8bf7\u6c42 QPS \u6765\u62c6\u5206\u5206\u533a\uff09\u6bcf\u79d2\u81f3\u5c11 3,500 \u4e2a PUT/COPY/POST/DELETE \u8bf7\u6c42\u548c 5,500 \u4e2a GET/HEAD \u8bf7\u6c42\uff0c\u800c\u5b58\u50a8\u7a7a\u95f4\u4e2d\u7684\u524d\u7f00\u6570\u91cf\u662f\u6ca1\u6709\u9650\u5236\u7684\uff0c\u56e0\u6b64\u60a8\u53ef\u4ee5\u901a\u8fc7\u5e76\u884c\u8bfb\u53d6\u6765\u63d0\u9ad8\u8bfb\u53d6\u6216\u5199\u5165\u6027\u80fd\u3002\u4e5f\u5c31\u662f\u8bf4\uff0c\u5982\u679c\u60a8\u7684\u8bf7\u6c42\u7684\u6587\u4ef6\u524d\u7f00\u8bbe\u8ba1\u5408\u7406\uff0c\u5728\u67d0\u4e2a OSS Bucket \u4e2d\u521b\u5efa\u4e8610\u4e2a\u524d\u7f00\u6765\u5e76\u884c\u4e0a\u4f20\uff0c\u5219\u53ef\u4ee5\u5c06\u6570\u636e\u5199\u5165\u6027\u80fd\u6269\u5c55\u5230\u6bcf\u79d2 35,000 \u4e2a PUT \u8bf7\u6c42\u3002 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1) \u662f JindoFS \u5b58\u50a8\u7cfb\u7edf\u5728\u963f\u91cc\u4e91\u4e0a\u7684\u670d\u52a1\u5316\u90e8\u7f72\u5f62\u6001\uff0c\u548c\u963f\u91cc\u4e91 OSS \u6df1\u5ea6\u878d\u5408\uff0c\u5f00\u7bb1\u5373\u7528\uff0c\u65e0\u987b\u5728 EMR \u96c6\u7fa4\u4e2d\u90e8\u7f72\u7ef4\u62a4 JindoFS\uff0c\u514d\u8fd0\u7ef4\u3002OSS-HDFS \u670d\u52a1\u5177\u4f53\u4ecb\u7ecd\u8bf7\u53c2\u8003 OSS-HDFS\u670d\u52a1\u6982\u8ff0 \u3002 \u7ba1\u7406 HTTP \u901f\u5ea6\u7f13\u6162\u54cd\u5e94\u7684\u6700\u4f73\u5b9e\u8df5 \u00a7 \u8bbf\u95ee OSS/OSS-HDFS \u6570\u636e\u65f6\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528\u4ee5\u4e0b\u65b9\u6cd5\u63d0\u9ad8 HTTP \u8bf7\u6c42\u7684\u54cd\u5e94\u901f\u5ea6\uff1a \u5c06 OSS/OSS-HDFS \u548c ECS \u5b9e\u4f8b\u914d\u7f6e\u5728\u540c\u4e00\u5730\u57df \u901a\u8fc7\u6c34\u5e73\u6269\u5c55\u548c\u5e76\u884c\u8bf7\u6c42\u5b9e\u73b0\u9ad8\u541e\u5410\u91cf \u5bf9\u65f6\u5ef6\u654f\u611f\u7684\u5e94\u7528\u7a0b\u5e8f\u8fdb\u884c\u8d85\u65f6\u91cd\u8bd5 \u5bf9\u9891\u7e41\u8bbf\u95ee\u7684\u5185\u5bb9\u4f7f\u7528\u7f13\u5b58 \u4f7f\u7528\u6700\u65b0\u7248\u672c JindoSDK \u6211\u4eec\u5efa\u8bae\u60a8\u6839\u636e\u81ea\u5df1\u7684\u4f7f\u7528\u573a\u666f\u9009\u62e9\u5e76\u5e94\u7528\u6700\u9002\u5408\u7684\u9009\u9879\uff0c\u4ee5\u4f18\u5316 OSS/OSS-HDFS \u4e0a\u7684\u6570\u636e\u5904\u7406\u3002\u5728\u4ee5\u4e0b\u5404\u8282\u4e2d\uff0c\u6211\u4eec\u5c06\u4ecb\u7ecd\u6bcf\u79cd\u65b9\u6cd5\u7684\u6700\u4f73\u5b9e\u8df5\u3002 \u5c06 OSS/OSS-HDFS \u548c ECS \u5b9e\u4f8b\u914d\u7f6e\u5728\u540c\u4e00\u5730\u57df \u00a7 OSS\u5b58\u50a8\u7a7a\u95f4\u540d\u79f0\u662f\u5168\u5c40\u552f\u4e00\u7684\uff0c\u60a8\u5728\u521b\u5efa\u5b58\u50a8\u7a7a\u95f4\u65f6\u5fc5\u987b\u6307\u5b9a\u5730\u57df\uff0c\u4e14\u521b\u5efa\u4ee5\u540e\u4e0d\u80fd\u66f4\u6539\u540d\u79f0\u548c\u5730\u57df\u3002\u4e3a\u4e86\u4f18\u5316\u6027\u80fd\uff0c\u6211\u4eec\u5efa\u8bae\u60a8\u5c3d\u53ef\u80fd\u4ece\u540c\u4e00\u963f\u91cc\u4e91\u5730\u57df\u4e2d\u7684 ECS \u5b9e\u4f8b\u8bbf\u95ee OSS \u5b58\u50a8\u7a7a\u95f4\uff0c\u4ece\u800c\u964d\u4f4e\u7f51\u7edc\u5ef6\u8fdf\u548c\u6570\u636e\u4f20\u8f93\u6210\u672c\u3002\u8be6\u60c5\u8bf7\u53c2\u89c1 ECS\u5b9e\u4f8b\u901a\u8fc7OSS\u5185\u7f51\u5730\u5740\u8bbf\u95eeOSS\u8d44\u6e90 \u3002 \u901a\u8fc7\u6c34\u5e73\u6269\u5c55\u548c\u5e76\u53d1/\u5e76\u884c\u8bf7\u6c42\u5b9e\u73b0\u9ad8\u541e\u5410\u91cf \u00a7 OSS \u662f\u4e00\u4e2a\u8d85\u5927\u7684\u5206\u5e03\u5f0f\u7cfb\u7edf\u3002\u4e3a\u4e86\u5e2e\u52a9\u60a8\u5229\u7528\u5176\u89c4\u6a21\uff0c\u5efa\u8bae\u60a8\u5c06\u5e76\u884c\u8bf7\u6c42\u6c34\u5e73\u6269\u5c55\u5230 OSS \u670d\u52a1\u7ec8\u7aef\u8282\u70b9\uff0c\u8fd9\u79cd\u6269\u5c55\u65b9\u5f0f\u6709\u52a9\u4e8e\u901a\u8fc7\u7f51\u7edc\u5c06\u8d1f\u8f7d\u5206\u5e03\u5728\u591a\u4e2a\u8def\u5f84\u4e0a\u3002 OSS-HDFS \u4f7f\u7528\u4e86\u4e0a\u8ff0\u6700\u4f73\u5b9e\u8df5\uff0c\u901a\u8fc7\u5143\u6570\u636e\u670d\u52a1\u6765\u652f\u6301\u5c06\u901a\u8fc7\u6587\u4ef6\u5757\u7684\u65b9\u5f0f\uff0c\u5c06\u6587\u4ef6\u6c34\u5e73\u6269\u5c55\u5230 \u591a\u4e2a OSS \u670d\u52a1\u7ec8\u7aef\u4e0a\uff0c\u4ee5\u83b7\u53d6\u6700\u4f73\u7684\u8bfb\u5199\u6027\u80fd\u3002 \u5bf9\u4e8e\u9ad8\u541e\u5410\u91cf\u4f20\u8f93\uff0c\u60a8\u53ef\u4ee5\u6839\u636e\u60a8\u7684 ECS \u89c4\u683c\u548c\u5e94\u7528\u7a0b\u5e8f\uff0c\u8c03\u6574\u7279\u5b9a\u53c2\u6570\uff0c\u5728\u591a\u4e2a\u7ebf\u7a0b\u6216\u591a\u4e2a\u5b9e\u4f8b\u4e2d\u63a7\u5236\u4e0a\u4f20\u548c\u4e0b\u8f7d\u6570\u636e\u7684\u5e76\u53d1/\u5e76\u884c\u5ea6\u6765\u4f18\u5316\u541e\u5410\u91cf\u3002 \u6d4b\u91cf\u6027\u80fd\u6307\u6807 \u00a7 \u6839\u636eCPU\u3001\u7f51\u7edc\u541e\u5410\u91cf\u7684\u8981\u6c42\u6765\u8bc4\u4f30\u4e0d\u540c\u7684 ECS \u5b9e\u4f8b\u89c4\u683c\u3002\u6709\u5173\u5b9e\u4f8b\u89c4\u683c\u7684\u66f4\u591a\u4fe1\u606f\uff0c\u8bf7\u53c2\u89c1 ECS\u5b9e\u4f8b\u89c4\u683c \u3002\u6b64\u5916\uff0c\u5efa\u8bae\u60a8\u5728\u6d4b\u91cf\u6027\u80fd\u65f6\u4f7f\u7528HTTP\u5206\u6790\u5de5\u5177\u67e5\u770bDNS\u67e5\u8be2\u65f6\u95f4\u3001\u65f6\u5ef6\u548c\u6570\u636e\u4f20\u8f93\u901f\u5ea6 \u5f53\u60a8\u8981\u8c03\u6574\u5e76\u53d1\u7684\u8bf7\u6c42\u6570\u65f6\uff0c\u6027\u80fd\u6d4b\u91cf\u975e\u5e38\u91cd\u8981\u3002\u5efa\u8bae\u4ece\u5355\u4e2a\u8bf7\u6c42\u5f00\u59cb\uff0c\u6d4b\u91cf\u5f53\u524d\u7684\u7f51\u7edc\u5e26\u5bbd\u4ee5\u53ca\u5176\u4ed6\u8d44\u6e90\u7684\u4f7f\u7528\u60c5\u51b5\uff0c\u4ece\u800c\u8bc6\u522b\u74f6\u9888\u8d44\u6e90\uff08\u5373\u4f7f\u7528\u7387\u6700\u9ad8\u7684\u8d44\u6e90\uff09\uff0c\u4ee5\u53ca\u53ef\u80fd\u7684\u5e76\u53d1\u8bf7\u6c42\u6570\u3002\u4f8b\u5982\uff0c\u5982\u679c\u4e00\u6b21\u5904\u7406\u4e00\u4e2a\u8bf7\u6c42\u5bfc\u81f4CPU\u4f7f\u7528\u7387\u4e3a10%\uff0c\u5219\u8868\u660e\u6700\u591a\u53ef\u4ee5\u652f\u630110\u4e2a\u5e76\u53d1\u8bf7\u6c42\u3002 \u8c03\u6574\u5e76\u53d1\u4e0e\u5e76\u884c\u7684\u53c2\u6570 \u00a7 \u60a8\u53ef\u4ee5\u901a\u8fc7\u4fee\u6539 fs.oss.download.thread.concurrency \u548c fs.oss.upload.thread.concurrency \u8c03\u6574\u5355\u4e2a\u8fdb\u7a0b\u5185\u4e0a\u4f20\u548c\u4e0b\u8f7d\u7684\u5e76\u53d1\u6570\u3002 \u5982\u679c\u60a8\u5728\u8fd0\u884c MapReduce \u4efb\u52a1\u65f6\u6216\u8005 Spark \u4efb\u52a1\uff0c\u60a8\u8fd8\u6709\u4ee5\u4e0b\u9009\u62e9\uff1a \u5bf9\u4e8e\u63d0\u4ea4 MapReduce \u4efb\u52a1\uff0c\u53ef\u4ee5\u901a\u8fc7 hadoop \u53c2\u6570 mapreduce.job.maps \u548c \u53c2\u6570mapreduce.job.reduces \u63a7\u5236\u5e76\u884c\u7684\u6267\u884c\u7a0b\u5e8f\u6570\u91cf\u3002 \u5bf9\u4e8e\u63d0\u4ea4 Spark \u6267\u884c\u7a0b\u5e8f\uff0c\u53ef\u4ee5\u901a\u8fc7\u9009\u9879 --num-executors \u6216\u8005 spark \u53c2\u6570 spark.executor.instance \u63a7\u5236\u5e76\u884c\u7684\u6267\u884c\u7a0b\u5e8f\u6570\u91cf\u3002 \u5bf9\u65f6\u5ef6\u654f\u611f\u7684\u5e94\u7528\u7a0b\u5e8f\u8fdb\u884c\u8d85\u65f6\u91cd\u8bd5 \u00a7 OSS \u9488\u5bf9\u7ba1\u63a7\u7c7bAPI\uff0c\u5982GetService\uff08ListBuckets\uff09\u3001PutBucket\u3001GetBucketLifecycle\u7b49\u8fdb\u884cQPS\u9650\u5236\u3002\u5982\u679c\u5e94\u7528\u7a0b\u5e8f\u4ea7\u751f\u9ad8\u8bf7\u6c42\u901f\u7387\uff0c\u53ef\u80fd\u4f1a\u6536\u5230 HTTP 503 \u51cf\u901f\u54cd\u5e94\u3002\u5982\u679c\u53d1\u751f\u8fd9\u7c7b\u9519\u8bef\uff0c\u5efa\u8bae\u60a8\u5ef6\u8fdf\u51e0\u79d2\u540e\u8fdb\u884c\u91cd\u8bd5\u3002 OSS\u4f1a\u5bf9\u8d26\u53f7\u7ea7\u7684\u603b\u8bbf\u95eeQPS\u505a\u9ed8\u8ba4\u9650\u5236\uff08\u8d26\u53f7\u7ea710,000/s\uff09\uff0c\u5982\u679c\u60a8\u9700\u8981\u66f4\u9ad8\u7684QPS\uff0c\u8bf7\u8054\u7cfb \u6280\u672f\u652f\u6301 \u3002\u6ce8\u610f\uff0c\u5728\u6574\u4f53\u8bbf\u95eeQPS\u672a\u8d85\u8fc7\u4e0a\u8ff0\u9608\u503c\u7684\u60c5\u51b5\u4e0b\uff0c\u5982\u679c\u8bf7\u6c42\u96c6\u4e2d\u5728\u7279\u5b9a\u5206\u533a\uff0c\u670d\u52a1\u7aef\u4e5f\u53ef\u80fd\u4f1a\u56e0\u4e3a\u8d85\u8fc7\u5355\u5206\u533a\u7684\u670d\u52a1\u80fd\u529b\u800c\u9650\u6d41\u5e76\u8fd4\u56de503\uff1b\u5982\u679c\u8bf7\u6c42\u524d\u7f00\u5408\u7406\u6253\u6563\uff08\u53c2\u89c1 OSS\u6027\u80fd\u4e0e\u6269\u5c55\u6027\u6700\u4f73\u5b9e\u8df5 \uff09\uff0cOSS\u4f1a\u81ea\u52a8\u6269\u5c55\u5206\u533a\u6570\u91cf\u6765\u652f\u6301\u66f4\u9ad8QPS\u7684\u8bbf\u95ee\uff0c\u60a8\u53ea\u9700\u8981\u7b49\u5f85\u5e76\u91cd\u8bd5\u5373\u53ef\u3002 \u60a8\u53ef\u4ee5\u901a\u8fc7\u8c03\u6574 fs.oss.retry.count \u63a7\u5236\u8bbf\u95ee OSS/OSS-HDFS \u7684\u91cd\u8bd5\u6b21\u6570\uff0c fs.oss.retry.interval.millisecond \u63a7\u5236\u8bbf\u95ee OSS/OSS-HDFS \u7684\u91cd\u8bd5\u95f4\u9694\uff08\u95f4\u9694\u4f1a\u968f\u91cd\u8bd5\u6b21\u6570\u7684\u589e\u52a0\u800c\u500d\u589e\uff09\u3002\u4e5f\u53ef\u4ee5\u6839\u636e\u7f51\u7edc\u60c5\u51b5\u6765\u8c03\u6574 fs.oss.timeout.millisecond \u8d85\u65f6\u65f6\u95f4\uff0c\u8fdb\u884c\u5408\u7406\u7684\u8d85\u65f6\u91cd\u8bd5\u3002 \u5bf9\u9891\u7e41\u8bbf\u95ee\u7684\u5185\u5bb9\u4f7f\u7528\u7f13\u5b58\u52a0\u901f \u00a7 \u5982\u679c\u60a8\u7684\u5e94\u7528\u7a0b\u5e8f\u9700\u8981\u652f\u6301\u540c\u4e00\u5730\u533a\u5927\u91cf\u8bbf\u95ee\u540c\u4e00\u9759\u6001\u6587\u4ef6\u7684\u573a\u666f\uff0c\u53ef\u4ee5\u4f7f\u7528 JindoCache \u7f13\u5b58\u670d\u52a1\u8fdb\u884c\u52a0\u901f\u3002JindoCache \u7f13\u5b58\u670d\u52a1\u5c06\u6587\u4ef6\u4ee5\u5757\u7684\u5f62\u5f0f\u5206\u6563\u5b58\u50a8\u5728\u5206\u5e03\u5f0f\u7f13\u5b58\u670d\u52a1\u4e2d\uff0c\u4ece\u800c\u907f\u514d\u91cd\u590d\u4ece OSS/OSS-HDFS \u53cd\u590d\u62c9\u53d6\u6570\u636e\uff0c\u6709\u6548\u51cf\u5c11\u8bbf\u95ee\u65f6\u5ef6\uff0c\u589e\u52a0\u8ba1\u7b97\u8d44\u6e90\u7684\u5229\u7528\u7387\u3002 \u4f7f\u7528\u6700\u65b0\u7248\u672c JindoSDK \u00a7 \u6700\u65b0\u7248\u672c\u7684 JindoSDK \u63d0\u4f9b\u4e86\u4f18\u5316\u7684\u81ea\u9002\u5e94\u914d\u7f6e\u548c\u9884\u8bfb\u7b97\u6cd5\uff0c\u5e76\u5b9a\u671f\u66f4\u65b0\u4ee5\u9075\u5faa\u65b0\u7684\u6700\u4f73\u5b9e\u8df5\u3002\u4f8b\u5982\uff0c\u5904\u7406\u4e0d\u540c\u7f51\u7edc\u9519\u8bef\u7684\u91cd\u8bd5\uff0c\u81ea\u9002\u5e94\u7684\u5e76\u53d1\u63a7\u5236\u7b49\u3002 \u4e0b\u8f7d\u94fe\u63a5","title":"OSS/OSS-HDFS \u7684\u6027\u80fd\u4f18\u5316\u6700\u4f73\u5b9e\u8df5"},{"location":"oss/practices/oss_performance/#ossoss-hdfs","text":"\u8d8a\u6765\u8d8a\u591a\u7684\u7528\u6237\u6784\u5efa\u6570\u636e\u6e56\uff0c\u5e76\u501f\u52a9 OSS \u6765\u5b58\u50a8\u6570\u636e\u3002OSS \u53ef\u63d0\u4f9b99.9999999999%\uff0812\u4e2a9\uff09\u7684\u6570\u636e\u6301\u4e45\u6027\uff0c99.995%\u7684\u6570\u636e\u53ef\u7528\u6027\u3002\u591a\u79cd\u5b58\u50a8\u7c7b\u578b\u4f9b\u9009\u62e9\uff0c\u5168\u9762\u4f18\u5316\u5b58\u50a8\u6210\u672c\u3002 \u4ece OSS \u4e0a\u4f20\u548c\u4e0b\u8f7d\u6570\u636e\u65f6\uff0c\u60a8\u7684\u5e94\u7528\u7a0b\u5e8f\u53ef\u4ee5\u8f7b\u677e\u5b9e\u73b0\u6bcf\u79d2\u6570\u5343\u4e2a\u4e8b\u52a1\u7684\u8bf7\u6c42\u6027\u80fd\u3002\u60a8\u7684\u5e94\u7528\u7a0b\u5e8f\u53ef\u4ee5\u5b9e\u73b0\u5b58\u50a8\u7a7a\u95f4\uff08Bucket\uff09\u4e2d\u6bcf\u4e2a\u5206\u533a\uff08OSS \u6309\u5bf9\u8c61\u540d\u7684\u5b57\u6bcd\u5e8f\u6392\u5217\u5bf9\u8c61\uff0c\u5e76\u6309\u7167\u6570\u636e\u91cf\u548c\u8bf7\u6c42 QPS \u6765\u62c6\u5206\u5206\u533a\uff09\u6bcf\u79d2\u81f3\u5c11 3,500 \u4e2a PUT/COPY/POST/DELETE \u8bf7\u6c42\u548c 5,500 \u4e2a GET/HEAD \u8bf7\u6c42\uff0c\u800c\u5b58\u50a8\u7a7a\u95f4\u4e2d\u7684\u524d\u7f00\u6570\u91cf\u662f\u6ca1\u6709\u9650\u5236\u7684\uff0c\u56e0\u6b64\u60a8\u53ef\u4ee5\u901a\u8fc7\u5e76\u884c\u8bfb\u53d6\u6765\u63d0\u9ad8\u8bfb\u53d6\u6216\u5199\u5165\u6027\u80fd\u3002\u4e5f\u5c31\u662f\u8bf4\uff0c\u5982\u679c\u60a8\u7684\u8bf7\u6c42\u7684\u6587\u4ef6\u524d\u7f00\u8bbe\u8ba1\u5408\u7406\uff0c\u5728\u67d0\u4e2a OSS Bucket \u4e2d\u521b\u5efa\u4e8610\u4e2a\u524d\u7f00\u6765\u5e76\u884c\u4e0a\u4f20\uff0c\u5219\u53ef\u4ee5\u5c06\u6570\u636e\u5199\u5165\u6027\u80fd\u6269\u5c55\u5230\u6bcf\u79d2 35,000 \u4e2a PUT \u8bf7\u6c42\u3002 \u963f\u91cc\u4e91 OSS-HDFS \u670d\u52a1\uff08JindoFS \u670d\u52a1) \u662f JindoFS \u5b58\u50a8\u7cfb\u7edf\u5728\u963f\u91cc\u4e91\u4e0a\u7684\u670d\u52a1\u5316\u90e8\u7f72\u5f62\u6001\uff0c\u548c\u963f\u91cc\u4e91 OSS \u6df1\u5ea6\u878d\u5408\uff0c\u5f00\u7bb1\u5373\u7528\uff0c\u65e0\u987b\u5728 EMR \u96c6\u7fa4\u4e2d\u90e8\u7f72\u7ef4\u62a4 JindoFS\uff0c\u514d\u8fd0\u7ef4\u3002OSS-HDFS \u670d\u52a1\u5177\u4f53\u4ecb\u7ecd\u8bf7\u53c2\u8003 OSS-HDFS\u670d\u52a1\u6982\u8ff0 \u3002","title":"OSS/OSS-HDFS \u7684\u6027\u80fd\u4f18\u5316\u6700\u4f73\u5b9e\u8df5"},{"location":"oss/practices/oss_performance/#http","text":"\u8bbf\u95ee OSS/OSS-HDFS \u6570\u636e\u65f6\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528\u4ee5\u4e0b\u65b9\u6cd5\u63d0\u9ad8 HTTP \u8bf7\u6c42\u7684\u54cd\u5e94\u901f\u5ea6\uff1a \u5c06 OSS/OSS-HDFS \u548c ECS \u5b9e\u4f8b\u914d\u7f6e\u5728\u540c\u4e00\u5730\u57df \u901a\u8fc7\u6c34\u5e73\u6269\u5c55\u548c\u5e76\u884c\u8bf7\u6c42\u5b9e\u73b0\u9ad8\u541e\u5410\u91cf \u5bf9\u65f6\u5ef6\u654f\u611f\u7684\u5e94\u7528\u7a0b\u5e8f\u8fdb\u884c\u8d85\u65f6\u91cd\u8bd5 \u5bf9\u9891\u7e41\u8bbf\u95ee\u7684\u5185\u5bb9\u4f7f\u7528\u7f13\u5b58 \u4f7f\u7528\u6700\u65b0\u7248\u672c JindoSDK \u6211\u4eec\u5efa\u8bae\u60a8\u6839\u636e\u81ea\u5df1\u7684\u4f7f\u7528\u573a\u666f\u9009\u62e9\u5e76\u5e94\u7528\u6700\u9002\u5408\u7684\u9009\u9879\uff0c\u4ee5\u4f18\u5316 OSS/OSS-HDFS \u4e0a\u7684\u6570\u636e\u5904\u7406\u3002\u5728\u4ee5\u4e0b\u5404\u8282\u4e2d\uff0c\u6211\u4eec\u5c06\u4ecb\u7ecd\u6bcf\u79cd\u65b9\u6cd5\u7684\u6700\u4f73\u5b9e\u8df5\u3002","title":"\u7ba1\u7406 HTTP \u901f\u5ea6\u7f13\u6162\u54cd\u5e94\u7684\u6700\u4f73\u5b9e\u8df5"},{"location":"oss/practices/oss_performance/#ossoss-hdfs-ecs","text":"OSS\u5b58\u50a8\u7a7a\u95f4\u540d\u79f0\u662f\u5168\u5c40\u552f\u4e00\u7684\uff0c\u60a8\u5728\u521b\u5efa\u5b58\u50a8\u7a7a\u95f4\u65f6\u5fc5\u987b\u6307\u5b9a\u5730\u57df\uff0c\u4e14\u521b\u5efa\u4ee5\u540e\u4e0d\u80fd\u66f4\u6539\u540d\u79f0\u548c\u5730\u57df\u3002\u4e3a\u4e86\u4f18\u5316\u6027\u80fd\uff0c\u6211\u4eec\u5efa\u8bae\u60a8\u5c3d\u53ef\u80fd\u4ece\u540c\u4e00\u963f\u91cc\u4e91\u5730\u57df\u4e2d\u7684 ECS \u5b9e\u4f8b\u8bbf\u95ee OSS \u5b58\u50a8\u7a7a\u95f4\uff0c\u4ece\u800c\u964d\u4f4e\u7f51\u7edc\u5ef6\u8fdf\u548c\u6570\u636e\u4f20\u8f93\u6210\u672c\u3002\u8be6\u60c5\u8bf7\u53c2\u89c1 ECS\u5b9e\u4f8b\u901a\u8fc7OSS\u5185\u7f51\u5730\u5740\u8bbf\u95eeOSS\u8d44\u6e90 \u3002","title":"\u5c06 OSS/OSS-HDFS \u548c ECS \u5b9e\u4f8b\u914d\u7f6e\u5728\u540c\u4e00\u5730\u57df"},{"location":"oss/practices/oss_performance/#_1","text":"OSS \u662f\u4e00\u4e2a\u8d85\u5927\u7684\u5206\u5e03\u5f0f\u7cfb\u7edf\u3002\u4e3a\u4e86\u5e2e\u52a9\u60a8\u5229\u7528\u5176\u89c4\u6a21\uff0c\u5efa\u8bae\u60a8\u5c06\u5e76\u884c\u8bf7\u6c42\u6c34\u5e73\u6269\u5c55\u5230 OSS \u670d\u52a1\u7ec8\u7aef\u8282\u70b9\uff0c\u8fd9\u79cd\u6269\u5c55\u65b9\u5f0f\u6709\u52a9\u4e8e\u901a\u8fc7\u7f51\u7edc\u5c06\u8d1f\u8f7d\u5206\u5e03\u5728\u591a\u4e2a\u8def\u5f84\u4e0a\u3002 OSS-HDFS \u4f7f\u7528\u4e86\u4e0a\u8ff0\u6700\u4f73\u5b9e\u8df5\uff0c\u901a\u8fc7\u5143\u6570\u636e\u670d\u52a1\u6765\u652f\u6301\u5c06\u901a\u8fc7\u6587\u4ef6\u5757\u7684\u65b9\u5f0f\uff0c\u5c06\u6587\u4ef6\u6c34\u5e73\u6269\u5c55\u5230 \u591a\u4e2a OSS \u670d\u52a1\u7ec8\u7aef\u4e0a\uff0c\u4ee5\u83b7\u53d6\u6700\u4f73\u7684\u8bfb\u5199\u6027\u80fd\u3002 \u5bf9\u4e8e\u9ad8\u541e\u5410\u91cf\u4f20\u8f93\uff0c\u60a8\u53ef\u4ee5\u6839\u636e\u60a8\u7684 ECS \u89c4\u683c\u548c\u5e94\u7528\u7a0b\u5e8f\uff0c\u8c03\u6574\u7279\u5b9a\u53c2\u6570\uff0c\u5728\u591a\u4e2a\u7ebf\u7a0b\u6216\u591a\u4e2a\u5b9e\u4f8b\u4e2d\u63a7\u5236\u4e0a\u4f20\u548c\u4e0b\u8f7d\u6570\u636e\u7684\u5e76\u53d1/\u5e76\u884c\u5ea6\u6765\u4f18\u5316\u541e\u5410\u91cf\u3002","title":"\u901a\u8fc7\u6c34\u5e73\u6269\u5c55\u548c\u5e76\u53d1/\u5e76\u884c\u8bf7\u6c42\u5b9e\u73b0\u9ad8\u541e\u5410\u91cf"},{"location":"oss/practices/oss_performance/#_2","text":"\u6839\u636eCPU\u3001\u7f51\u7edc\u541e\u5410\u91cf\u7684\u8981\u6c42\u6765\u8bc4\u4f30\u4e0d\u540c\u7684 ECS \u5b9e\u4f8b\u89c4\u683c\u3002\u6709\u5173\u5b9e\u4f8b\u89c4\u683c\u7684\u66f4\u591a\u4fe1\u606f\uff0c\u8bf7\u53c2\u89c1 ECS\u5b9e\u4f8b\u89c4\u683c \u3002\u6b64\u5916\uff0c\u5efa\u8bae\u60a8\u5728\u6d4b\u91cf\u6027\u80fd\u65f6\u4f7f\u7528HTTP\u5206\u6790\u5de5\u5177\u67e5\u770bDNS\u67e5\u8be2\u65f6\u95f4\u3001\u65f6\u5ef6\u548c\u6570\u636e\u4f20\u8f93\u901f\u5ea6 \u5f53\u60a8\u8981\u8c03\u6574\u5e76\u53d1\u7684\u8bf7\u6c42\u6570\u65f6\uff0c\u6027\u80fd\u6d4b\u91cf\u975e\u5e38\u91cd\u8981\u3002\u5efa\u8bae\u4ece\u5355\u4e2a\u8bf7\u6c42\u5f00\u59cb\uff0c\u6d4b\u91cf\u5f53\u524d\u7684\u7f51\u7edc\u5e26\u5bbd\u4ee5\u53ca\u5176\u4ed6\u8d44\u6e90\u7684\u4f7f\u7528\u60c5\u51b5\uff0c\u4ece\u800c\u8bc6\u522b\u74f6\u9888\u8d44\u6e90\uff08\u5373\u4f7f\u7528\u7387\u6700\u9ad8\u7684\u8d44\u6e90\uff09\uff0c\u4ee5\u53ca\u53ef\u80fd\u7684\u5e76\u53d1\u8bf7\u6c42\u6570\u3002\u4f8b\u5982\uff0c\u5982\u679c\u4e00\u6b21\u5904\u7406\u4e00\u4e2a\u8bf7\u6c42\u5bfc\u81f4CPU\u4f7f\u7528\u7387\u4e3a10%\uff0c\u5219\u8868\u660e\u6700\u591a\u53ef\u4ee5\u652f\u630110\u4e2a\u5e76\u53d1\u8bf7\u6c42\u3002","title":"\u6d4b\u91cf\u6027\u80fd\u6307\u6807"},{"location":"oss/practices/oss_performance/#_3","text":"\u60a8\u53ef\u4ee5\u901a\u8fc7\u4fee\u6539 fs.oss.download.thread.concurrency \u548c fs.oss.upload.thread.concurrency \u8c03\u6574\u5355\u4e2a\u8fdb\u7a0b\u5185\u4e0a\u4f20\u548c\u4e0b\u8f7d\u7684\u5e76\u53d1\u6570\u3002 \u5982\u679c\u60a8\u5728\u8fd0\u884c MapReduce \u4efb\u52a1\u65f6\u6216\u8005 Spark \u4efb\u52a1\uff0c\u60a8\u8fd8\u6709\u4ee5\u4e0b\u9009\u62e9\uff1a \u5bf9\u4e8e\u63d0\u4ea4 MapReduce \u4efb\u52a1\uff0c\u53ef\u4ee5\u901a\u8fc7 hadoop \u53c2\u6570 mapreduce.job.maps \u548c \u53c2\u6570mapreduce.job.reduces \u63a7\u5236\u5e76\u884c\u7684\u6267\u884c\u7a0b\u5e8f\u6570\u91cf\u3002 \u5bf9\u4e8e\u63d0\u4ea4 Spark \u6267\u884c\u7a0b\u5e8f\uff0c\u53ef\u4ee5\u901a\u8fc7\u9009\u9879 --num-executors \u6216\u8005 spark \u53c2\u6570 spark.executor.instance \u63a7\u5236\u5e76\u884c\u7684\u6267\u884c\u7a0b\u5e8f\u6570\u91cf\u3002","title":"\u8c03\u6574\u5e76\u53d1\u4e0e\u5e76\u884c\u7684\u53c2\u6570"},{"location":"oss/practices/oss_performance/#_4","text":"OSS \u9488\u5bf9\u7ba1\u63a7\u7c7bAPI\uff0c\u5982GetService\uff08ListBuckets\uff09\u3001PutBucket\u3001GetBucketLifecycle\u7b49\u8fdb\u884cQPS\u9650\u5236\u3002\u5982\u679c\u5e94\u7528\u7a0b\u5e8f\u4ea7\u751f\u9ad8\u8bf7\u6c42\u901f\u7387\uff0c\u53ef\u80fd\u4f1a\u6536\u5230 HTTP 503 \u51cf\u901f\u54cd\u5e94\u3002\u5982\u679c\u53d1\u751f\u8fd9\u7c7b\u9519\u8bef\uff0c\u5efa\u8bae\u60a8\u5ef6\u8fdf\u51e0\u79d2\u540e\u8fdb\u884c\u91cd\u8bd5\u3002 OSS\u4f1a\u5bf9\u8d26\u53f7\u7ea7\u7684\u603b\u8bbf\u95eeQPS\u505a\u9ed8\u8ba4\u9650\u5236\uff08\u8d26\u53f7\u7ea710,000/s\uff09\uff0c\u5982\u679c\u60a8\u9700\u8981\u66f4\u9ad8\u7684QPS\uff0c\u8bf7\u8054\u7cfb \u6280\u672f\u652f\u6301 \u3002\u6ce8\u610f\uff0c\u5728\u6574\u4f53\u8bbf\u95eeQPS\u672a\u8d85\u8fc7\u4e0a\u8ff0\u9608\u503c\u7684\u60c5\u51b5\u4e0b\uff0c\u5982\u679c\u8bf7\u6c42\u96c6\u4e2d\u5728\u7279\u5b9a\u5206\u533a\uff0c\u670d\u52a1\u7aef\u4e5f\u53ef\u80fd\u4f1a\u56e0\u4e3a\u8d85\u8fc7\u5355\u5206\u533a\u7684\u670d\u52a1\u80fd\u529b\u800c\u9650\u6d41\u5e76\u8fd4\u56de503\uff1b\u5982\u679c\u8bf7\u6c42\u524d\u7f00\u5408\u7406\u6253\u6563\uff08\u53c2\u89c1 OSS\u6027\u80fd\u4e0e\u6269\u5c55\u6027\u6700\u4f73\u5b9e\u8df5 \uff09\uff0cOSS\u4f1a\u81ea\u52a8\u6269\u5c55\u5206\u533a\u6570\u91cf\u6765\u652f\u6301\u66f4\u9ad8QPS\u7684\u8bbf\u95ee\uff0c\u60a8\u53ea\u9700\u8981\u7b49\u5f85\u5e76\u91cd\u8bd5\u5373\u53ef\u3002 \u60a8\u53ef\u4ee5\u901a\u8fc7\u8c03\u6574 fs.oss.retry.count \u63a7\u5236\u8bbf\u95ee OSS/OSS-HDFS \u7684\u91cd\u8bd5\u6b21\u6570\uff0c fs.oss.retry.interval.millisecond \u63a7\u5236\u8bbf\u95ee OSS/OSS-HDFS \u7684\u91cd\u8bd5\u95f4\u9694\uff08\u95f4\u9694\u4f1a\u968f\u91cd\u8bd5\u6b21\u6570\u7684\u589e\u52a0\u800c\u500d\u589e\uff09\u3002\u4e5f\u53ef\u4ee5\u6839\u636e\u7f51\u7edc\u60c5\u51b5\u6765\u8c03\u6574 fs.oss.timeout.millisecond \u8d85\u65f6\u65f6\u95f4\uff0c\u8fdb\u884c\u5408\u7406\u7684\u8d85\u65f6\u91cd\u8bd5\u3002","title":"\u5bf9\u65f6\u5ef6\u654f\u611f\u7684\u5e94\u7528\u7a0b\u5e8f\u8fdb\u884c\u8d85\u65f6\u91cd\u8bd5"},{"location":"oss/practices/oss_performance/#_5","text":"\u5982\u679c\u60a8\u7684\u5e94\u7528\u7a0b\u5e8f\u9700\u8981\u652f\u6301\u540c\u4e00\u5730\u533a\u5927\u91cf\u8bbf\u95ee\u540c\u4e00\u9759\u6001\u6587\u4ef6\u7684\u573a\u666f\uff0c\u53ef\u4ee5\u4f7f\u7528 JindoCache \u7f13\u5b58\u670d\u52a1\u8fdb\u884c\u52a0\u901f\u3002JindoCache \u7f13\u5b58\u670d\u52a1\u5c06\u6587\u4ef6\u4ee5\u5757\u7684\u5f62\u5f0f\u5206\u6563\u5b58\u50a8\u5728\u5206\u5e03\u5f0f\u7f13\u5b58\u670d\u52a1\u4e2d\uff0c\u4ece\u800c\u907f\u514d\u91cd\u590d\u4ece OSS/OSS-HDFS \u53cd\u590d\u62c9\u53d6\u6570\u636e\uff0c\u6709\u6548\u51cf\u5c11\u8bbf\u95ee\u65f6\u5ef6\uff0c\u589e\u52a0\u8ba1\u7b97\u8d44\u6e90\u7684\u5229\u7528\u7387\u3002","title":"\u5bf9\u9891\u7e41\u8bbf\u95ee\u7684\u5185\u5bb9\u4f7f\u7528\u7f13\u5b58\u52a0\u901f"},{"location":"oss/practices/oss_performance/#jindosdk","text":"\u6700\u65b0\u7248\u672c\u7684 JindoSDK \u63d0\u4f9b\u4e86\u4f18\u5316\u7684\u81ea\u9002\u5e94\u914d\u7f6e\u548c\u9884\u8bfb\u7b97\u6cd5\uff0c\u5e76\u5b9a\u671f\u66f4\u65b0\u4ee5\u9075\u5faa\u65b0\u7684\u6700\u4f73\u5b9e\u8df5\u3002\u4f8b\u5982\uff0c\u5904\u7406\u4e0d\u540c\u7f51\u7edc\u9519\u8bef\u7684\u91cd\u8bd5\uff0c\u81ea\u9002\u5e94\u7684\u5e76\u53d1\u63a7\u5236\u7b49\u3002 \u4e0b\u8f7d\u94fe\u63a5","title":"\u4f7f\u7528\u6700\u65b0\u7248\u672c JindoSDK"},{"location":"oss/usages/oss_hadoop_shell/","text":"\u901a\u8fc7 Hadoop Shell \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS \u00a7 \u73af\u5883\u51c6\u5907 \u00a7 EMR\u73af\u5883\u4e2d\uff0c\u9ed8\u8ba4\u5df2\u5b89\u88c5 JindoSDK\uff0c\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528\u3002\u6ce8\u610f\uff1a \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u521b\u5efaEMR-3.42.0\u53ca\u4ee5\u4e0a\u7248\u672c\u6216EMR-5.8.0\u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u96c6\u7fa4\u3002 \u975e EMR \u73af\u5883\uff0c\u8bf7\u5148\u5b89\u88c5\u90e8\u7f72 JindoSDK\u3002\u90e8\u7f72\u65b9\u5f0f\u8bf7\u53c2\u8003 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u90e8\u7f72 JindoSDK 4.x \u53ca\u4ee5\u4e0a\u7248\u672c\u3002 \u901a\u8fc7 Hadoop Shell \u547d\u4ee4\u8bbf\u95ee \u00a7 \u901a\u8fc7 hadoop \u547d\u4ee4\u8bbf\u95ee OSS \u548c OSS-HDFS \u7684\u65b9\u5f0f\u57fa\u672c\u76f8\u540c\uff0c\u4ec5\u8def\u5f84\u4e2d\u7684 endpoint \u7565\u6709\u4e0d\u540c\u3002\u6839\u8def\u5f84\u793a\u4f8b\u53ef\u53c2\u89c1\uff1a \u300aOSS/OSS-HDFS \u5feb\u901f\u5165\u95e8\u300b \u4e0a\u4f20\u6587\u4ef6 \u5c06\u672c\u5730\u6839\u76ee\u5f55\u4e0b\u7684examplefile.txt\u6587\u4ef6\u4e0a\u4f20\u81f3examplebucket\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -put examplefile.txt oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u65b0\u5efa\u76ee\u5f55 \u5728examplebucket\u4e0b\u521b\u5efa\u540d\u4e3adir/\u7684\u76ee\u5f55\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -mkdir oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/dir/ \u67e5\u770b\u6587\u4ef6\u6216\u76ee\u5f55\u4fe1\u606f \u67e5\u770bexamplebucket\u4e0b\u7684\u6587\u4ef6\u6216\u76ee\u5f55\u4fe1\u606f\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -ls oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u83b7\u53d6\u6587\u4ef6\u6216\u76ee\u5f55\u5927\u5c0f \u83b7\u53d6examplebucket\u4e0b\u6240\u6709\u6587\u4ef6\u6216\u76ee\u5f55\u7684\u5927\u5c0f\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -du oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u67e5\u770b\u6587\u4ef6\u5185\u5bb9 \u67e5\u770bexamplebucket\u4e0b\u540d\u4e3alocalfile.txt\u7684\u6587\u4ef6\u5185\u5bb9\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -cat oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/localfile.txt \u91cd\u8981 \u67e5\u770b\u6587\u4ef6\u5185\u5bb9\u65f6\uff0c\u6587\u4ef6\u5185\u5bb9\u5c06\u4ee5\u7eaf\u6587\u672c\u5f62\u5f0f\u6253\u5370\u5230\u5c4f\u5e55\u4e0a\u3002\u5982\u679c\u6587\u4ef6\u5185\u5bb9\u8fdb\u884c\u4e86\u7279\u5b9a\u683c\u5f0f\u7684\u7f16\u7801\uff0c\u8bf7\u4f7f\u7528HDFS\u7684Java API\u8bfb\u53d6\u5e76\u89e3\u7801\u6587\u4ef6\u5185\u5bb9\u3002 \u62f7\u8d1d\u76ee\u5f55\u6216\u6587\u4ef6 \u4f8b\u5982\uff0c\u5c06examplebucket\u4e0b\u6839\u76ee\u5f55subdir1\u62f7\u8d1d\u5230\u76ee\u5f55subdir2\u4e0b\uff0c\u4e14\u6839\u76ee\u5f55subdir1\u6240\u5728\u7684\u4f4d\u7f6e\u3001\u6839\u76ee\u5f55\u4e0b\u7684\u6587\u4ef6\u548c\u5b50\u76ee\u5f55\u7ed3\u6784\u548c\u5185\u5bb9\u4fdd\u6301\u4e0d\u53d8\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -cp oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/subdir1 oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/subdir2/subdir1 \u79fb\u52a8\u76ee\u5f55\u6216\u6587\u4ef6 \u4f8b\u5982\uff0c\u5c06examplebucket\u6839\u76ee\u5f55srcdir\u53ca\u5176\u5305\u542b\u7684\u6587\u4ef6\u6216\u8005\u5b50\u76ee\u5f55\u79fb\u52a8\u81f3\u53e6\u4e00\u4e2a\u6839\u76ee\u5f55destdir\u4e0b\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -mv oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/srcdir oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/destdir \u4e0b\u8f7d\u6587\u4ef6 \u5c06examplebucket\u4e0b\u7684exampleobject.txt\u4e0b\u8f7d\u5230\u672c\u5730\u6839\u76ee\u5f55\u6587\u4ef6\u5939/tmp\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -get oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/exampleobject.txt /tmp/ \u5220\u9664\u76ee\u5f55\u6216\u6587\u4ef6 \u5220\u9664examplebucket\u4e0bdestfolder/\u76ee\u5f55\u53ca\u5176\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6587\u4ef6\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -rm oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/destfolder/","title":"HadoopShell"},{"location":"oss/usages/oss_hadoop_shell/#hadoop-shell-ossoss-hdfs","text":"","title":"\u901a\u8fc7 Hadoop Shell \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS"},{"location":"oss/usages/oss_hadoop_shell/#_1","text":"EMR\u73af\u5883\u4e2d\uff0c\u9ed8\u8ba4\u5df2\u5b89\u88c5 JindoSDK\uff0c\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528\u3002\u6ce8\u610f\uff1a \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u521b\u5efaEMR-3.42.0\u53ca\u4ee5\u4e0a\u7248\u672c\u6216EMR-5.8.0\u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u96c6\u7fa4\u3002 \u975e EMR \u73af\u5883\uff0c\u8bf7\u5148\u5b89\u88c5\u90e8\u7f72 JindoSDK\u3002\u90e8\u7f72\u65b9\u5f0f\u8bf7\u53c2\u8003 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u90e8\u7f72 JindoSDK 4.x \u53ca\u4ee5\u4e0a\u7248\u672c\u3002","title":"\u73af\u5883\u51c6\u5907"},{"location":"oss/usages/oss_hadoop_shell/#hadoop-shell","text":"\u901a\u8fc7 hadoop \u547d\u4ee4\u8bbf\u95ee OSS \u548c OSS-HDFS \u7684\u65b9\u5f0f\u57fa\u672c\u76f8\u540c\uff0c\u4ec5\u8def\u5f84\u4e2d\u7684 endpoint \u7565\u6709\u4e0d\u540c\u3002\u6839\u8def\u5f84\u793a\u4f8b\u53ef\u53c2\u89c1\uff1a \u300aOSS/OSS-HDFS \u5feb\u901f\u5165\u95e8\u300b \u4e0a\u4f20\u6587\u4ef6 \u5c06\u672c\u5730\u6839\u76ee\u5f55\u4e0b\u7684examplefile.txt\u6587\u4ef6\u4e0a\u4f20\u81f3examplebucket\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -put examplefile.txt oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u65b0\u5efa\u76ee\u5f55 \u5728examplebucket\u4e0b\u521b\u5efa\u540d\u4e3adir/\u7684\u76ee\u5f55\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -mkdir oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/dir/ \u67e5\u770b\u6587\u4ef6\u6216\u76ee\u5f55\u4fe1\u606f \u67e5\u770bexamplebucket\u4e0b\u7684\u6587\u4ef6\u6216\u76ee\u5f55\u4fe1\u606f\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -ls oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u83b7\u53d6\u6587\u4ef6\u6216\u76ee\u5f55\u5927\u5c0f \u83b7\u53d6examplebucket\u4e0b\u6240\u6709\u6587\u4ef6\u6216\u76ee\u5f55\u7684\u5927\u5c0f\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -du oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u67e5\u770b\u6587\u4ef6\u5185\u5bb9 \u67e5\u770bexamplebucket\u4e0b\u540d\u4e3alocalfile.txt\u7684\u6587\u4ef6\u5185\u5bb9\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -cat oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/localfile.txt \u91cd\u8981 \u67e5\u770b\u6587\u4ef6\u5185\u5bb9\u65f6\uff0c\u6587\u4ef6\u5185\u5bb9\u5c06\u4ee5\u7eaf\u6587\u672c\u5f62\u5f0f\u6253\u5370\u5230\u5c4f\u5e55\u4e0a\u3002\u5982\u679c\u6587\u4ef6\u5185\u5bb9\u8fdb\u884c\u4e86\u7279\u5b9a\u683c\u5f0f\u7684\u7f16\u7801\uff0c\u8bf7\u4f7f\u7528HDFS\u7684Java API\u8bfb\u53d6\u5e76\u89e3\u7801\u6587\u4ef6\u5185\u5bb9\u3002 \u62f7\u8d1d\u76ee\u5f55\u6216\u6587\u4ef6 \u4f8b\u5982\uff0c\u5c06examplebucket\u4e0b\u6839\u76ee\u5f55subdir1\u62f7\u8d1d\u5230\u76ee\u5f55subdir2\u4e0b\uff0c\u4e14\u6839\u76ee\u5f55subdir1\u6240\u5728\u7684\u4f4d\u7f6e\u3001\u6839\u76ee\u5f55\u4e0b\u7684\u6587\u4ef6\u548c\u5b50\u76ee\u5f55\u7ed3\u6784\u548c\u5185\u5bb9\u4fdd\u6301\u4e0d\u53d8\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -cp oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/subdir1 oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/subdir2/subdir1 \u79fb\u52a8\u76ee\u5f55\u6216\u6587\u4ef6 \u4f8b\u5982\uff0c\u5c06examplebucket\u6839\u76ee\u5f55srcdir\u53ca\u5176\u5305\u542b\u7684\u6587\u4ef6\u6216\u8005\u5b50\u76ee\u5f55\u79fb\u52a8\u81f3\u53e6\u4e00\u4e2a\u6839\u76ee\u5f55destdir\u4e0b\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -mv oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/srcdir oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/destdir \u4e0b\u8f7d\u6587\u4ef6 \u5c06examplebucket\u4e0b\u7684exampleobject.txt\u4e0b\u8f7d\u5230\u672c\u5730\u6839\u76ee\u5f55\u6587\u4ef6\u5939/tmp\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -get oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/exampleobject.txt /tmp/ \u5220\u9664\u76ee\u5f55\u6216\u6587\u4ef6 \u5220\u9664examplebucket\u4e0bdestfolder/\u76ee\u5f55\u53ca\u5176\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6587\u4ef6\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a hadoop fs -rm oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/destfolder/","title":"\u901a\u8fc7 Hadoop Shell \u547d\u4ee4\u8bbf\u95ee"},{"location":"oss/usages/oss_jindo_cli/","text":"\u901a\u8fc7 Jindo CLI \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS \u00a7 \u73af\u5883\u51c6\u5907 \u00a7 EMR\u73af\u5883\u4e2d\uff0c\u9ed8\u8ba4\u5df2\u5b89\u88c5 JindoSDK\uff0c\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528\u3002\u6ce8\u610f\uff1a \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u521b\u5efaEMR-3.44.0\u53ca\u4ee5\u4e0a\u7248\u672c\u6216EMR-5.10.0\u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u96c6\u7fa4\u3002 \u975e EMR \u73af\u5883\uff0c\u8bf7\u5148\u5b89\u88c5\u90e8\u7f72 JindoSDK\u3002\u90e8\u7f72\u65b9\u5f0f\u8bf7\u53c2\u8003 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u90e8\u7f72 JindoSDK 4.6.x \u53ca\u4ee5\u4e0a\u7248\u672c\u3002 \u901a\u8fc7 Jindo CLI \u547d\u4ee4\u8bbf\u95ee \u00a7 \u901a\u8fc7 Jindo CLI \u547d\u4ee4\u8bbf\u95ee OSS \u548c OSS-HDFS \u7684\u65b9\u5f0f\u57fa\u672c\u76f8\u540c\uff0c\u4ec5\u8def\u5f84\u4e2d\u7684 endpoint \u7565\u6709\u4e0d\u540c\u3002\u6839\u8def\u5f84\u793a\u4f8b\u53ef\u53c2\u89c1\uff1a \u300aOSS/OSS-HDFS \u5feb\u901f\u5165\u95e8\u300b \u4e0a\u4f20\u6587\u4ef6 \u5c06\u672c\u5730\u6839\u76ee\u5f55\u4e0b\u7684examplefile.txt\u6587\u4ef6\u4e0a\u4f20\u81f3examplebucket\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -put examplefile.txt oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u65b0\u5efa\u76ee\u5f55 \u5728examplebucket\u4e0b\u521b\u5efa\u540d\u4e3adir/\u7684\u76ee\u5f55\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -mkdir oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/dir/ \u67e5\u770b\u6587\u4ef6\u6216\u76ee\u5f55\u4fe1\u606f \u67e5\u770bexamplebucket\u4e0b\u7684\u6587\u4ef6\u6216\u76ee\u5f55\u4fe1\u606f\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -ls oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u83b7\u53d6\u6587\u4ef6\u6216\u76ee\u5f55\u5927\u5c0f \u83b7\u53d6examplebucket\u4e0b\u6240\u6709\u6587\u4ef6\u6216\u76ee\u5f55\u7684\u5927\u5c0f\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -du oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u67e5\u770b\u6587\u4ef6\u5185\u5bb9 \u67e5\u770bexamplebucket\u4e0b\u540d\u4e3alocalfile.txt\u7684\u6587\u4ef6\u5185\u5bb9\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -cat oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/localfile.txt \u91cd\u8981 \u67e5\u770b\u6587\u4ef6\u5185\u5bb9\u65f6\uff0c\u6587\u4ef6\u5185\u5bb9\u5c06\u4ee5\u7eaf\u6587\u672c\u5f62\u5f0f\u6253\u5370\u5230\u5c4f\u5e55\u4e0a\u3002\u5982\u679c\u6587\u4ef6\u5185\u5bb9\u8fdb\u884c\u4e86\u7279\u5b9a\u683c\u5f0f\u7684\u7f16\u7801\uff0c\u8bf7\u4f7f\u7528HDFS\u7684Java API\u8bfb\u53d6\u5e76\u89e3\u7801\u6587\u4ef6\u5185\u5bb9\u3002 \u4e0b\u8f7d\u6587\u4ef6 \u5c06examplebucket\u4e0b\u7684exampleobject.txt\u4e0b\u8f7d\u5230\u672c\u5730\u6839\u76ee\u5f55\u6587\u4ef6\u5939/tmp\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -get oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/exampleobject.txt /tmp/ \u5220\u9664\u76ee\u5f55\u6216\u6587\u4ef6 \u5220\u9664examplebucket\u4e0bdestfolder/\u76ee\u5f55\u53ca\u5176\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6587\u4ef6\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -rm oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/destfolder/ \u9644\u5f55 \u00a7 \u66f4\u591a\u547d\u4ee4\u53ca\u63cf\u8ff0\uff0c\u8be6\u89c1 \u300aJindo CLI \u53c2\u6570\u8bf4\u660e\u300b","title":"JindoCli \u5feb\u901f\u5165\u95e8"},{"location":"oss/usages/oss_jindo_cli/#jindo-cli-ossoss-hdfs","text":"","title":"\u901a\u8fc7 Jindo CLI \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS"},{"location":"oss/usages/oss_jindo_cli/#_1","text":"EMR\u73af\u5883\u4e2d\uff0c\u9ed8\u8ba4\u5df2\u5b89\u88c5 JindoSDK\uff0c\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528\u3002\u6ce8\u610f\uff1a \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u521b\u5efaEMR-3.44.0\u53ca\u4ee5\u4e0a\u7248\u672c\u6216EMR-5.10.0\u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u96c6\u7fa4\u3002 \u975e EMR \u73af\u5883\uff0c\u8bf7\u5148\u5b89\u88c5\u90e8\u7f72 JindoSDK\u3002\u90e8\u7f72\u65b9\u5f0f\u8bf7\u53c2\u8003 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u90e8\u7f72 JindoSDK 4.6.x \u53ca\u4ee5\u4e0a\u7248\u672c\u3002","title":"\u73af\u5883\u51c6\u5907"},{"location":"oss/usages/oss_jindo_cli/#jindo-cli","text":"\u901a\u8fc7 Jindo CLI \u547d\u4ee4\u8bbf\u95ee OSS \u548c OSS-HDFS \u7684\u65b9\u5f0f\u57fa\u672c\u76f8\u540c\uff0c\u4ec5\u8def\u5f84\u4e2d\u7684 endpoint \u7565\u6709\u4e0d\u540c\u3002\u6839\u8def\u5f84\u793a\u4f8b\u53ef\u53c2\u89c1\uff1a \u300aOSS/OSS-HDFS \u5feb\u901f\u5165\u95e8\u300b \u4e0a\u4f20\u6587\u4ef6 \u5c06\u672c\u5730\u6839\u76ee\u5f55\u4e0b\u7684examplefile.txt\u6587\u4ef6\u4e0a\u4f20\u81f3examplebucket\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -put examplefile.txt oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u65b0\u5efa\u76ee\u5f55 \u5728examplebucket\u4e0b\u521b\u5efa\u540d\u4e3adir/\u7684\u76ee\u5f55\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -mkdir oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/dir/ \u67e5\u770b\u6587\u4ef6\u6216\u76ee\u5f55\u4fe1\u606f \u67e5\u770bexamplebucket\u4e0b\u7684\u6587\u4ef6\u6216\u76ee\u5f55\u4fe1\u606f\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -ls oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u83b7\u53d6\u6587\u4ef6\u6216\u76ee\u5f55\u5927\u5c0f \u83b7\u53d6examplebucket\u4e0b\u6240\u6709\u6587\u4ef6\u6216\u76ee\u5f55\u7684\u5927\u5c0f\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -du oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/ \u67e5\u770b\u6587\u4ef6\u5185\u5bb9 \u67e5\u770bexamplebucket\u4e0b\u540d\u4e3alocalfile.txt\u7684\u6587\u4ef6\u5185\u5bb9\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -cat oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/localfile.txt \u91cd\u8981 \u67e5\u770b\u6587\u4ef6\u5185\u5bb9\u65f6\uff0c\u6587\u4ef6\u5185\u5bb9\u5c06\u4ee5\u7eaf\u6587\u672c\u5f62\u5f0f\u6253\u5370\u5230\u5c4f\u5e55\u4e0a\u3002\u5982\u679c\u6587\u4ef6\u5185\u5bb9\u8fdb\u884c\u4e86\u7279\u5b9a\u683c\u5f0f\u7684\u7f16\u7801\uff0c\u8bf7\u4f7f\u7528HDFS\u7684Java API\u8bfb\u53d6\u5e76\u89e3\u7801\u6587\u4ef6\u5185\u5bb9\u3002 \u4e0b\u8f7d\u6587\u4ef6 \u5c06examplebucket\u4e0b\u7684exampleobject.txt\u4e0b\u8f7d\u5230\u672c\u5730\u6839\u76ee\u5f55\u6587\u4ef6\u5939/tmp\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -get oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/exampleobject.txt /tmp/ \u5220\u9664\u76ee\u5f55\u6216\u6587\u4ef6 \u5220\u9664examplebucket\u4e0bdestfolder/\u76ee\u5f55\u53ca\u5176\u76ee\u5f55\u4e0b\u7684\u6240\u6709\u6587\u4ef6\uff0c\u8bbf\u95ee OSS-HDFS \u793a\u4f8b\u5982\u4e0b\uff1a jindo fs -rm oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/destfolder/","title":"\u901a\u8fc7 Jindo CLI \u547d\u4ee4\u8bbf\u95ee"},{"location":"oss/usages/oss_jindo_cli/#_2","text":"\u66f4\u591a\u547d\u4ee4\u53ca\u63cf\u8ff0\uff0c\u8be6\u89c1 \u300aJindo CLI \u53c2\u6570\u8bf4\u660e\u300b","title":"\u9644\u5f55"},{"location":"oss/usages/oss_posix/","text":"\u901a\u8fc7 POSIX \u8bbf\u95ee OSS/OSS-HDFS \u00a7 \u73af\u5883\u51c6\u5907 \u00a7 EMR\u73af\u5883\u4e2d\uff0c\u9ed8\u8ba4\u5df2\u5b89\u88c5 JindoSDK\uff0c\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528\u3002\u6ce8\u610f\uff1a \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u521b\u5efaEMR-3.42.0\u53ca\u4ee5\u4e0a\u7248\u672c\u6216EMR-5.8.0\u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u96c6\u7fa4\u3002 \u975e EMR \u73af\u5883\uff0c\u8bf7\u5148\u5b89\u88c5\u90e8\u7f72 JindoSDK\u3002\u90e8\u7f72\u65b9\u5f0f\u8bf7\u53c2\u8003 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u90e8\u7f72 JindoSDK 4.x \u53ca\u4ee5\u4e0a\u7248\u672c\u3002 \u4f9d\u8d56\u51c6\u5907 \u00a7 EMR-3.44.0\u53ca\u4ee5\u4e0a\u7248\u672c\u6216EMR-5.10.0\u53ca\u4ee5\u4e0a\u7248\u672c\uff0c\u5df2\u9ed8\u8ba4\u5b89\u88c5\u6240\u9700\u4f9d\u8d56\uff0c\u53ef\u8df3\u8fc7\u672c\u7ae0\u8282\u3002 JindoSDK 4.5.0 \u53ca\u4e4b\u524d\u7248\u672c # CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev JindoSDK 4.5.1 \u53ca\u4ee5\u4e0a\u7248\u672c\uff0c\u9700\u4f9d\u8d56libfuse 3.7+\uff0c\u4ee5\u5b89\u88c5fuse-3.11\u4e3a\u4f8b\uff1a # build fuse required meson & ninja, for debian: apt install -y pkg-config meson ninja-build sudo yum install -y meson ninja-build # compile fuse required newer g++ (only CentOS) sudo yum install -y scl-utils sudo yum install -y alinux-release-experimentals sudo yum install -y devtoolset-8-gcc devtoolset-8-gdb devtoolset-8-binutils devtoolset-8-make devtoolset-8-gcc-c++ sudo su -c \"echo 'source /opt/rh/devtoolset-8/enable' > /etc/profile.d/g++.sh\" source /opt/rh/devtoolset-8/enable sudo ln -s /opt/rh/devtoolset-8/root/bin/gcc /usr/local/bin/gcc sudo ln -s /opt/rh/devtoolset-8/root/bin/g++ /usr/local/bin/g++ # compile & install libfuse wget https://github.com/libfuse/libfuse/releases/download/fuse-3.11.0/fuse-3.11.0.tar.xz xz -d fuse-3.11.0.tar.xz tar xf fuse-3.11.0.tar cd fuse-3.11.0/ mkdir build; cd build meson .. sudo ninja install \u6302\u8f7d JindoFuse \u00a7 \u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002 \u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a mkdir -p <mountpoint> \u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a jindo-fuse <mount_point> -ouri=[<osspath>] \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 <oss_path> \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 <mount_point> \u3002 <mount_point> \u9700\u66ff\u6362\u4e3a\u4e00\u4e2a\u672c\u5730\u8def\u5f84\u3002 <oss_path> \u9700\u66ff\u6362\u4e3a\u5f85\u6620\u5c04\u7684 OSS/OSS-HDFS \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a Bucket \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002\u5982 oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/subdir/ \u6302\u8f7d OSS \u548c OSS-HDFS \u8def\u5f84\u7684\u65b9\u5f0f\u57fa\u672c\u76f8\u540c\uff0c\u4ec5\u8def\u5f84\u4e2d\u7684 endpoint \u7565\u6709\u4e0d\u540c\u3002\u6839\u8def\u5f84\u793a\u4f8b\u53ef\u53c2\u89c1\uff1a \u300aOSS/OSS-HDFS \u5feb\u901f\u5165\u95e8\u300b \u786e\u8ba4\u6302\u8f7d\u6210\u529f\uff0cjindo-fuse \u8fdb\u7a0b\u5b58\u5728\uff0c\u4e14\u542f\u52a8\u53c2\u6570\u4e0e\u9884\u671f\u4e00\u81f4 ps -ef | grep jindo-fuse \u8bbf\u95ee JindoFuse \u00a7 \u5982\u679c\u5c06 JindoFS \u670d\u52a1\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/ \u5378\u8f7d JindoFuse \u00a7 \u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point> \u81ea\u52a8\u5378\u8f7d JindoFuse \u00a7 \u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u9644\u5f55 \u00a7 \u66f4\u591a\u547d\u4ee4\u53ca\u63cf\u8ff0\uff0c\u8be6\u89c1 \u300aJindoFUSE \u4f7f\u7528\u6307\u5357\u300b","title":"\u7528 POSIX \u547d\u4ee4\u8bbf\u95ee OSS/OSS-HDFS"},{"location":"oss/usages/oss_posix/#posix-ossoss-hdfs","text":"","title":"\u901a\u8fc7 POSIX \u8bbf\u95ee OSS/OSS-HDFS"},{"location":"oss/usages/oss_posix/#_1","text":"EMR\u73af\u5883\u4e2d\uff0c\u9ed8\u8ba4\u5df2\u5b89\u88c5 JindoSDK\uff0c\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528\u3002\u6ce8\u610f\uff1a \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u521b\u5efaEMR-3.42.0\u53ca\u4ee5\u4e0a\u7248\u672c\u6216EMR-5.8.0\u53ca\u4ee5\u4e0a\u7248\u672c\u7684\u96c6\u7fa4\u3002 \u975e EMR \u73af\u5883\uff0c\u8bf7\u5148\u5b89\u88c5\u90e8\u7f72 JindoSDK\u3002\u90e8\u7f72\u65b9\u5f0f\u8bf7\u53c2\u8003 \u300a\u5728 Hadoop \u73af\u5883\u4e2d\u90e8\u7f72 JindoSDK\u300b \u3002 \u8bbf\u95ee OSS-HDFS\uff0c\u9700\u90e8\u7f72 JindoSDK 4.x \u53ca\u4ee5\u4e0a\u7248\u672c\u3002","title":"\u73af\u5883\u51c6\u5907"},{"location":"oss/usages/oss_posix/#_2","text":"EMR-3.44.0\u53ca\u4ee5\u4e0a\u7248\u672c\u6216EMR-5.10.0\u53ca\u4ee5\u4e0a\u7248\u672c\uff0c\u5df2\u9ed8\u8ba4\u5b89\u88c5\u6240\u9700\u4f9d\u8d56\uff0c\u53ef\u8df3\u8fc7\u672c\u7ae0\u8282\u3002 JindoSDK 4.5.0 \u53ca\u4e4b\u524d\u7248\u672c # CentOS yum install -y fuse3 fuse3-devel # Debian apt install -y fuse3 libfuse3-dev JindoSDK 4.5.1 \u53ca\u4ee5\u4e0a\u7248\u672c\uff0c\u9700\u4f9d\u8d56libfuse 3.7+\uff0c\u4ee5\u5b89\u88c5fuse-3.11\u4e3a\u4f8b\uff1a # build fuse required meson & ninja, for debian: apt install -y pkg-config meson ninja-build sudo yum install -y meson ninja-build # compile fuse required newer g++ (only CentOS) sudo yum install -y scl-utils sudo yum install -y alinux-release-experimentals sudo yum install -y devtoolset-8-gcc devtoolset-8-gdb devtoolset-8-binutils devtoolset-8-make devtoolset-8-gcc-c++ sudo su -c \"echo 'source /opt/rh/devtoolset-8/enable' > /etc/profile.d/g++.sh\" source /opt/rh/devtoolset-8/enable sudo ln -s /opt/rh/devtoolset-8/root/bin/gcc /usr/local/bin/gcc sudo ln -s /opt/rh/devtoolset-8/root/bin/g++ /usr/local/bin/g++ # compile & install libfuse wget https://github.com/libfuse/libfuse/releases/download/fuse-3.11.0/fuse-3.11.0.tar.xz xz -d fuse-3.11.0.tar.xz tar xf fuse-3.11.0.tar cd fuse-3.11.0/ mkdir build; cd build meson .. sudo ninja install","title":"\u4f9d\u8d56\u51c6\u5907"},{"location":"oss/usages/oss_posix/#jindofuse","text":"\u5728\u5b8c\u6210\u5bf9 JindoSDK \u7684\u914d\u7f6e\u540e\u3002 \u521b\u5efa\u4e00\u4e2a\u6302\u8f7d\u70b9\uff0c \u547d\u4ee4\u5982\u4e0b\uff1a mkdir -p <mountpoint> \u6302\u8f7d Fuse, \u547d\u4ee4\u5982\u4e0b\uff1a jindo-fuse <mount_point> -ouri=[<osspath>] \u8fd9\u4e2a\u547d\u4ee4\u4f1a\u542f\u52a8\u4e00\u4e2a\u540e\u53f0\u7684\u5b88\u62a4\u8fdb\u7a0b\uff0c\u5c06\u6307\u5b9a\u7684 <oss_path> \u6302\u8f7d\u5230\u672c\u5730\u6587\u4ef6\u7cfb\u7edf\u7684 <mount_point> \u3002 <mount_point> \u9700\u66ff\u6362\u4e3a\u4e00\u4e2a\u672c\u5730\u8def\u5f84\u3002 <oss_path> \u9700\u66ff\u6362\u4e3a\u5f85\u6620\u5c04\u7684 OSS/OSS-HDFS \u8def\u5f84\uff0c\u8def\u5f84\u53ef\u4ee5\u4e3a Bucket \u6839\u76ee\u5f55\u6216\u8005\u5b50\u76ee\u5f55\u3002\u5982 oss://examplebucket.cn-shanghai.oss-dls.aliyuncs.com/subdir/ \u6302\u8f7d OSS \u548c OSS-HDFS \u8def\u5f84\u7684\u65b9\u5f0f\u57fa\u672c\u76f8\u540c\uff0c\u4ec5\u8def\u5f84\u4e2d\u7684 endpoint \u7565\u6709\u4e0d\u540c\u3002\u6839\u8def\u5f84\u793a\u4f8b\u53ef\u53c2\u89c1\uff1a \u300aOSS/OSS-HDFS \u5feb\u901f\u5165\u95e8\u300b \u786e\u8ba4\u6302\u8f7d\u6210\u529f\uff0cjindo-fuse \u8fdb\u7a0b\u5b58\u5728\uff0c\u4e14\u542f\u52a8\u53c2\u6570\u4e0e\u9884\u671f\u4e00\u81f4 ps -ef | grep jindo-fuse","title":"\u6302\u8f7d JindoFuse"},{"location":"oss/usages/oss_posix/#jindofuse_1","text":"\u5982\u679c\u5c06 JindoFS \u670d\u52a1\u6302\u8f7d\u5230\u4e86\u672c\u5730 /mnt/oss/\uff0c\u53ef\u4ee5\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u8bbf\u95ee JindoFuse\u3002 \u5217\u51fa/mnt/oss/\u4e0b\u7684\u6240\u6709\u76ee\u5f55\uff1a ls /mnt/oss/ \u521b\u5efa\u76ee\u5f55\uff1a mkdir /mnt/oss/dir1ls /mnt/oss/ \u5199\u5165\u6587\u4ef6\uff1a echo \"hello world\" > /mnt/oss/dir1/hello.txt \u8bfb\u53d6\u6587\u4ef6\uff1a cat /mnt/oss/dir1/hello.txt \u663e\u793a hello world \u3002 \u5220\u9664\u76ee\u5f55\uff1a rm -rf /mnt/oss/dir1/","title":"\u8bbf\u95ee JindoFuse"},{"location":"oss/usages/oss_posix/#jindofuse_2","text":"\u60f3\u5378\u8f7d\u4e4b\u524d\u6302\u8f7d\u7684\u6302\u8f7d\u70b9\uff0c\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4\uff1a umount <mount_point>","title":"\u5378\u8f7d JindoFuse"},{"location":"oss/usages/oss_posix/#jindofuse_3","text":"\u53ef\u4ee5\u4f7f\u7528 -oauto_unmount \u53c2\u6570\uff0c\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002 \u4f7f\u7528\u8be5\u53c2\u6570\u540e\uff0c\u53ef\u4ee5\u652f\u6301 killall -9 jindo-fuse \u53d1\u9001 SIGINT \u7ed9 jindo-fuse \u8fdb\u7a0b\uff0c\u8fdb\u7a0b\u9000\u51fa\u524d\u4f1a\u81ea\u52a8\u5378\u8f7d\u6302\u8f7d\u70b9\u3002","title":"\u81ea\u52a8\u5378\u8f7d JindoFuse"},{"location":"oss/usages/oss_posix/#_3","text":"\u66f4\u591a\u547d\u4ee4\u53ca\u63cf\u8ff0\uff0c\u8be6\u89c1 \u300aJindoFUSE \u4f7f\u7528\u6307\u5357\u300b","title":"\u9644\u5f55"},{"location":"upgrade/emr2_upgrade_jindocache/","text":"EMR \u65b0\u7248\u96c6\u7fa4 JindoCache \u5347\u7ea7\u6587\u6863 \u00a7 \u573a\u666f\u4e00\uff1a\u5347\u7ea7\u5df2\u6709\u65b0\u7248\u96c6\u7fa4 \u00a7 \u82e5\u5df2\u6709\u65b0\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u3002\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u6216\u8005\u9700\u8981\u4f7f\u7528 JindoCache \u7684\u65b0\u529f\u80fd, \u53ef\u4ee5\u6839\u636e\u4e0b\u9762\u7684\u6b65\u9aa4\u5b8c\u6210 JindoCache \u5347\u7ea7\u3002 1\u3001\u505c\u6b62\u5df2\u6709\u7684 JindoCache \u96c6\u7fa4 \u00a7 \u5728EMR\u96c6\u7fa4\u7684 JindoCache \u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u505c\u6b62 \u3002 2\u3001\u51c6\u5907\u8f6f\u4ef6\u5305\u548c\u5347\u7ea7\u811a\u672c\u3001\u914d\u7f6e \u00a7 \u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728emr-user\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528emr-user\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - emr-user cd /home/emr-user/ wget http://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindocache-patches.tar.gz tar zxf jindocache-patches.tar.gz \u4e0b\u8f7d jindocache \u8f6f\u4ef6\u5305 jindocache-{VERSION}-{PLATFORM}.tar.gz\uff0c\u89e3\u538b\u540e\u5230 jindocache-patches\u76ee\u5f55\u4e0b\u3002 \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 jindocache \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86 \u5e73\u53f0\u4e3a\u4f8b: cd jindocache-patches wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindocache-6.7.3-linux.tar.gz tar -zxvf jindocache-6.7.3-linux.tar.gz ls -l jindocache-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-rw-r-- 1 emr-user emr-user 29 6\u6708 6 11:21 hosts drwxr-xr-x 7 emr-user emr-user 4096 8\u6708 8 16:19 jindocache-6.7.3-linux -rw-r--r-- 1 emr-user emr-user 276406433 8\u6708 12 14:32 jindocache-6.7.3-linux.tar.gz -rwxr-xr-x 1 emr-user emr-user 1241 6\u6708 6 11:18 upgrade.sh 3\u3001\u914d\u7f6e\u5347\u7ea7\u8282\u70b9\u4fe1\u606f \u00a7 \u7f16\u8f91 patch \u5305\u4e0b\u7684 hosts \u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684 hostname\uff0c\u5982 master-1-1 \u6216 core-1-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindocache-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a master-1-1 core-1-1 core-1-2 \u53ef\u5c1d\u8bd5\u4f7f\u7528\u811a\u672c\u83b7\u53d6\u5168\u90e8\u8282\u70b9\u4fe1\u606f\uff0c\u5982\u679c hosts \u83b7\u53d6\u5931\u8d25\uff0c\u9700\u8981\u624b\u52a8\u8865\u5168 cat /usr/local/taihao-executor-all/data/cache/.cluster_context | jq --raw-output '.nodes[].hostname.alias[]' > hosts 4\u3001\u6267\u884c\u5347\u7ea7 \u00a7 \u901a\u8fc7 upgrade.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 bash upgrade.sh $NEW_jindocache_VERSION $PLATFORM \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 jindocache \u5347\u7ea7\u5230 linux \u5e73\u53f0 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash upgrade.sh 6.7.3 linux \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 Start upgrade JindoCache Service to 6.7.3 with platform linux [1] 11:14:25 [SUCCESS] core-1-2 [2] 11:14:26 [SUCCESS] core-1-1 [3] 11:14:27 [SUCCESS] master-1-1 [1] 11:14:52 [SUCCESS] core-1-2 [2] 11:14:52 [SUCCESS] master-1-1 [3] 11:14:52 [SUCCESS] core-1-1 [1] 11:14:55 [SUCCESS] core-1-2 [2] 11:14:55 [SUCCESS] master-1-1 [3] 11:14:55 [SUCCESS] core-1-1 [1] 11:14:56 [SUCCESS] core-1-1 [2] 11:14:56 [SUCCESS] core-1-2 [3] 11:14:58 [SUCCESS] master-1-1 [1] 11:14:58 [SUCCESS] core-1-2 [2] 11:14:58 [SUCCESS] core-1-1 [3] 11:15:00 [SUCCESS] master-1-1 [1] 11:15:01 [SUCCESS] master-1-1 [2] 11:15:01 [SUCCESS] core-1-2 [3] 11:15:02 [SUCCESS] core-1-1 [1] 11:15:02 [SUCCESS] core-1-2 [2] 11:15:02 [SUCCESS] core-1-1 [3] 11:15:02 [SUCCESS] master-1-1 Successfully upgrade JindoCache Service to 6.7.3 with platform linux, Please restart JindoCache Service 5\u3001\u786e\u8ba4\u5347\u7ea7\u6210\u529f \u00a7 [root@master-1-1]# jindocache -version Version: 6.7.3 CommitId: ###### 6\u3001\u5347\u7ea7\u540e\u542f\u52a8 JindoCache \u670d\u52a1 \u00a7 \u5728EMR\u96c6\u7fa4\u7684 JindoCache \u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u542f\u52a8 \u3002 \u542f\u52a8\u5b8c\u6210\u540e\uff0c\u786e\u8ba4\u96c6\u7fa4\u670d\u52a1\u6b63\u5e38 [root@master-1-1]# jindocache -report Namespace Address: master-1-1:8101 Rpc Port: 8101 Started: Thu Jun 6 11:06:05 2024 Version: 6.7.3 Live Nodes: 2 Decommission Nodes: 0 Total Disk Capacity: 625.519GB Used Disk Capacity: 0B Total MEM Capacity: 0B Used MEM Capacity: 0B \u573a\u666f\u4e8c\uff1a\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4 \u00a7 \u82e5\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 JindoCache, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 JindoCache \u5347\u7ea7\u3002 1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u4e0b\u8f7d\u7684 jindocache-patches.tar.gz, jindocache-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindocache.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 jindocache \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindocache-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindocache-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindocache.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindocache.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindocache-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindocache-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindocache.sh -gen $NEW_JINDOCACHE_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 jindocache \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindocache.sh -gen 6.7.3 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindocache-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindocache-bootstrap-patches.tar.gz \u3002 2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ hadoop dfs -put jindocache-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindocache.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindocache.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindocache-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindocache-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindocache.sh \u3002 3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u00a7 \u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOCACHE\u3002 update_jindocache \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindocache.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindocache-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002 \u573a\u666f\u4e09\uff1a\u65b0\u5efa\u96c6\u7fa4 \u00a7 \u82e5\u65b0\u5efa EMR \u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 jindocache, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 jindocache \u5347\u7ea7\u3002 1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u4e0b\u8f7d\u7684 jindocache-patches.tar.gz \uff0cjindocache-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindocache.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 jindocache \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindocache-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindocache-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindocache.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindocache.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindocache-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindocache-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindocache.sh -gen $NEW_JINDOCACHE_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 Jindocache \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindocache.sh -gen 6.7.3 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindocache-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindocache-bootstrap-patches.tar.gz \u3002 2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ hadoop dfs -put jindocache-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindocache.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindocache.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindocache-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindocache-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindocache.sh \u3002 3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u00a7 \u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOCACHE\u3002 update_jindocache \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindocache.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindocache-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0ax86\u7248\u672c\uff08\u65b0\u7248\u63a7\u5236\u53f0\uff09"},{"location":"upgrade/emr2_upgrade_jindocache/#emr-jindocache","text":"","title":"EMR \u65b0\u7248\u96c6\u7fa4 JindoCache \u5347\u7ea7\u6587\u6863"},{"location":"upgrade/emr2_upgrade_jindocache/#_1","text":"\u82e5\u5df2\u6709\u65b0\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u3002\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u6216\u8005\u9700\u8981\u4f7f\u7528 JindoCache \u7684\u65b0\u529f\u80fd, \u53ef\u4ee5\u6839\u636e\u4e0b\u9762\u7684\u6b65\u9aa4\u5b8c\u6210 JindoCache \u5347\u7ea7\u3002","title":"\u573a\u666f\u4e00\uff1a\u5347\u7ea7\u5df2\u6709\u65b0\u7248\u96c6\u7fa4"},{"location":"upgrade/emr2_upgrade_jindocache/#1-jindocache","text":"\u5728EMR\u96c6\u7fa4\u7684 JindoCache \u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u505c\u6b62 \u3002","title":"1\u3001\u505c\u6b62\u5df2\u6709\u7684 JindoCache \u96c6\u7fa4"},{"location":"upgrade/emr2_upgrade_jindocache/#2","text":"\u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728emr-user\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528emr-user\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - emr-user cd /home/emr-user/ wget http://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindocache-patches.tar.gz tar zxf jindocache-patches.tar.gz \u4e0b\u8f7d jindocache \u8f6f\u4ef6\u5305 jindocache-{VERSION}-{PLATFORM}.tar.gz\uff0c\u89e3\u538b\u540e\u5230 jindocache-patches\u76ee\u5f55\u4e0b\u3002 \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 jindocache \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86 \u5e73\u53f0\u4e3a\u4f8b: cd jindocache-patches wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindocache-6.7.3-linux.tar.gz tar -zxvf jindocache-6.7.3-linux.tar.gz ls -l jindocache-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-rw-r-- 1 emr-user emr-user 29 6\u6708 6 11:21 hosts drwxr-xr-x 7 emr-user emr-user 4096 8\u6708 8 16:19 jindocache-6.7.3-linux -rw-r--r-- 1 emr-user emr-user 276406433 8\u6708 12 14:32 jindocache-6.7.3-linux.tar.gz -rwxr-xr-x 1 emr-user emr-user 1241 6\u6708 6 11:18 upgrade.sh","title":"2\u3001\u51c6\u5907\u8f6f\u4ef6\u5305\u548c\u5347\u7ea7\u811a\u672c\u3001\u914d\u7f6e"},{"location":"upgrade/emr2_upgrade_jindocache/#3","text":"\u7f16\u8f91 patch \u5305\u4e0b\u7684 hosts \u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684 hostname\uff0c\u5982 master-1-1 \u6216 core-1-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindocache-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a master-1-1 core-1-1 core-1-2 \u53ef\u5c1d\u8bd5\u4f7f\u7528\u811a\u672c\u83b7\u53d6\u5168\u90e8\u8282\u70b9\u4fe1\u606f\uff0c\u5982\u679c hosts \u83b7\u53d6\u5931\u8d25\uff0c\u9700\u8981\u624b\u52a8\u8865\u5168 cat /usr/local/taihao-executor-all/data/cache/.cluster_context | jq --raw-output '.nodes[].hostname.alias[]' > hosts","title":"3\u3001\u914d\u7f6e\u5347\u7ea7\u8282\u70b9\u4fe1\u606f"},{"location":"upgrade/emr2_upgrade_jindocache/#4","text":"\u901a\u8fc7 upgrade.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 bash upgrade.sh $NEW_jindocache_VERSION $PLATFORM \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 jindocache \u5347\u7ea7\u5230 linux \u5e73\u53f0 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash upgrade.sh 6.7.3 linux \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 Start upgrade JindoCache Service to 6.7.3 with platform linux [1] 11:14:25 [SUCCESS] core-1-2 [2] 11:14:26 [SUCCESS] core-1-1 [3] 11:14:27 [SUCCESS] master-1-1 [1] 11:14:52 [SUCCESS] core-1-2 [2] 11:14:52 [SUCCESS] master-1-1 [3] 11:14:52 [SUCCESS] core-1-1 [1] 11:14:55 [SUCCESS] core-1-2 [2] 11:14:55 [SUCCESS] master-1-1 [3] 11:14:55 [SUCCESS] core-1-1 [1] 11:14:56 [SUCCESS] core-1-1 [2] 11:14:56 [SUCCESS] core-1-2 [3] 11:14:58 [SUCCESS] master-1-1 [1] 11:14:58 [SUCCESS] core-1-2 [2] 11:14:58 [SUCCESS] core-1-1 [3] 11:15:00 [SUCCESS] master-1-1 [1] 11:15:01 [SUCCESS] master-1-1 [2] 11:15:01 [SUCCESS] core-1-2 [3] 11:15:02 [SUCCESS] core-1-1 [1] 11:15:02 [SUCCESS] core-1-2 [2] 11:15:02 [SUCCESS] core-1-1 [3] 11:15:02 [SUCCESS] master-1-1 Successfully upgrade JindoCache Service to 6.7.3 with platform linux, Please restart JindoCache Service","title":"4\u3001\u6267\u884c\u5347\u7ea7"},{"location":"upgrade/emr2_upgrade_jindocache/#5","text":"[root@master-1-1]# jindocache -version Version: 6.7.3 CommitId: ######","title":"5\u3001\u786e\u8ba4\u5347\u7ea7\u6210\u529f"},{"location":"upgrade/emr2_upgrade_jindocache/#6-jindocache","text":"\u5728EMR\u96c6\u7fa4\u7684 JindoCache \u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u542f\u52a8 \u3002 \u542f\u52a8\u5b8c\u6210\u540e\uff0c\u786e\u8ba4\u96c6\u7fa4\u670d\u52a1\u6b63\u5e38 [root@master-1-1]# jindocache -report Namespace Address: master-1-1:8101 Rpc Port: 8101 Started: Thu Jun 6 11:06:05 2024 Version: 6.7.3 Live Nodes: 2 Decommission Nodes: 0 Total Disk Capacity: 625.519GB Used Disk Capacity: 0B Total MEM Capacity: 0B Used MEM Capacity: 0B","title":"6\u3001\u5347\u7ea7\u540e\u542f\u52a8 JindoCache \u670d\u52a1"},{"location":"upgrade/emr2_upgrade_jindocache/#_2","text":"\u82e5\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 JindoCache, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 JindoCache \u5347\u7ea7\u3002","title":"\u573a\u666f\u4e8c\uff1a\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4"},{"location":"upgrade/emr2_upgrade_jindocache/#1","text":"\u4e0b\u8f7d\u7684 jindocache-patches.tar.gz, jindocache-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindocache.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 jindocache \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindocache-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindocache-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindocache.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindocache.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindocache-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindocache-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindocache.sh -gen $NEW_JINDOCACHE_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 jindocache \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindocache.sh -gen 6.7.3 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindocache-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindocache-bootstrap-patches.tar.gz \u3002","title":"1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindocache/#2_1","text":"\u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ hadoop dfs -put jindocache-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindocache.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindocache.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindocache-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindocache-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindocache.sh \u3002","title":"2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindocache/#3-emr","text":"\u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOCACHE\u3002 update_jindocache \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindocache.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindocache-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c"},{"location":"upgrade/emr2_upgrade_jindocache/#_3","text":"\u82e5\u65b0\u5efa EMR \u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 jindocache, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 jindocache \u5347\u7ea7\u3002","title":"\u573a\u666f\u4e09\uff1a\u65b0\u5efa\u96c6\u7fa4"},{"location":"upgrade/emr2_upgrade_jindocache/#1_1","text":"\u4e0b\u8f7d\u7684 jindocache-patches.tar.gz \uff0cjindocache-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindocache.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 jindocache \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindocache-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindocache-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindocache.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindocache.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindocache-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindocache-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindocache.sh -gen $NEW_JINDOCACHE_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 Jindocache \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindocache.sh -gen 6.7.3 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindocache-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindocache-bootstrap-patches.tar.gz \u3002","title":"1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindocache/#2_2","text":"\u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ hadoop dfs -put jindocache-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindocache.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindocache.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindocache-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindocache-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindocache.sh \u3002","title":"2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindocache/#3-emr_1","text":"\u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOCACHE\u3002 update_jindocache \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindocache.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindocache-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c"},{"location":"upgrade/emr2_upgrade_jindosdk/","text":"EMR \u65b0\u7248\u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6587\u6863 \u00a7 \u573a\u666f\u4e00\uff1a\u5347\u7ea7\u5df2\u6709\u65b0\u7248\u96c6\u7fa4 \u00a7 \u82e5\u5df2\u6709\u65b0\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u3002\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u6216\u8005\u9700\u8981\u4f7f\u7528 JindoSDK \u7684\u65b0\u529f\u80fd\uff0c\u5177\u4f53\u67e5\u770b \u7248\u672c\u8bf4\u660e , \u53ef\u4ee5\u6839\u636e\u4e0b\u9762\u7684\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002 1. \u51c6\u5907\u8f6f\u4ef6\u5305\u548c\u5347\u7ea7\u811a\u672c\u3001\u914d\u7f6e \u00a7 \u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728emr-user\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528emr-user\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - emr-user cd /home/emr-user/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz \u4e0b\u8f7d JindoSDK \u8f6f\u4ef6\u5305 jindosdk-{VERSION}-{PLATFORM}.tar.gz\uff0c\u653e\u5728\u89e3\u538b\u540e\u7684\u76ee\u5f55\u3002 \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86 \u5e73\u53f0\u4e3a\u4f8b: cd jindosdk-patches wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 emr-user emr-user 2439 May 01 00:00 apply_all.sh -rwxrwxr-x 1 emr-user emr-user 7315 May 01 00:00 apply.sh -rw-rw-r-- 1 emr-user emr-user 40 May 01 00:00 hosts -rw-r----- 1 emr-user emr-user xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rwxrwxr-x 1 emr-user emr-user 1112 May 01 00:00 revert_all.sh -rwxrwxr-x 1 emr-user emr-user 2042 May 01 00:00 revert.sh \u6ce8\u610f \uff1a\u5982\u679c\u4ece4.6.8\u4ee5\u4e0b\u7248\u672c\u5347\u7ea7\u52304.6.9\u4ee5\u4e0a\u62166.x\u7248\u672c\u65f6\uff0c\u7531\u4e8e JindoCommitter \u9ed8\u8ba4\u4f7f\u7528\u7684\u4f5c\u4e1a\u4e34\u65f6\u8def\u5f84\u53d1\u751f\u53d8\u5316\uff0c\u9700\u8981\u5728\u5347\u7ea7\u524d \u5148\u8bbe\u7f6e fs.jdo.committer.allow.concurrent=false \uff08core-site.xml\uff09 \u6216\u5728 Spark \u914d\u7f6e\u4e2d\u8bbe\u7f6e spark.hadoop.fs.jdo.committer.allow.concurrent=false \uff0c\u786e\u4fdd\u5347\u7ea7\u671f\u95f4\u4e0d\u4f1a\u51fa\u73b0\u4e22\u6570\u636e\u7684\u60c5\u51b5\u3002 \u540e\u7eed\u5728\u5305\u542bGATEWAY\u8282\u70b9\u7684\u6240\u6709JindoSDK\u5168\u90e8\u5347\u7ea7\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u62e9\u673a\u53bb\u6389\u8be5\u914d\u7f6e\u3002 2. \u914d\u7f6e\u5347\u7ea7\u8282\u70b9\u4fe1\u606f \u00a7 \u7f16\u8f91 patch \u5305\u4e0b\u7684 hosts \u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684 hostname\uff0c\u5982 master-1-1 \u6216 core-1-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a master-1-1 core-1-1 core-1-2 \u53ef\u5c1d\u8bd5\u4f7f\u7528\u811a\u672c\u83b7\u53d6\u5168\u90e8\u8282\u70b9\u4fe1\u606f\uff0c\u5982\u679c hosts \u83b7\u53d6\u5931\u8d25\uff0c\u9700\u8981\u624b\u52a8\u8865\u5168 cat /usr/local/taihao-executor-all/data/cache/.cluster_context | jq --raw-output '.nodes[].hostname.alias[]' > hosts 3. \u6267\u884c\u5347\u7ea7 \u00a7 \u901a\u8fc7 apply_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./apply_all.sh $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: ./apply_all.sh 6.7.3 \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... master-1-1 >>> updating ... core-1-1 >>> updating ... core-1-2 ### DONE 4. \u4fee\u6539\u96c6\u7fa4\u914d\u7f6e \u00a7 4.1 \u517c\u5bb9\u8001\u7248\u672c EMR Ranger \u7ec4\u4ef6 \u00a7 \u6ce8\u610f \uff1a\u5982\u679c\u5f00\u542f\u4e86 EMR Ranger\uff0c\u4e14\u4ece EMR-3.51.1/EMR-5.17.1 \u53ca\u4e4b\u524d\u7248\u672c\u5347\u7ea7 JindoSDK \u5230 6.5.0 \uff5e 6.7.2 \u65f6\u6709\u517c\u5bb9\u95ee\u9898\uff0c\u5efa\u8bae\u5347\u7ea7\u5230 6.7.3 \u53ca\u4ee5\u4e0a\u7248\u672c, \u5e76\u6309\u4ee5\u4e0b\u65b9\u5f0f\u4fee\u6539\u96c6\u7fa4\u914d\u7f6e\u3002 a. \u5728 HADOOP-COMMON \u670d\u52a1\u7684 \u914d\u7f6e \u9875\u9762\uff0c\u5355\u51fb core-sites.xml \u9875\u7b7e\u3002 b. \u5728 core-sites.xml \u9875\u9762\uff0c\u641c\u7d22 \u914d\u7f6e\u9879\u540d\u79f0 \u3002 c. \u4fee\u6539\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 fs.jdo.plugin.dir \u5c06\u52a0\u8f7d plugin \u76ee\u5f55\uff0c\u5c06 /opt/apps/RANGER/jindoauth-current/plugins \u4fee\u6539\u4e3a /opt/apps/JINDOSDK/jindosdk-current/plugins \u3002 5. \u786e\u8ba4\u5347\u7ea7\u6210\u529f \u00a7 ls -l /opt/apps/JINDOSDK/jindosdk-current/lib \u4ee5\u4ece\u96c6\u7fa4\u9ed8\u8ba4\u7248\u672c 6.2.0 \u5347\u7ea7\u4e3a 6.7.3 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a lrwxrwxrwx 1 root root 64 Apr 12 11:08 jindo-core-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-core-6.7.3.jar lrwxrwxrwx 1 root root 82 Apr 12 11:08 jindo-core-linux-el7-aarch64-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-core-linux-el7-aarch64-6.7.3.jar lrwxrwxrwx 1 root root 63 Apr 12 11:08 jindo-sdk-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-sdk-6.7.3.jar lrwxrwxrwx 1 root root 50 Apr 12 11:08 native -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/native lrwxrwxrwx 1 root root 57 Apr 12 11:08 site-packages -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/site-packages 6. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1 \u00a7 \u8bf4\u660e: \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u91cd\u542f \u3002 \u573a\u666f\u4e8c\uff1a\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4 \u00a7 \u82e5\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 JindoSDK, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002 1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindosdk.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindosdk.sh -gen 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002 2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002 3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u00a7 \u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002 4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d \u00a7 \u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542f Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u573a\u666f\u4e09\uff1a\u65b0\u5efa\u96c6\u7fa4 \u00a7 \u82e5\u65b0\u5efa EMR \u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 JindoSDK, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002 1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindosdk.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen-full $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindosdk.sh -gen-full 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002 2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002 3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u00a7 \u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002 4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d \u00a7 \u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542f Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u573a\u666f\u56db\uff1a\u56de\u6eda JindoSDK \u5230\u96c6\u7fa4\u9ed8\u8ba4\u7248\u672c \u00a7 \u82e5\u5df2\u6709\u65b0\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u3002\u5728\u5347\u7ea7\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u9700\u8981\u56de\u6eda\u5230\u96c6\u7fa4\u9ed8\u8ba4\u7684JindoSDK\u7248\u672c 1. \u51c6\u5907\u56de\u6eda\u811a\u672c \u00a7 \u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728emr-user\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528emr-user\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - emr-user cd /home/emr-user/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz cd jindosdk-patches ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 emr-user emr-user 2439 May 01 00:00 apply_all.sh -rwxrwxr-x 1 emr-user emr-user 7315 May 01 00:00 apply.sh -rw-rw-r-- 1 emr-user emr-user 40 May 01 00:00 hosts -rwxrwxr-x 1 emr-user emr-user 1112 May 01 00:00 revert_all.sh -rwxrwxr-x 1 emr-user emr-user 2042 May 01 00:00 revert.sh 2. \u914d\u7f6e\u56de\u6eda\u8282\u70b9\u4fe1\u606f \u00a7 \u7f16\u8f91 patch \u5305\u4e0b\u7684 hosts \u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684 hostname\uff0c\u5982 master-1-1 \u6216 core-1-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a master-1-1 core-1-1 core-1-2 \u53ef\u5c1d\u8bd5\u4f7f\u7528\u811a\u672c\u83b7\u53d6\u5168\u90e8\u8282\u70b9\u4fe1\u606f\uff0c\u5982\u679c hosts \u83b7\u53d6\u5931\u8d25\uff0c\u9700\u8981\u624b\u52a8\u8865\u5168 cat /usr/local/taihao-executor-all/data/cache/.cluster_context | jq --raw-output '.nodes[].hostname.alias[]' > hosts 3. \u6267\u884c\u56de\u6eda \u00a7 \u901a\u8fc7 revert_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./revert_all.sh \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... master-1-1 >>> updating ... core-1-1 >>> updating ... core-1-2 ### DONE 4. \u786e\u8ba4\u56de\u6eda\u6210\u529f \u00a7 ls -l /opt/apps/JINDOSDK/jindosdk-current/lib \u4ee5\u56de\u6eda\u4e3a 6.2.0 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r--r-- 1 root root 1253740 Apr 24 17:40 jindo-core-6.2.0.jar -rw-r--r-- 1 root root 13110547 Apr 24 17:40 jindo-core-linux-el7-aarch64-6.2.0.jar -rw-r--r-- 1 root root 4432227 Apr 24 17:40 jindo-sdk-6.2.0.jar drwxr-xr-x 2 root root 4096 Apr 24 17:40 native 5. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1 \u00a7 \u8bf4\u660e: \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u91cd\u542f \u3002","title":"EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0ax86\u7248\u672c\uff08\u65b0\u7248\u63a7\u5236\u53f0\uff09"},{"location":"upgrade/emr2_upgrade_jindosdk/#emr-jindosdk","text":"","title":"EMR \u65b0\u7248\u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6587\u6863"},{"location":"upgrade/emr2_upgrade_jindosdk/#_1","text":"\u82e5\u5df2\u6709\u65b0\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u3002\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u6216\u8005\u9700\u8981\u4f7f\u7528 JindoSDK \u7684\u65b0\u529f\u80fd\uff0c\u5177\u4f53\u67e5\u770b \u7248\u672c\u8bf4\u660e , \u53ef\u4ee5\u6839\u636e\u4e0b\u9762\u7684\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002","title":"\u573a\u666f\u4e00\uff1a\u5347\u7ea7\u5df2\u6709\u65b0\u7248\u96c6\u7fa4"},{"location":"upgrade/emr2_upgrade_jindosdk/#1","text":"\u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728emr-user\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528emr-user\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - emr-user cd /home/emr-user/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz \u4e0b\u8f7d JindoSDK \u8f6f\u4ef6\u5305 jindosdk-{VERSION}-{PLATFORM}.tar.gz\uff0c\u653e\u5728\u89e3\u538b\u540e\u7684\u76ee\u5f55\u3002 \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86 \u5e73\u53f0\u4e3a\u4f8b: cd jindosdk-patches wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 emr-user emr-user 2439 May 01 00:00 apply_all.sh -rwxrwxr-x 1 emr-user emr-user 7315 May 01 00:00 apply.sh -rw-rw-r-- 1 emr-user emr-user 40 May 01 00:00 hosts -rw-r----- 1 emr-user emr-user xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rwxrwxr-x 1 emr-user emr-user 1112 May 01 00:00 revert_all.sh -rwxrwxr-x 1 emr-user emr-user 2042 May 01 00:00 revert.sh \u6ce8\u610f \uff1a\u5982\u679c\u4ece4.6.8\u4ee5\u4e0b\u7248\u672c\u5347\u7ea7\u52304.6.9\u4ee5\u4e0a\u62166.x\u7248\u672c\u65f6\uff0c\u7531\u4e8e JindoCommitter \u9ed8\u8ba4\u4f7f\u7528\u7684\u4f5c\u4e1a\u4e34\u65f6\u8def\u5f84\u53d1\u751f\u53d8\u5316\uff0c\u9700\u8981\u5728\u5347\u7ea7\u524d \u5148\u8bbe\u7f6e fs.jdo.committer.allow.concurrent=false \uff08core-site.xml\uff09 \u6216\u5728 Spark \u914d\u7f6e\u4e2d\u8bbe\u7f6e spark.hadoop.fs.jdo.committer.allow.concurrent=false \uff0c\u786e\u4fdd\u5347\u7ea7\u671f\u95f4\u4e0d\u4f1a\u51fa\u73b0\u4e22\u6570\u636e\u7684\u60c5\u51b5\u3002 \u540e\u7eed\u5728\u5305\u542bGATEWAY\u8282\u70b9\u7684\u6240\u6709JindoSDK\u5168\u90e8\u5347\u7ea7\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u62e9\u673a\u53bb\u6389\u8be5\u914d\u7f6e\u3002","title":"1. \u51c6\u5907\u8f6f\u4ef6\u5305\u548c\u5347\u7ea7\u811a\u672c\u3001\u914d\u7f6e"},{"location":"upgrade/emr2_upgrade_jindosdk/#2","text":"\u7f16\u8f91 patch \u5305\u4e0b\u7684 hosts \u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684 hostname\uff0c\u5982 master-1-1 \u6216 core-1-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a master-1-1 core-1-1 core-1-2 \u53ef\u5c1d\u8bd5\u4f7f\u7528\u811a\u672c\u83b7\u53d6\u5168\u90e8\u8282\u70b9\u4fe1\u606f\uff0c\u5982\u679c hosts \u83b7\u53d6\u5931\u8d25\uff0c\u9700\u8981\u624b\u52a8\u8865\u5168 cat /usr/local/taihao-executor-all/data/cache/.cluster_context | jq --raw-output '.nodes[].hostname.alias[]' > hosts","title":"2. \u914d\u7f6e\u5347\u7ea7\u8282\u70b9\u4fe1\u606f"},{"location":"upgrade/emr2_upgrade_jindosdk/#3","text":"\u901a\u8fc7 apply_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./apply_all.sh $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: ./apply_all.sh 6.7.3 \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... master-1-1 >>> updating ... core-1-1 >>> updating ... core-1-2 ### DONE","title":"3. \u6267\u884c\u5347\u7ea7"},{"location":"upgrade/emr2_upgrade_jindosdk/#4","text":"","title":"4. \u4fee\u6539\u96c6\u7fa4\u914d\u7f6e"},{"location":"upgrade/emr2_upgrade_jindosdk/#41-emr-ranger","text":"\u6ce8\u610f \uff1a\u5982\u679c\u5f00\u542f\u4e86 EMR Ranger\uff0c\u4e14\u4ece EMR-3.51.1/EMR-5.17.1 \u53ca\u4e4b\u524d\u7248\u672c\u5347\u7ea7 JindoSDK \u5230 6.5.0 \uff5e 6.7.2 \u65f6\u6709\u517c\u5bb9\u95ee\u9898\uff0c\u5efa\u8bae\u5347\u7ea7\u5230 6.7.3 \u53ca\u4ee5\u4e0a\u7248\u672c, \u5e76\u6309\u4ee5\u4e0b\u65b9\u5f0f\u4fee\u6539\u96c6\u7fa4\u914d\u7f6e\u3002 a. \u5728 HADOOP-COMMON \u670d\u52a1\u7684 \u914d\u7f6e \u9875\u9762\uff0c\u5355\u51fb core-sites.xml \u9875\u7b7e\u3002 b. \u5728 core-sites.xml \u9875\u9762\uff0c\u641c\u7d22 \u914d\u7f6e\u9879\u540d\u79f0 \u3002 c. \u4fee\u6539\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 fs.jdo.plugin.dir \u5c06\u52a0\u8f7d plugin \u76ee\u5f55\uff0c\u5c06 /opt/apps/RANGER/jindoauth-current/plugins \u4fee\u6539\u4e3a /opt/apps/JINDOSDK/jindosdk-current/plugins \u3002","title":"4.1 \u517c\u5bb9\u8001\u7248\u672c EMR Ranger \u7ec4\u4ef6"},{"location":"upgrade/emr2_upgrade_jindosdk/#5","text":"ls -l /opt/apps/JINDOSDK/jindosdk-current/lib \u4ee5\u4ece\u96c6\u7fa4\u9ed8\u8ba4\u7248\u672c 6.2.0 \u5347\u7ea7\u4e3a 6.7.3 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a lrwxrwxrwx 1 root root 64 Apr 12 11:08 jindo-core-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-core-6.7.3.jar lrwxrwxrwx 1 root root 82 Apr 12 11:08 jindo-core-linux-el7-aarch64-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-core-linux-el7-aarch64-6.7.3.jar lrwxrwxrwx 1 root root 63 Apr 12 11:08 jindo-sdk-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-sdk-6.7.3.jar lrwxrwxrwx 1 root root 50 Apr 12 11:08 native -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/native lrwxrwxrwx 1 root root 57 Apr 12 11:08 site-packages -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/site-packages","title":"5. \u786e\u8ba4\u5347\u7ea7\u6210\u529f"},{"location":"upgrade/emr2_upgrade_jindosdk/#6","text":"\u8bf4\u660e: \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u91cd\u542f \u3002","title":"6. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1"},{"location":"upgrade/emr2_upgrade_jindosdk/#_2","text":"\u82e5\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 JindoSDK, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002","title":"\u573a\u666f\u4e8c\uff1a\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4"},{"location":"upgrade/emr2_upgrade_jindosdk/#1_1","text":"\u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindosdk.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindosdk.sh -gen 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002","title":"1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindosdk/#2_1","text":"\u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002","title":"2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindosdk/#3-emr","text":"\u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c"},{"location":"upgrade/emr2_upgrade_jindosdk/#4_1","text":"\u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542f Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002","title":"4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d"},{"location":"upgrade/emr2_upgrade_jindosdk/#_3","text":"\u82e5\u65b0\u5efa EMR \u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 JindoSDK, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002","title":"\u573a\u666f\u4e09\uff1a\u65b0\u5efa\u96c6\u7fa4"},{"location":"upgrade/emr2_upgrade_jindosdk/#1_2","text":"\u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindosdk.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux x86\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen-full $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindosdk.sh -gen-full 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002","title":"1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindosdk/#2_2","text":"\u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002","title":"2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindosdk/#3-emr_1","text":"\u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c"},{"location":"upgrade/emr2_upgrade_jindosdk/#4_2","text":"\u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542f Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002","title":"4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d"},{"location":"upgrade/emr2_upgrade_jindosdk/#jindosdk","text":"\u82e5\u5df2\u6709\u65b0\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u3002\u5728\u5347\u7ea7\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u9700\u8981\u56de\u6eda\u5230\u96c6\u7fa4\u9ed8\u8ba4\u7684JindoSDK\u7248\u672c","title":"\u573a\u666f\u56db\uff1a\u56de\u6eda JindoSDK \u5230\u96c6\u7fa4\u9ed8\u8ba4\u7248\u672c"},{"location":"upgrade/emr2_upgrade_jindosdk/#1_3","text":"\u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728emr-user\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528emr-user\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - emr-user cd /home/emr-user/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz cd jindosdk-patches ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 emr-user emr-user 2439 May 01 00:00 apply_all.sh -rwxrwxr-x 1 emr-user emr-user 7315 May 01 00:00 apply.sh -rw-rw-r-- 1 emr-user emr-user 40 May 01 00:00 hosts -rwxrwxr-x 1 emr-user emr-user 1112 May 01 00:00 revert_all.sh -rwxrwxr-x 1 emr-user emr-user 2042 May 01 00:00 revert.sh","title":"1. \u51c6\u5907\u56de\u6eda\u811a\u672c"},{"location":"upgrade/emr2_upgrade_jindosdk/#2_3","text":"\u7f16\u8f91 patch \u5305\u4e0b\u7684 hosts \u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684 hostname\uff0c\u5982 master-1-1 \u6216 core-1-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a master-1-1 core-1-1 core-1-2 \u53ef\u5c1d\u8bd5\u4f7f\u7528\u811a\u672c\u83b7\u53d6\u5168\u90e8\u8282\u70b9\u4fe1\u606f\uff0c\u5982\u679c hosts \u83b7\u53d6\u5931\u8d25\uff0c\u9700\u8981\u624b\u52a8\u8865\u5168 cat /usr/local/taihao-executor-all/data/cache/.cluster_context | jq --raw-output '.nodes[].hostname.alias[]' > hosts","title":"2. \u914d\u7f6e\u56de\u6eda\u8282\u70b9\u4fe1\u606f"},{"location":"upgrade/emr2_upgrade_jindosdk/#3_1","text":"\u901a\u8fc7 revert_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./revert_all.sh \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... master-1-1 >>> updating ... core-1-1 >>> updating ... core-1-2 ### DONE","title":"3. \u6267\u884c\u56de\u6eda"},{"location":"upgrade/emr2_upgrade_jindosdk/#4_3","text":"ls -l /opt/apps/JINDOSDK/jindosdk-current/lib \u4ee5\u56de\u6eda\u4e3a 6.2.0 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r--r-- 1 root root 1253740 Apr 24 17:40 jindo-core-6.2.0.jar -rw-r--r-- 1 root root 13110547 Apr 24 17:40 jindo-core-linux-el7-aarch64-6.2.0.jar -rw-r--r-- 1 root root 4432227 Apr 24 17:40 jindo-sdk-6.2.0.jar drwxr-xr-x 2 root root 4096 Apr 24 17:40 native","title":"4. \u786e\u8ba4\u56de\u6eda\u6210\u529f"},{"location":"upgrade/emr2_upgrade_jindosdk/#5_1","text":"\u8bf4\u660e: \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u91cd\u542f \u3002","title":"5. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/","text":"EMR \u65b0\u7248\u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6587\u6863\uff08arm/\u501a\u5929\u73af\u5883\uff09 \u00a7 \u573a\u666f\u4e00\uff1a\u5347\u7ea7\u5df2\u6709\u65b0\u7248\u96c6\u7fa4 \u00a7 \u82e5\u5df2\u6709\u65b0\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u3002\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u6216\u8005\u9700\u8981\u4f7f\u7528 JindoSDK \u7684\u65b0\u529f\u80fd\uff0c\u5177\u4f53\u67e5\u770b \u7248\u672c\u8bf4\u660e , \u53ef\u4ee5\u6839\u636e\u4e0b\u9762\u7684\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002 1. \u51c6\u5907\u8f6f\u4ef6\u5305\u548c\u5347\u7ea7\u811a\u672c\u3001\u914d\u7f6e \u00a7 \u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728emr-user\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528emr-user\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - emr-user cd /home/emr-user/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz \u4e0b\u8f7d JindoSDK \u8f6f\u4ef6\u5305 jindosdk-{VERSION}-{PLATFORM}.tar.gz\uff0c\u653e\u5728\u89e3\u538b\u540e\u7684\u76ee\u5f55\u3002 \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux arm \u5e73\u53f0\u4e3a\u4f8b: cd jindosdk-patches wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux-el7-aarch64.tar.gz ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 emr-user emr-user 575 May 01 00:00 apply_all.sh -rwxrwxr-x 1 emr-user emr-user 4047 May 01 00:00 apply.sh -rw-rw-r-- 1 emr-user emr-user 40 May 01 00:00 hosts -rw-r----- 1 emr-user emr-user xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux-el7-aarch64.tar.gz \u6ce8\u610f \uff1a\u5982\u679c\u4ece4.6.8\u4ee5\u4e0b\u7248\u672c\u5347\u7ea7\u52304.6.9\u4ee5\u4e0a\u62166.x\u7248\u672c\u65f6\uff0c\u7531\u4e8e JindoCommitter \u9ed8\u8ba4\u4f7f\u7528\u7684\u4f5c\u4e1a\u4e34\u65f6\u8def\u5f84\u53d1\u751f\u53d8\u5316\uff0c\u9700\u8981\u5728\u5347\u7ea7\u524d \u5148\u8bbe\u7f6e fs.jdo.committer.allow.concurrent=false \uff08core-site.xml\uff09 \u6216\u5728 Spark \u914d\u7f6e\u4e2d\u8bbe\u7f6e spark.hadoop.fs.jdo.committer.allow.concurrent=false \uff0c\u786e\u4fdd\u5347\u7ea7\u671f\u95f4\u4e0d\u4f1a\u51fa\u73b0\u4e22\u6570\u636e\u7684\u60c5\u51b5\u3002 \u540e\u7eed\u5728\u5305\u542bGATEWAY\u8282\u70b9\u7684\u6240\u6709JindoSDK\u5168\u90e8\u5347\u7ea7\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u62e9\u673a\u53bb\u6389\u8be5\u914d\u7f6e\u3002 2. \u914d\u7f6e\u5347\u7ea7\u8282\u70b9\u4fe1\u606f \u00a7 \u7f16\u8f91 patch \u5305\u4e0b\u7684 hosts \u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684 hostname\uff0c\u5982 master-1-1 \u6216 core-1-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a master-1-1 core-1-1 core-1-2 \u53ef\u5c1d\u8bd5\u4f7f\u7528\u811a\u672c\u83b7\u53d6\u5168\u90e8\u8282\u70b9\u4fe1\u606f\uff0c\u5982\u679c hosts \u83b7\u53d6\u5931\u8d25\uff0c\u9700\u8981\u624b\u52a8\u8865\u5168 cat /usr/local/taihao-executor-all/data/cache/.cluster_context | jq --raw-output '.nodes[].hostname.alias[]' > hosts 3. \u6267\u884c\u5347\u7ea7 \u00a7 \u901a\u8fc7 apply_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./apply_all.sh $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: ./apply_all.sh 6.7.3 \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... master-1-1 >>> updating ... core-1-1 >>> updating ... core-1-2 ### DONE 4. \u786e\u8ba4\u5347\u7ea7\u6210\u529f \u00a7 ls -l /opt/apps/JINDOSDK/jindosdk-current/lib \u4ee5\u4ece\u96c6\u7fa4\u9ed8\u8ba4\u7248\u672c 6.2.0 \u5347\u7ea7\u4e3a 6.7.3 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a lrwxrwxrwx 1 root root 64 Apr 12 11:08 jindo-core-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-core-6.7.3.jar lrwxrwxrwx 1 root root 82 Apr 12 11:08 jindo-core-linux-el7-aarch64-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-core-linux-el7-aarch64-6.7.3.jar lrwxrwxrwx 1 root root 63 Apr 12 11:08 jindo-sdk-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-sdk-6.7.3.jar lrwxrwxrwx 1 root root 50 Apr 12 11:08 native -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/native lrwxrwxrwx 1 root root 57 Apr 12 11:08 site-packages -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/site-packages 5. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1 \u00a7 \u8bf4\u660e: \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u91cd\u542f \u3002 \u573a\u666f\u4e8c\uff1a\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4 \u00a7 \u82e5\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 JindoSDK, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002 1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindosdk.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux aarch64\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux-el7-aarch64.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux-el7-aarch64.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindosdk.sh -gen 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002 2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002 3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u00a7 \u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002 4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d \u00a7 \u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542f Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u573a\u666f\u4e09\uff1a\u65b0\u5efa\u96c6\u7fa4 \u00a7 \u82e5\u65b0\u5efa EMR \u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 JindoSDK, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002 1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindosdk.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux aarch64\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux-el7-aarch64.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux-el7-aarch64.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen-full $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindosdk.sh -gen-full 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002 2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002 3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u00a7 \u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002 4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d \u00a7 \u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542f Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u573a\u666f\u56db\uff1a\u56de\u6eda JindoSDK \u5230\u96c6\u7fa4\u521d\u59cb\u7248\u672c \u00a7 \u82e5\u5df2\u6709\u65b0\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u3002\u5728\u5347\u7ea7\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u9700\u8981\u56de\u6eda\u5230\u96c6\u7fa4\u521d\u59cbJindoSDk\u7248\u672c 1. \u51c6\u5907\u56de\u6eda\u811a\u672c \u00a7 \u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728emr-user\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528emr-user\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - emr-user cd /home/emr-user/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz cd jindosdk-patches ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 emr-user emr-user 2439 May 01 00:00 apply_all.sh -rwxrwxr-x 1 emr-user emr-user 7315 May 01 00:00 apply.sh -rw-rw-r-- 1 emr-user emr-user 40 May 01 00:00 hosts -rwxrwxr-x 1 emr-user emr-user 1112 May 01 00:00 revert_all.sh -rwxrwxr-x 1 emr-user emr-user 2042 May 01 00:00 revert.sh 2. \u914d\u7f6e\u56de\u6eda\u8282\u70b9\u4fe1\u606f \u00a7 \u7f16\u8f91 patch \u5305\u4e0b\u7684 hosts \u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684 hostname\uff0c\u5982 master-1-1 \u6216 core-1-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a master-1-1 core-1-1 core-1-2 \u53ef\u5c1d\u8bd5\u4f7f\u7528\u811a\u672c\u83b7\u53d6\u5168\u90e8\u8282\u70b9\u4fe1\u606f\uff0c\u5982\u679c hosts \u83b7\u53d6\u5931\u8d25\uff0c\u9700\u8981\u624b\u52a8\u8865\u5168 cat /usr/local/taihao-executor-all/data/cache/.cluster_context | jq --raw-output '.nodes[].hostname.alias[]' > hosts 3. \u6267\u884c\u56de\u6eda \u00a7 \u901a\u8fc7 revert_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./revert_all.sh \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... master-1-1 >>> updating ... core-1-1 >>> updating ... core-1-2 ### DONE 4. \u786e\u8ba4\u56de\u6eda\u6210\u529f \u00a7 ls -l /opt/apps/JINDOSDK/jindosdk-current/lib \u4ee5\u56de\u6eda\u4e3a 6.2.0 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r--r-- 1 root root 1253740 Apr 24 17:40 jindo-core-6.2.0.jar -rw-r--r-- 1 root root 13110547 Apr 24 17:40 jindo-core-linux-el7-aarch64-6.2.0.jar -rw-r--r-- 1 root root 4432227 Apr 24 17:40 jindo-sdk-6.2.0.jar drwxr-xr-x 2 root root 4096 Apr 24 17:40 native 5. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1 \u00a7 \u8bf4\u660e: \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u91cd\u542f \u3002","title":"EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0aarm\u7248\u672c\uff08\u65b0\u7248\u63a7\u5236\u53f0\uff09"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#emr-jindosdk-arm","text":"","title":"EMR \u65b0\u7248\u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6587\u6863\uff08arm/\u501a\u5929\u73af\u5883\uff09"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#_1","text":"\u82e5\u5df2\u6709\u65b0\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u3002\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u6216\u8005\u9700\u8981\u4f7f\u7528 JindoSDK \u7684\u65b0\u529f\u80fd\uff0c\u5177\u4f53\u67e5\u770b \u7248\u672c\u8bf4\u660e , \u53ef\u4ee5\u6839\u636e\u4e0b\u9762\u7684\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002","title":"\u573a\u666f\u4e00\uff1a\u5347\u7ea7\u5df2\u6709\u65b0\u7248\u96c6\u7fa4"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#1","text":"\u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728emr-user\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528emr-user\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - emr-user cd /home/emr-user/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz \u4e0b\u8f7d JindoSDK \u8f6f\u4ef6\u5305 jindosdk-{VERSION}-{PLATFORM}.tar.gz\uff0c\u653e\u5728\u89e3\u538b\u540e\u7684\u76ee\u5f55\u3002 \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux arm \u5e73\u53f0\u4e3a\u4f8b: cd jindosdk-patches wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux-el7-aarch64.tar.gz ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 emr-user emr-user 575 May 01 00:00 apply_all.sh -rwxrwxr-x 1 emr-user emr-user 4047 May 01 00:00 apply.sh -rw-rw-r-- 1 emr-user emr-user 40 May 01 00:00 hosts -rw-r----- 1 emr-user emr-user xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux-el7-aarch64.tar.gz \u6ce8\u610f \uff1a\u5982\u679c\u4ece4.6.8\u4ee5\u4e0b\u7248\u672c\u5347\u7ea7\u52304.6.9\u4ee5\u4e0a\u62166.x\u7248\u672c\u65f6\uff0c\u7531\u4e8e JindoCommitter \u9ed8\u8ba4\u4f7f\u7528\u7684\u4f5c\u4e1a\u4e34\u65f6\u8def\u5f84\u53d1\u751f\u53d8\u5316\uff0c\u9700\u8981\u5728\u5347\u7ea7\u524d \u5148\u8bbe\u7f6e fs.jdo.committer.allow.concurrent=false \uff08core-site.xml\uff09 \u6216\u5728 Spark \u914d\u7f6e\u4e2d\u8bbe\u7f6e spark.hadoop.fs.jdo.committer.allow.concurrent=false \uff0c\u786e\u4fdd\u5347\u7ea7\u671f\u95f4\u4e0d\u4f1a\u51fa\u73b0\u4e22\u6570\u636e\u7684\u60c5\u51b5\u3002 \u540e\u7eed\u5728\u5305\u542bGATEWAY\u8282\u70b9\u7684\u6240\u6709JindoSDK\u5168\u90e8\u5347\u7ea7\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u62e9\u673a\u53bb\u6389\u8be5\u914d\u7f6e\u3002","title":"1. \u51c6\u5907\u8f6f\u4ef6\u5305\u548c\u5347\u7ea7\u811a\u672c\u3001\u914d\u7f6e"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#2","text":"\u7f16\u8f91 patch \u5305\u4e0b\u7684 hosts \u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684 hostname\uff0c\u5982 master-1-1 \u6216 core-1-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a master-1-1 core-1-1 core-1-2 \u53ef\u5c1d\u8bd5\u4f7f\u7528\u811a\u672c\u83b7\u53d6\u5168\u90e8\u8282\u70b9\u4fe1\u606f\uff0c\u5982\u679c hosts \u83b7\u53d6\u5931\u8d25\uff0c\u9700\u8981\u624b\u52a8\u8865\u5168 cat /usr/local/taihao-executor-all/data/cache/.cluster_context | jq --raw-output '.nodes[].hostname.alias[]' > hosts","title":"2. \u914d\u7f6e\u5347\u7ea7\u8282\u70b9\u4fe1\u606f"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#3","text":"\u901a\u8fc7 apply_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./apply_all.sh $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: ./apply_all.sh 6.7.3 \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... master-1-1 >>> updating ... core-1-1 >>> updating ... core-1-2 ### DONE","title":"3. \u6267\u884c\u5347\u7ea7"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#4","text":"ls -l /opt/apps/JINDOSDK/jindosdk-current/lib \u4ee5\u4ece\u96c6\u7fa4\u9ed8\u8ba4\u7248\u672c 6.2.0 \u5347\u7ea7\u4e3a 6.7.3 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a lrwxrwxrwx 1 root root 64 Apr 12 11:08 jindo-core-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-core-6.7.3.jar lrwxrwxrwx 1 root root 82 Apr 12 11:08 jindo-core-linux-el7-aarch64-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-core-linux-el7-aarch64-6.7.3.jar lrwxrwxrwx 1 root root 63 Apr 12 11:08 jindo-sdk-6.2.0.jar -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/jindo-sdk-6.7.3.jar lrwxrwxrwx 1 root root 50 Apr 12 11:08 native -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/native lrwxrwxrwx 1 root root 57 Apr 12 11:08 site-packages -> /opt/apps/JINDOSDK/jindosdk-6.7.3-linux/lib/site-packages","title":"4. \u786e\u8ba4\u5347\u7ea7\u6210\u529f"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#5","text":"\u8bf4\u660e: \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u91cd\u542f \u3002","title":"5. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#_2","text":"\u82e5\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 JindoSDK, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002","title":"\u573a\u666f\u4e8c\uff1a\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#1_1","text":"\u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindosdk.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux aarch64\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux-el7-aarch64.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux-el7-aarch64.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindosdk.sh -gen 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002","title":"1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#2_1","text":"\u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002","title":"2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#3-emr","text":"\u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#4_1","text":"\u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542f Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002","title":"4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#_3","text":"\u82e5\u65b0\u5efa EMR \u96c6\u7fa4\u65f6\u9700\u8981\u4f7f\u7528\u65b0\u7248 JindoSDK, \u53ef\u4ee5\u901a\u8fc7\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u5b8c\u6210\u65b0\u5efa\u96c6\u7fa4\u6216\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u53c2\u7167\u5982\u4e0b\u64cd\u4f5c\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002","title":"\u573a\u666f\u4e09\uff1a\u65b0\u5efa\u96c6\u7fa4"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#1_2","text":"\u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-{VERSION}-{PLATFORM}.tar.gz \u548c bootstrap_jindosdk.sh , \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\uff0clinux aarch64\u5e73\u53f0\u4e3a\u4f8b: mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux-el7-aarch64.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux-el7-aarch64.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen-full $NEW_JINDOSDK_VERSION \u4ee5\u5c06\u65b0\u7248\u96c6\u7fa4\u4e2d\u7684 JindoSDK \u5347\u7ea7\u5230 6.7.3 \u7248\u672c\u4e3a\u4f8b: bash bootstrap_jindosdk.sh -gen-full 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/emr-user/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230 patch \u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002","title":"1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#2_2","text":"\u5c06 patch\u5305 \u548c bootstrap \u811a\u672c\u4e0a\u4f20\u5230 OSS \u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 OSS \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002","title":"2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#3-emr_1","text":"\u5728 EMR \u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#4_2","text":"\u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542f Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u3002","title":"4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#jindosdk","text":"\u82e5\u5df2\u6709\u65b0\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\u3002\u5728\u5347\u7ea7\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u9700\u8981\u56de\u6eda\u5230\u96c6\u7fa4\u521d\u59cbJindoSDk\u7248\u672c","title":"\u573a\u666f\u56db\uff1a\u56de\u6eda JindoSDK \u5230\u96c6\u7fa4\u521d\u59cb\u7248\u672c"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#1_3","text":"\u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728emr-user\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528emr-user\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - emr-user cd /home/emr-user/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/emr-taihao/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz cd jindosdk-patches ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 emr-user emr-user 2439 May 01 00:00 apply_all.sh -rwxrwxr-x 1 emr-user emr-user 7315 May 01 00:00 apply.sh -rw-rw-r-- 1 emr-user emr-user 40 May 01 00:00 hosts -rwxrwxr-x 1 emr-user emr-user 1112 May 01 00:00 revert_all.sh -rwxrwxr-x 1 emr-user emr-user 2042 May 01 00:00 revert.sh","title":"1. \u51c6\u5907\u56de\u6eda\u811a\u672c"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#2_3","text":"\u7f16\u8f91 patch \u5305\u4e0b\u7684 hosts \u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684 hostname\uff0c\u5982 master-1-1 \u6216 core-1-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a master-1-1 core-1-1 core-1-2 \u53ef\u5c1d\u8bd5\u4f7f\u7528\u811a\u672c\u83b7\u53d6\u5168\u90e8\u8282\u70b9\u4fe1\u606f\uff0c\u5982\u679c hosts \u83b7\u53d6\u5931\u8d25\uff0c\u9700\u8981\u624b\u52a8\u8865\u5168 cat /usr/local/taihao-executor-all/data/cache/.cluster_context | jq --raw-output '.nodes[].hostname.alias[]' > hosts","title":"2. \u914d\u7f6e\u56de\u6eda\u8282\u70b9\u4fe1\u606f"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#3_1","text":"\u901a\u8fc7 revert_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./revert_all.sh \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... master-1-1 >>> updating ... core-1-1 >>> updating ... core-1-2 ### DONE","title":"3. \u6267\u884c\u56de\u6eda"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#4_3","text":"ls -l /opt/apps/JINDOSDK/jindosdk-current/lib \u4ee5\u56de\u6eda\u4e3a 6.2.0 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r--r-- 1 root root 1253740 Apr 24 17:40 jindo-core-6.2.0.jar -rw-r--r-- 1 root root 13110547 Apr 24 17:40 jindo-core-linux-el7-aarch64-6.2.0.jar -rw-r--r-- 1 root root 4432227 Apr 24 17:40 jindo-sdk-6.2.0.jar drwxr-xr-x 2 root root 4096 Apr 24 17:40 native","title":"4. \u786e\u8ba4\u56de\u6eda\u6210\u529f"},{"location":"upgrade/emr2_upgrade_jindosdk_arm/#5_1","text":"\u8bf4\u660e: \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Flink\u3001Ranger\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u66f4\u591a\u64cd\u4f5c > \u91cd\u542f \u3002","title":"5. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1"},{"location":"upgrade/emr_upgrade_jindosdk/","text":"EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b\uff08\u65e7\u7248\u63a7\u5236\u53f0\uff09 \u00a7 \u80cc\u666f \u00a7 \u5728\u65e7\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\uff0c\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u6216\u8005\u9700\u8981\u4f7f\u7528 JindoSDK \u7684\u65b0\u529f\u80fd\uff0c\u5177\u4f53\u67e5\u770b \u7248\u672c\u8bf4\u660e , \u53ef\u4ee5\u6839\u636e\u4e0b\u9762\u7684\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002 \u573a\u666f\u4e00\uff1a\u5347\u7ea7\u5df2\u6709\u96c6\u7fa4 \u00a7 1. \u51c6\u5907\u8f6f\u4ef6\u5305\u548c\u5347\u7ea7\u811a\u672c \u00a7 \u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728hadoop\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528hadoop\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - hadoop cd /home/hadoop/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz \u4e0b\u8f7d JindoSDK \u8f6f\u4ef6\u5305 jindosdk-{VERSION}-{PLATFORM}.tar.gz\uff08\u4ee5\u4e0b\u4ee5linux x86\u5e73\u53f0\u4e3a\u4f8b\uff09\uff0c\u653e\u5728\u89e3\u538b\u540e\u7684\u76ee\u5f55\u3002 cd jindosdk-patches wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 hadoop hadoop 1263 May 01 00:00 apply_all.sh -rwxrwxr-x 1 hadoop hadoop 6840 May 01 00:00 apply.sh -rw-rw-r-- 1 hadoop hadoop 40 May 01 00:00 hosts -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz \u6ce8\u610f \uff1a\u5982\u679c\u4ece4.6.8\u4ee5\u4e0b\u7248\u672c\u5347\u7ea7\u52304.6.9\u4ee5\u4e0a\u62166.x\u7248\u672c\u65f6\uff0c\u7531\u4e8e JindoCommitter \u9ed8\u8ba4\u4f7f\u7528\u7684\u4f5c\u4e1a\u4e34\u65f6\u8def\u5f84\u53d1\u751f\u53d8\u5316\uff0c\u9700\u8981\u5728\u5347\u7ea7\u524d \u5148\u8bbe\u7f6e fs.jdo.committer.allow.concurrent=false \uff08core-site.xml\uff09 \u6216\u5728 Spark \u914d\u7f6e\u4e2d\u8bbe\u7f6e spark.hadoop.fs.jdo.committer.allow.concurrent=false \uff0c\u786e\u4fdd\u5347\u7ea7\u671f\u95f4\u4e0d\u4f1a\u51fa\u73b0\u4e22\u6570\u636e\u7684\u60c5\u51b5\u3002 \u540e\u7eed\u5728\u5305\u542bGATEWAY\u8282\u70b9\u7684\u6240\u6709JindoSDK\u5168\u90e8\u5347\u7ea7\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u62e9\u673a\u53bb\u6389\u8be5\u914d\u7f6e\u3002 2. \u914d\u7f6e\u5347\u7ea7\u8282\u70b9\u4fe1\u606f \u00a7 \u7f16\u8f91patch\u5305\u4e0b\u7684hosts\u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684host name\uff0c\u5982emr-header-1\u6216emr-worker-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a emr-header-1 emr-worker-1 emr-worker-2 3. \u6267\u884c\u5347\u7ea7 \u00a7 \u901a\u8fc7apply_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./apply_all.sh $JINDOSDK_VERSION \u5982 ./apply_all.sh 6.7.3 \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... emr-header-1 >>> updating ... emr-worker-1 >>> updating ... emr-worker-2 ### DONE 4. \u786e\u8ba4\u5347\u7ea7\u6210\u529f \u00a7 ls -l /opt/apps/extra-jars/ \u4ee5\u5347\u7ea7\u4e3a 6.7.3 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a drwxr-xr-x 2 root root 4096 Apr 24 15:49 flink -rw-r--r-- 1 hadoop hadoop 189081 Apr 24 15:20 hadoop-lzo-0.4.21-SNAPSHOT.jar -rw-r--r-- 1 root root 16264149 Apr 24 15:49 jindo-core-6.7.3.jar -rw-r--r-- 1 root root 14429862 Apr 24 15:49 jindo-core-linux-el7-aarch64-6.7.3.jar -rw-r--r-- 1 root root 4459297 Apr 24 15:49 jindo-sdk-6.7.3.jar drwxr-xr-x 2 root root 4096 Apr 24 15:49 spark drwxr-xr-x 2 root root 4096 Apr 24 15:49 spark3 5. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1 \u00a7 \u8bf4\u660e \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u64cd\u4f5c > \u91cd\u542f All Components \u3002 \u573a\u666f\u4e8c\uff1a\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4 \u00a7 \u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u53ef\u4ee5\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u64cd\u4f5c\u6b65\u9aa4\u5982\u4e0b\uff1a 1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-6.7.3-linux.tar.gz \u548c bootstrap_jindosdk.sh\u3002 mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen $JINDOSDK_VERSION \u5982 bash bootstrap_jindosdk.sh -gen 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/hadoop/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230patch\u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002 2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u5c06 patch\u5305 \u548c bootstrap\u811a\u672c\u4e0a\u4f20\u5230OSS\u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 oss \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/jindo-patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002 3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u00a7 \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002 4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d \u00a7 \u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542fHive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u573a\u666f\u4e09\uff1a\u65b0\u5efa\u96c6\u7fa4 \u00a7 \u65b0\u5efaEMR\u96c6\u7fa4\u65f6\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\u3002\u5177\u4f53\u64cd\u4f5c\u6b65\u9aa4\u5982\u4e0b\uff1a 1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-6.7.3-linux.tar.gz \u548c bootstrap_jindosdk.sh\u3002 mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen-full $JINDOSDK_VERSION \u5982 bash bootstrap_jindosdk.sh -gen-full 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/hadoop/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230patch\u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002 2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u5c06 patch\u5305 \u548c bootstrap\u811a\u672c\u4e0a\u4f20\u5230OSS\u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 oss \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/jindo-patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002 \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u00a7 \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002 \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d \u00a7 \u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542fHive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u573a\u666f\u56db\uff1a\u56de\u6eda JindoSDK \u5230\u96c6\u7fa4\u521d\u59cb\u7248\u672c \u00a7 1. \u51c6\u5907\u56de\u6eda\u811a\u672c \u00a7 \u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728 hadoop \u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528 hadoop \u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - hadoop cd /home/hadoop/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz cd jindosdk-patches ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 hadoop hadoop 1263 May 01 00:00 apply_all.sh -rwxrwxr-x 1 hadoop hadoop 6840 May 01 00:00 apply.sh -rw-rw-r-- 1 hadoop hadoop 40 May 01 00:00 hosts -rwxrwxr-x 1 hadoop hadoop 1308 May 01 00:00 revert_all.sh -rwxrwxr-x 1 hadoop hadoop 6326 May 01 00:00 revert.sh 2. \u914d\u7f6e\u56de\u6eda\u8282\u70b9\u4fe1\u606f \u00a7 \u7f16\u8f91patch\u5305\u4e0b\u7684hosts\u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684host name\uff0c\u5982emr-header-1\u6216emr-worker-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a emr-header-1 emr-worker-1 emr-worker-2 3. \u6267\u884c\u56de\u6eda \u00a7 \u901a\u8fc7revert_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./revert_all.sh \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... emr-header-1 >>> updating ... emr-worker-1 >>> updating ... emr-worker-2 ### DONE 4. \u786e\u8ba4\u56de\u6eda\u6210\u529f \u00a7 ls -l /opt/apps/extra-jars/ \u4ee5\u56de\u6eda\u4e3a 4.3.0 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a drwxr-xr-x 2 root root 4096 Apr 24 17:37 flink -rw-r--r-- 1 hadoop hadoop 189081 Apr 24 17:21 hadoop-lzo-0.4.21-SNAPSHOT.jar drwxr-xr-x 2 root root 4096 Apr 24 17:37 hive -rw-r--r-- 1 root root 85025668 Apr 24 17:36 jindo-core-4.3.0.jar -rw-r--r-- 1 root root 3165000 Apr 24 17:36 jindo-sdk-4.3.0.jar drwxr-xr-x 2 root root 4096 Apr 24 17:37 native drwxr-xr-x 2 root root 4096 Apr 24 17:37 spark drwxr-xr-x 2 root root 4096 Apr 24 17:37 spark3 5. \u56de\u6eda\u540e\u91cd\u542f\u670d\u52a1 \u00a7 \u8bf4\u660e \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u56de\u6eda\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u64cd\u4f5c > \u91cd\u542f All Components \u3002","title":"EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\uff08\u65e7\u7248\u63a7\u5236\u53f0\uff09"},{"location":"upgrade/emr_upgrade_jindosdk/#emr-jindosdk","text":"","title":"EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b\uff08\u65e7\u7248\u63a7\u5236\u53f0\uff09"},{"location":"upgrade/emr_upgrade_jindosdk/#_1","text":"\u5728\u65e7\u7248\u7ba1\u63a7\u5e73\u53f0\u521b\u5efa\u7684 E-MapReduce EMR-5.6.0/EMR-3.40.0 \u53ca\u4ee5\u4e0a\u7248\u672c\u96c6\u7fa4\uff0c\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u6216\u8005\u9700\u8981\u4f7f\u7528 JindoSDK \u7684\u65b0\u529f\u80fd\uff0c\u5177\u4f53\u67e5\u770b \u7248\u672c\u8bf4\u660e , \u53ef\u4ee5\u6839\u636e\u4e0b\u9762\u7684\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002","title":"\u80cc\u666f"},{"location":"upgrade/emr_upgrade_jindosdk/#_2","text":"","title":"\u573a\u666f\u4e00\uff1a\u5347\u7ea7\u5df2\u6709\u96c6\u7fa4"},{"location":"upgrade/emr_upgrade_jindosdk/#1","text":"\u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728hadoop\u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528hadoop\u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - hadoop cd /home/hadoop/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz \u4e0b\u8f7d JindoSDK \u8f6f\u4ef6\u5305 jindosdk-{VERSION}-{PLATFORM}.tar.gz\uff08\u4ee5\u4e0b\u4ee5linux x86\u5e73\u53f0\u4e3a\u4f8b\uff09\uff0c\u653e\u5728\u89e3\u538b\u540e\u7684\u76ee\u5f55\u3002 cd jindosdk-patches wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 hadoop hadoop 1263 May 01 00:00 apply_all.sh -rwxrwxr-x 1 hadoop hadoop 6840 May 01 00:00 apply.sh -rw-rw-r-- 1 hadoop hadoop 40 May 01 00:00 hosts -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz \u6ce8\u610f \uff1a\u5982\u679c\u4ece4.6.8\u4ee5\u4e0b\u7248\u672c\u5347\u7ea7\u52304.6.9\u4ee5\u4e0a\u62166.x\u7248\u672c\u65f6\uff0c\u7531\u4e8e JindoCommitter \u9ed8\u8ba4\u4f7f\u7528\u7684\u4f5c\u4e1a\u4e34\u65f6\u8def\u5f84\u53d1\u751f\u53d8\u5316\uff0c\u9700\u8981\u5728\u5347\u7ea7\u524d \u5148\u8bbe\u7f6e fs.jdo.committer.allow.concurrent=false \uff08core-site.xml\uff09 \u6216\u5728 Spark \u914d\u7f6e\u4e2d\u8bbe\u7f6e spark.hadoop.fs.jdo.committer.allow.concurrent=false \uff0c\u786e\u4fdd\u5347\u7ea7\u671f\u95f4\u4e0d\u4f1a\u51fa\u73b0\u4e22\u6570\u636e\u7684\u60c5\u51b5\u3002 \u540e\u7eed\u5728\u5305\u542bGATEWAY\u8282\u70b9\u7684\u6240\u6709JindoSDK\u5168\u90e8\u5347\u7ea7\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u62e9\u673a\u53bb\u6389\u8be5\u914d\u7f6e\u3002","title":"1. \u51c6\u5907\u8f6f\u4ef6\u5305\u548c\u5347\u7ea7\u811a\u672c"},{"location":"upgrade/emr_upgrade_jindosdk/#2","text":"\u7f16\u8f91patch\u5305\u4e0b\u7684hosts\u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684host name\uff0c\u5982emr-header-1\u6216emr-worker-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a emr-header-1 emr-worker-1 emr-worker-2","title":"2. \u914d\u7f6e\u5347\u7ea7\u8282\u70b9\u4fe1\u606f"},{"location":"upgrade/emr_upgrade_jindosdk/#3","text":"\u901a\u8fc7apply_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./apply_all.sh $JINDOSDK_VERSION \u5982 ./apply_all.sh 6.7.3 \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... emr-header-1 >>> updating ... emr-worker-1 >>> updating ... emr-worker-2 ### DONE","title":"3. \u6267\u884c\u5347\u7ea7"},{"location":"upgrade/emr_upgrade_jindosdk/#4","text":"ls -l /opt/apps/extra-jars/ \u4ee5\u5347\u7ea7\u4e3a 6.7.3 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a drwxr-xr-x 2 root root 4096 Apr 24 15:49 flink -rw-r--r-- 1 hadoop hadoop 189081 Apr 24 15:20 hadoop-lzo-0.4.21-SNAPSHOT.jar -rw-r--r-- 1 root root 16264149 Apr 24 15:49 jindo-core-6.7.3.jar -rw-r--r-- 1 root root 14429862 Apr 24 15:49 jindo-core-linux-el7-aarch64-6.7.3.jar -rw-r--r-- 1 root root 4459297 Apr 24 15:49 jindo-sdk-6.7.3.jar drwxr-xr-x 2 root root 4096 Apr 24 15:49 spark drwxr-xr-x 2 root root 4096 Apr 24 15:49 spark3","title":"4. \u786e\u8ba4\u5347\u7ea7\u6210\u529f"},{"location":"upgrade/emr_upgrade_jindosdk/#5","text":"\u8bf4\u660e \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u64cd\u4f5c > \u91cd\u542f All Components \u3002","title":"5. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1"},{"location":"upgrade/emr_upgrade_jindosdk/#_3","text":"\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u53ef\u4ee5\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u64cd\u4f5c\u6b65\u9aa4\u5982\u4e0b\uff1a","title":"\u573a\u666f\u4e8c\uff1a\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4"},{"location":"upgrade/emr_upgrade_jindosdk/#1_1","text":"\u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-6.7.3-linux.tar.gz \u548c bootstrap_jindosdk.sh\u3002 mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen $JINDOSDK_VERSION \u5982 bash bootstrap_jindosdk.sh -gen 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/hadoop/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230patch\u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002","title":"1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr_upgrade_jindosdk/#2_1","text":"\u5c06 patch\u5305 \u548c bootstrap\u811a\u672c\u4e0a\u4f20\u5230OSS\u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 oss \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/jindo-patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002","title":"2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr_upgrade_jindosdk/#3-emr","text":"\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c"},{"location":"upgrade/emr_upgrade_jindosdk/#4_1","text":"\u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542fHive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002","title":"4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d"},{"location":"upgrade/emr_upgrade_jindosdk/#_4","text":"\u65b0\u5efaEMR\u96c6\u7fa4\u65f6\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\u3002\u5177\u4f53\u64cd\u4f5c\u6b65\u9aa4\u5982\u4e0b\uff1a","title":"\u573a\u666f\u4e09\uff1a\u65b0\u5efa\u96c6\u7fa4"},{"location":"upgrade/emr_upgrade_jindosdk/#1_2","text":"\u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-6.7.3-linux.tar.gz \u548c bootstrap_jindosdk.sh\u3002 mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen-full $JINDOSDK_VERSION \u5982 bash bootstrap_jindosdk.sh -gen-full 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/hadoop/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230patch\u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002","title":"1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr_upgrade_jindosdk/#2_2","text":"\u5c06 patch\u5305 \u548c bootstrap\u811a\u672c\u4e0a\u4f20\u5230OSS\u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 oss \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/jindo-patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002","title":"2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr_upgrade_jindosdk/#emr","text":"\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c"},{"location":"upgrade/emr_upgrade_jindosdk/#_5","text":"\u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542fHive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002","title":"\u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d"},{"location":"upgrade/emr_upgrade_jindosdk/#jindosdk","text":"","title":"\u573a\u666f\u56db\uff1a\u56de\u6eda JindoSDK \u5230\u96c6\u7fa4\u521d\u59cb\u7248\u672c"},{"location":"upgrade/emr_upgrade_jindosdk/#1_3","text":"\u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728 hadoop \u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528 hadoop \u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - hadoop cd /home/hadoop/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz cd jindosdk-patches ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 hadoop hadoop 1263 May 01 00:00 apply_all.sh -rwxrwxr-x 1 hadoop hadoop 6840 May 01 00:00 apply.sh -rw-rw-r-- 1 hadoop hadoop 40 May 01 00:00 hosts -rwxrwxr-x 1 hadoop hadoop 1308 May 01 00:00 revert_all.sh -rwxrwxr-x 1 hadoop hadoop 6326 May 01 00:00 revert.sh","title":"1. \u51c6\u5907\u56de\u6eda\u811a\u672c"},{"location":"upgrade/emr_upgrade_jindosdk/#2_3","text":"\u7f16\u8f91patch\u5305\u4e0b\u7684hosts\u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684host name\uff0c\u5982emr-header-1\u6216emr-worker-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a emr-header-1 emr-worker-1 emr-worker-2","title":"2. \u914d\u7f6e\u56de\u6eda\u8282\u70b9\u4fe1\u606f"},{"location":"upgrade/emr_upgrade_jindosdk/#3_1","text":"\u901a\u8fc7revert_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./revert_all.sh \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... emr-header-1 >>> updating ... emr-worker-1 >>> updating ... emr-worker-2 ### DONE","title":"3. \u6267\u884c\u56de\u6eda"},{"location":"upgrade/emr_upgrade_jindosdk/#4_2","text":"ls -l /opt/apps/extra-jars/ \u4ee5\u56de\u6eda\u4e3a 4.3.0 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a drwxr-xr-x 2 root root 4096 Apr 24 17:37 flink -rw-r--r-- 1 hadoop hadoop 189081 Apr 24 17:21 hadoop-lzo-0.4.21-SNAPSHOT.jar drwxr-xr-x 2 root root 4096 Apr 24 17:37 hive -rw-r--r-- 1 root root 85025668 Apr 24 17:36 jindo-core-4.3.0.jar -rw-r--r-- 1 root root 3165000 Apr 24 17:36 jindo-sdk-4.3.0.jar drwxr-xr-x 2 root root 4096 Apr 24 17:37 native drwxr-xr-x 2 root root 4096 Apr 24 17:37 spark drwxr-xr-x 2 root root 4096 Apr 24 17:37 spark3","title":"4. \u786e\u8ba4\u56de\u6eda\u6210\u529f"},{"location":"upgrade/emr_upgrade_jindosdk/#5_1","text":"\u8bf4\u660e \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u56de\u6eda\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u64cd\u4f5c > \u91cd\u542f All Components \u3002","title":"5. \u56de\u6eda\u540e\u91cd\u542f\u670d\u52a1"},{"location":"upgrade/emr_upgrade_smartdata/","text":"EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b\uff08\u65e7\u7248\u63a7\u5236\u53f0\uff09 \u00a7 \u80cc\u666f \u00a7 \u5728\u65e7\u7248\u7ba1\u63a7\u5e73\u53f0\u4e2d\u521b\u5efaE-MapReduce EMR-5.5.0/EMR-3.39.0 \u4ee5\u524d\u7248\u672c\u7684\u96c6\u7fa4\uff0c\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u6216\u8005\u9700\u8981\u4f7f\u7528 JindoSDK \u7684\u65b0\u529f\u80fd\uff0c\u5177\u4f53\u67e5\u770b \u7248\u672c\u8bf4\u660e , \u53ef\u4ee5\u6839\u636e\u4e0b\u9762\u7684\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002 \u573a\u666f\u4e00\uff1a\u5347\u7ea7\u5df2\u6709\u96c6\u7fa4 \u00a7 1. \u51c6\u5907\u8f6f\u4ef6\u5305\u548c\u5347\u7ea7\u811a\u672c \u00a7 \u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728 hadoop \u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528 hadoop \u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - hadoop cd /home/hadoop/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz \u4e0b\u8f7d JindoSDK \u8f6f\u4ef6\u5305 jindosdk-{VERSION}-{PLATFORM}.tar.gz\uff08\u4ee5\u4e0b\u4ee5linux x86\u5e73\u53f0\u4e3a\u4f8b\uff09 \uff0c\u653e\u5728\u89e3\u538b\u540e\u7684\u76ee\u5f55\u3002 cd jindosdk-patches wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 hadoop hadoop 1263 May 01 00:00 apply_all.sh -rwxrwxr-x 1 hadoop hadoop 6840 May 01 00:00 apply.sh -rw-rw-r-- 1 hadoop hadoop 40 May 01 00:00 hosts -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rwxrwxr-x 1 hadoop hadoop 1308 May 01 00:00 revert_all.sh -rwxrwxr-x 1 hadoop hadoop 6326 May 01 00:00 revert.sh \u6ce8\u610f \uff1a\u5982\u679c\u4ece4.6.8\u4ee5\u4e0b\u7248\u672c\u5347\u7ea7\u52304.6.9\u4ee5\u4e0a\u62166.x\u7248\u672c\u65f6\uff0c\u7531\u4e8e JindoCommitter \u9ed8\u8ba4\u4f7f\u7528\u7684\u4f5c\u4e1a\u4e34\u65f6\u8def\u5f84\u53d1\u751f\u53d8\u5316\uff0c\u9700\u8981\u5728\u5347\u7ea7\u524d \u5148\u8bbe\u7f6e fs.jdo.committer.allow.concurrent=false \uff08core-site.xml\uff09 \u6216\u5728 Spark \u914d\u7f6e\u4e2d\u8bbe\u7f6e spark.hadoop.fs.jdo.committer.allow.concurrent=false \uff0c\u786e\u4fdd\u5347\u7ea7\u671f\u95f4\u4e0d\u4f1a\u51fa\u73b0\u4e22\u6570\u636e\u7684\u60c5\u51b5\u3002 \u540e\u7eed\u5728\u5305\u542bGATEWAY\u8282\u70b9\u7684\u6240\u6709JindoSDK\u5168\u90e8\u5347\u7ea7\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u62e9\u673a\u53bb\u6389\u8be5\u914d\u7f6e\u3002 2. \u914d\u7f6e\u5347\u7ea7\u8282\u70b9\u4fe1\u606f \u00a7 \u7f16\u8f91patch\u5305\u4e0b\u7684hosts\u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684host name\uff0c\u5982emr-header-1\u6216emr-worker-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a emr-header-1 emr-worker-1 emr-worker-2 3. \u6267\u884c\u5347\u7ea7 \u00a7 \u901a\u8fc7apply_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./apply_all.sh $JINDOSDK_VERSION \u5982 ./apply_all.sh 6.7.3 \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... emr-header-1 >>> updating ... emr-worker-1 >>> updating ... emr-worker-2 ### DONE 4. \u4fee\u6539\u96c6\u7fa4\u914d\u7f6e \u00a7 4.1 \u517c\u5bb9 JindoSDK \u8001\u7248\u672c\u914d\u7f6e\uff08\u5347\u7ea7\u5230 JindoSDK-6.1.1\u7248\u672c\u53ca\u4ee5\u4e0a\u652f\u6301\uff09 \u00a7 a. \u5728SmartData\u670d\u52a1\u7684 \u914d\u7f6e \u9875\u9762\uff0c\u5355\u51fb smartdata-site \u9875\u7b7e\u3002 b. \u5728 smartdata-site \u9875\u9762\uff0c\u5355\u51fb \u81ea\u5b9a\u4e49\u914d\u7f6e \u3002 c. \u5728 \u65b0\u589e\u914d\u7f6e \u9879\u5bf9\u8bdd\u6846\u4e2d\uff0c\u6dfb\u52a0\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 fs.oss.legacy.version \u914d\u7f6e\u9700\u8981\u517c\u5bb9smartdata\u7248\u672c\u7684\u524d\u4e24\u4f4d\u3002\u4f8b\u5982\uff1a3.8 **\u6ce8\uff1a \u4ec5\u6d4b\u8bd5\u4e863.8.x\u7248\u672c\uff0c\u5176\u4ed6\u7248\u672c\u8bf7\u8c28\u614e\u4f7f\u7528\u3002 4.2 \u4fee\u6539 hive/spark \u914d\u7f6e \u00a7 a. \u5bf9\u5e94 \u7ec4\u4ef6(\u5982HIVE) \u670d\u52a1\u7684\u9875\u9762\uff0c\u5355\u51fb \u914d\u7f6e\uff08\u5982hive-site.xml\uff09 \u9875\u7b7e\u3002 b. \u5728 \u914d\u7f6e\uff08\u5982hive-site.xml\uff09 \u9875\u9762\uff0c\u5728 \u914d\u7f6e\u641c\u7d22 \u4e2d\u641c\u7d22 \u53c2\u6570\uff08\u5982hive.exec.post.hooks\uff09 \u3002 c. \u5982\u53c2\u6570value\u5b58\u5728\u9700\u8981\u5220\u9664\u7684\u53c2\u6570\uff0c\u4fee\u6539\u540e\u70b9\u51fb\u4fdd\u5b58\uff0c\u5e76\u66f4\u65b0\u90e8\u7f72\u914d\u7f6e\u3002\u4e0d\u5b58\u5728\u5219\u8df3\u8fc7\u3002 \u7ec4\u4ef6 \u914d\u7f6e \u53c2\u6570 \u63cf\u8ff0 HIVE hive-site.xml hive.exec.post.hooks \u5220\u9664com.aliyun.emr.table.hive.HivePostHook(\u6216com.aliyun.jindotable.hive.HivePostHook)\uff0c\u4fdd\u7559\u5176\u4f59hook\uff0c\u5982com.aliyun.emr.meta.hive.hook.LineageLoggerHook SPARK spark-defaults.conf spark.sql.queryExecutionListeners \u5220\u9664com.aliyun.emr.table.spark.SparkSQLQueryListener(\u6216com.aliyun.jindotable.spark.SparkSQLQueryListener)\uff0c\u4fdd\u7559\u5176\u4f59hook\uff0c\u5982com.aliyun.emr.meta.spark.listener.EMRQueryLogger 5. \u786e\u8ba4\u5347\u7ea7\u6210\u529f \u00a7 ls -l /opt/apps/extra-jars/ \u4ee5\u5347\u7ea7\u4e3a 6.7.3 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a drwxr-xr-x 2 root root 4096 Apr 24 16:19 flink -rw-r--r-- 1 hadoop hadoop 189081 Apr 24 15:33 hadoop-lzo-0.4.21-SNAPSHOT.jar drwxr-xr-x 2 hadoop hadoop 4096 Apr 24 15:33 hive -rw-r--r-- 1 root root 16264149 Apr 24 16:19 jindo-core-6.7.3.jar -rw-r--r-- 1 root root 14429862 Apr 24 16:19 jindo-core-linux-el7-aarch64-6.7.3.jar -rw-r--r-- 1 root root 4459297 Apr 24 16:19 jindo-sdk-6.7.3.jar -rw-r--r-- 1 hadoop hadoop 1438552 Apr 24 15:33 jindotable-common.jar drwxr-xr-x 2 root root 4096 Apr 24 16:19 spark drwxr-xr-x 2 root root 4096 Apr 24 16:19 spark3 6. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1 \u00a7 \u8bf4\u660e \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u64cd\u4f5c > \u91cd\u542f All Components \u3002 7. \u65e5\u5fd7\u7b49\u7ea7\u8c03\u6574 \u00a7 Hadoop\u547d\u4ee4\uff0cSpark\u4f5c\u4e1a\u65e5\u5fd7\u7b49log4j\u65e5\u5fd7\u53ef\u901a\u8fc7\u4ee5\u4e0b\u914d\u7f6e\u8c03\u6574\u3002\u4ee5\u8c03\u6574 hadoop \u547d\u4ee4\u65e5\u5fd7\u914d\u7f6e\u4e3a\u4f8b\uff1a vim /etc/ecm/hadoop-conf/log4j.properties \u6dfb\u52a0\u4ee5\u4e0b\u914d\u7f6e\uff1a log4j.logger.com.aliyun.jindodata=WARN \u5982\u9700\u770b\u5230\u64cd\u4f5c\u65e5\u5fd7\uff0c\u5219\u518d\u6dfb\u52a0\u4ee5\u4e0b\u914d\u7f6e\uff1a log4j.logger.com.aliyun.jindodata=WARN log4j.logger.com.aliyun.jindodata.common.FsStats=INFO \u573a\u666f\u4e8c\uff1a\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4 \u00a7 \u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u53ef\u4ee5\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u64cd\u4f5c\u6b65\u9aa4\u5982\u4e0b\uff1a 1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-6.7.3-linux.tar.gz \u548c bootstrap_jindosdk.sh\u3002 mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen $JINDOSDK_VERSION \u5982 bash bootstrap_jindosdk.sh -gen 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/hadoop/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230patch\u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002 2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u5c06 patch\u5305 \u548c bootstrap\u811a\u672c\u4e0a\u4f20\u5230OSS\u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 oss \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/jindo-patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u200b\u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002 3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u00a7 \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002 4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d \u00a7 \u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542fHive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u573a\u666f\u4e09\uff1a\u65b0\u5efa\u96c6\u7fa4 \u00a7 \u65b0\u5efaEMR\u96c6\u7fa4\u65f6\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\u3002\u5177\u4f53\u64cd\u4f5c\u6b65\u9aa4\u5982\u4e0b\uff1a 1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-6.7.3-linux.tar.gz \u548c bootstrap_jindosdk.sh\u3002 mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen-full $JINDOSDK_VERSION \u5982 bash bootstrap_jindosdk.sh -gen-full 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/hadoop/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230patch\u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002 2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305 \u00a7 \u5c06 patch\u5305 \u548c bootstrap\u811a\u672c\u4e0a\u4f20\u5230OSS\u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 oss \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/jindo-patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u200b\u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002 \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u00a7 \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002 \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d \u00a7 \u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542fHive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u573a\u666f\u56db\uff1a\u56de\u6eda JindoSDK \u5230\u96c6\u7fa4\u521d\u59cb\u7248\u672c \u00a7 1. \u51c6\u5907\u56de\u6eda\u811a\u672c \u00a7 \u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728 hadoop \u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528 hadoop \u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - hadoop cd /home/hadoop/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz cd jindosdk-patches ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 hadoop hadoop 1263 May 01 00:00 apply_all.sh -rwxrwxr-x 1 hadoop hadoop 6840 May 01 00:00 apply.sh -rw-rw-r-- 1 hadoop hadoop 40 May 01 00:00 hosts -rwxrwxr-x 1 hadoop hadoop 1308 May 01 00:00 revert_all.sh -rwxrwxr-x 1 hadoop hadoop 6326 May 01 00:00 revert.sh 2. \u914d\u7f6e\u56de\u6eda\u8282\u70b9\u4fe1\u606f \u00a7 \u7f16\u8f91patch\u5305\u4e0b\u7684hosts\u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684host name\uff0c\u5982emr-header-1\u6216emr-worker-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a emr-header-1 emr-worker-1 emr-worker-2 3. \u6267\u884c\u56de\u6eda \u00a7 \u901a\u8fc7revert_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./revert_all.sh \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... emr-header-1 >>> updating ... emr-worker-1 >>> updating ... emr-worker-2 ### DONE 4. \u786e\u8ba4\u56de\u6eda\u6210\u529f \u00a7 ls -l /opt/apps/extra-jars/ \u4ee5\u56de\u6eda\u4e3a 3.7.2 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a drwxr-xr-x 2 root root 4096 Apr 24 16:22 flink -rw-r--r-- 1 hadoop hadoop 189081 Apr 24 15:33 hadoop-lzo-0.4.21-SNAPSHOT.jar drwxr-xr-x 2 root root 4096 Apr 24 16:22 hive -rw-r--r-- 1 root root 1438552 Apr 24 16:22 jindotable-common.jar drwxr-xr-x 2 root root 4096 Apr 24 16:22 native -rw-r--r-- 1 root root 29389473 Apr 24 16:22 smartdata-jindofs-3.7.2.jar drwxr-xr-x 2 root root 4096 Apr 24 16:22 spark drwxr-xr-x 2 root root 4096 Apr 24 16:22 spark3 5. \u56de\u6eda\u540e\u91cd\u542f\u670d\u52a1 \u00a7 \u8bf4\u660e \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u56de\u6eda\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u64cd\u4f5c > \u91cd\u542f All Components \u3002","title":"EMR-5.5.0/EMR-3.39.0 \u4ee5\u524d\u7248\u672c\uff08\u65e7\u7248\u63a7\u5236\u53f0\uff09"},{"location":"upgrade/emr_upgrade_smartdata/#emr-jindosdk","text":"","title":"EMR \u96c6\u7fa4 JindoSDK \u5347\u7ea7\u6d41\u7a0b\uff08\u65e7\u7248\u63a7\u5236\u53f0\uff09"},{"location":"upgrade/emr_upgrade_smartdata/#_1","text":"\u5728\u65e7\u7248\u7ba1\u63a7\u5e73\u53f0\u4e2d\u521b\u5efaE-MapReduce EMR-5.5.0/EMR-3.39.0 \u4ee5\u524d\u7248\u672c\u7684\u96c6\u7fa4\uff0c\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u95ee\u9898\uff0c\u6216\u8005\u9700\u8981\u4f7f\u7528 JindoSDK \u7684\u65b0\u529f\u80fd\uff0c\u5177\u4f53\u67e5\u770b \u7248\u672c\u8bf4\u660e , \u53ef\u4ee5\u6839\u636e\u4e0b\u9762\u7684\u6b65\u9aa4\u5b8c\u6210 JindoSDK \u5347\u7ea7\u3002","title":"\u80cc\u666f"},{"location":"upgrade/emr_upgrade_smartdata/#_2","text":"","title":"\u573a\u666f\u4e00\uff1a\u5347\u7ea7\u5df2\u6709\u96c6\u7fa4"},{"location":"upgrade/emr_upgrade_smartdata/#1","text":"\u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728 hadoop \u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528 hadoop \u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - hadoop cd /home/hadoop/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz \u4e0b\u8f7d JindoSDK \u8f6f\u4ef6\u5305 jindosdk-{VERSION}-{PLATFORM}.tar.gz\uff08\u4ee5\u4e0b\u4ee5linux x86\u5e73\u53f0\u4e3a\u4f8b\uff09 \uff0c\u653e\u5728\u89e3\u538b\u540e\u7684\u76ee\u5f55\u3002 cd jindosdk-patches wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 hadoop hadoop 1263 May 01 00:00 apply_all.sh -rwxrwxr-x 1 hadoop hadoop 6840 May 01 00:00 apply.sh -rw-rw-r-- 1 hadoop hadoop 40 May 01 00:00 hosts -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rwxrwxr-x 1 hadoop hadoop 1308 May 01 00:00 revert_all.sh -rwxrwxr-x 1 hadoop hadoop 6326 May 01 00:00 revert.sh \u6ce8\u610f \uff1a\u5982\u679c\u4ece4.6.8\u4ee5\u4e0b\u7248\u672c\u5347\u7ea7\u52304.6.9\u4ee5\u4e0a\u62166.x\u7248\u672c\u65f6\uff0c\u7531\u4e8e JindoCommitter \u9ed8\u8ba4\u4f7f\u7528\u7684\u4f5c\u4e1a\u4e34\u65f6\u8def\u5f84\u53d1\u751f\u53d8\u5316\uff0c\u9700\u8981\u5728\u5347\u7ea7\u524d \u5148\u8bbe\u7f6e fs.jdo.committer.allow.concurrent=false \uff08core-site.xml\uff09 \u6216\u5728 Spark \u914d\u7f6e\u4e2d\u8bbe\u7f6e spark.hadoop.fs.jdo.committer.allow.concurrent=false \uff0c\u786e\u4fdd\u5347\u7ea7\u671f\u95f4\u4e0d\u4f1a\u51fa\u73b0\u4e22\u6570\u636e\u7684\u60c5\u51b5\u3002 \u540e\u7eed\u5728\u5305\u542bGATEWAY\u8282\u70b9\u7684\u6240\u6709JindoSDK\u5168\u90e8\u5347\u7ea7\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u62e9\u673a\u53bb\u6389\u8be5\u914d\u7f6e\u3002","title":"1. \u51c6\u5907\u8f6f\u4ef6\u5305\u548c\u5347\u7ea7\u811a\u672c"},{"location":"upgrade/emr_upgrade_smartdata/#2","text":"\u7f16\u8f91patch\u5305\u4e0b\u7684hosts\u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684host name\uff0c\u5982emr-header-1\u6216emr-worker-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a emr-header-1 emr-worker-1 emr-worker-2","title":"2. \u914d\u7f6e\u5347\u7ea7\u8282\u70b9\u4fe1\u606f"},{"location":"upgrade/emr_upgrade_smartdata/#3","text":"\u901a\u8fc7apply_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./apply_all.sh $JINDOSDK_VERSION \u5982 ./apply_all.sh 6.7.3 \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... emr-header-1 >>> updating ... emr-worker-1 >>> updating ... emr-worker-2 ### DONE","title":"3. \u6267\u884c\u5347\u7ea7"},{"location":"upgrade/emr_upgrade_smartdata/#4","text":"","title":"4. \u4fee\u6539\u96c6\u7fa4\u914d\u7f6e"},{"location":"upgrade/emr_upgrade_smartdata/#41-jindosdk-jindosdk-611","text":"a. \u5728SmartData\u670d\u52a1\u7684 \u914d\u7f6e \u9875\u9762\uff0c\u5355\u51fb smartdata-site \u9875\u7b7e\u3002 b. \u5728 smartdata-site \u9875\u9762\uff0c\u5355\u51fb \u81ea\u5b9a\u4e49\u914d\u7f6e \u3002 c. \u5728 \u65b0\u589e\u914d\u7f6e \u9879\u5bf9\u8bdd\u6846\u4e2d\uff0c\u6dfb\u52a0\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 fs.oss.legacy.version \u914d\u7f6e\u9700\u8981\u517c\u5bb9smartdata\u7248\u672c\u7684\u524d\u4e24\u4f4d\u3002\u4f8b\u5982\uff1a3.8 **\u6ce8\uff1a \u4ec5\u6d4b\u8bd5\u4e863.8.x\u7248\u672c\uff0c\u5176\u4ed6\u7248\u672c\u8bf7\u8c28\u614e\u4f7f\u7528\u3002","title":"4.1 \u517c\u5bb9 JindoSDK \u8001\u7248\u672c\u914d\u7f6e\uff08\u5347\u7ea7\u5230 JindoSDK-6.1.1\u7248\u672c\u53ca\u4ee5\u4e0a\u652f\u6301\uff09"},{"location":"upgrade/emr_upgrade_smartdata/#42-hivespark","text":"a. \u5bf9\u5e94 \u7ec4\u4ef6(\u5982HIVE) \u670d\u52a1\u7684\u9875\u9762\uff0c\u5355\u51fb \u914d\u7f6e\uff08\u5982hive-site.xml\uff09 \u9875\u7b7e\u3002 b. \u5728 \u914d\u7f6e\uff08\u5982hive-site.xml\uff09 \u9875\u9762\uff0c\u5728 \u914d\u7f6e\u641c\u7d22 \u4e2d\u641c\u7d22 \u53c2\u6570\uff08\u5982hive.exec.post.hooks\uff09 \u3002 c. \u5982\u53c2\u6570value\u5b58\u5728\u9700\u8981\u5220\u9664\u7684\u53c2\u6570\uff0c\u4fee\u6539\u540e\u70b9\u51fb\u4fdd\u5b58\uff0c\u5e76\u66f4\u65b0\u90e8\u7f72\u914d\u7f6e\u3002\u4e0d\u5b58\u5728\u5219\u8df3\u8fc7\u3002 \u7ec4\u4ef6 \u914d\u7f6e \u53c2\u6570 \u63cf\u8ff0 HIVE hive-site.xml hive.exec.post.hooks \u5220\u9664com.aliyun.emr.table.hive.HivePostHook(\u6216com.aliyun.jindotable.hive.HivePostHook)\uff0c\u4fdd\u7559\u5176\u4f59hook\uff0c\u5982com.aliyun.emr.meta.hive.hook.LineageLoggerHook SPARK spark-defaults.conf spark.sql.queryExecutionListeners \u5220\u9664com.aliyun.emr.table.spark.SparkSQLQueryListener(\u6216com.aliyun.jindotable.spark.SparkSQLQueryListener)\uff0c\u4fdd\u7559\u5176\u4f59hook\uff0c\u5982com.aliyun.emr.meta.spark.listener.EMRQueryLogger","title":"4.2 \u4fee\u6539 hive/spark \u914d\u7f6e"},{"location":"upgrade/emr_upgrade_smartdata/#5","text":"ls -l /opt/apps/extra-jars/ \u4ee5\u5347\u7ea7\u4e3a 6.7.3 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a drwxr-xr-x 2 root root 4096 Apr 24 16:19 flink -rw-r--r-- 1 hadoop hadoop 189081 Apr 24 15:33 hadoop-lzo-0.4.21-SNAPSHOT.jar drwxr-xr-x 2 hadoop hadoop 4096 Apr 24 15:33 hive -rw-r--r-- 1 root root 16264149 Apr 24 16:19 jindo-core-6.7.3.jar -rw-r--r-- 1 root root 14429862 Apr 24 16:19 jindo-core-linux-el7-aarch64-6.7.3.jar -rw-r--r-- 1 root root 4459297 Apr 24 16:19 jindo-sdk-6.7.3.jar -rw-r--r-- 1 hadoop hadoop 1438552 Apr 24 15:33 jindotable-common.jar drwxr-xr-x 2 root root 4096 Apr 24 16:19 spark drwxr-xr-x 2 root root 4096 Apr 24 16:19 spark3","title":"5. \u786e\u8ba4\u5347\u7ea7\u6210\u529f"},{"location":"upgrade/emr_upgrade_smartdata/#6","text":"\u8bf4\u660e \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u5347\u7ea7\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u64cd\u4f5c > \u91cd\u542f All Components \u3002","title":"6. \u5347\u7ea7\u540e\u91cd\u542f\u670d\u52a1"},{"location":"upgrade/emr_upgrade_smartdata/#7","text":"Hadoop\u547d\u4ee4\uff0cSpark\u4f5c\u4e1a\u65e5\u5fd7\u7b49log4j\u65e5\u5fd7\u53ef\u901a\u8fc7\u4ee5\u4e0b\u914d\u7f6e\u8c03\u6574\u3002\u4ee5\u8c03\u6574 hadoop \u547d\u4ee4\u65e5\u5fd7\u914d\u7f6e\u4e3a\u4f8b\uff1a vim /etc/ecm/hadoop-conf/log4j.properties \u6dfb\u52a0\u4ee5\u4e0b\u914d\u7f6e\uff1a log4j.logger.com.aliyun.jindodata=WARN \u5982\u9700\u770b\u5230\u64cd\u4f5c\u65e5\u5fd7\uff0c\u5219\u518d\u6dfb\u52a0\u4ee5\u4e0b\u914d\u7f6e\uff1a log4j.logger.com.aliyun.jindodata=WARN log4j.logger.com.aliyun.jindodata.common.FsStats=INFO","title":"7. \u65e5\u5fd7\u7b49\u7ea7\u8c03\u6574"},{"location":"upgrade/emr_upgrade_smartdata/#_3","text":"\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4\u65f6\u53ef\u4ee5\u81ea\u52a8\u5347\u7ea7\u4fee\u590d\u3002\u5177\u4f53\u64cd\u4f5c\u6b65\u9aa4\u5982\u4e0b\uff1a","title":"\u573a\u666f\u4e8c\uff1a\u6269\u5bb9\u5df2\u6709\u96c6\u7fa4"},{"location":"upgrade/emr_upgrade_smartdata/#1_1","text":"\u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-6.7.3-linux.tar.gz \u548c bootstrap_jindosdk.sh\u3002 mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen $JINDOSDK_VERSION \u5982 bash bootstrap_jindosdk.sh -gen 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/hadoop/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230patch\u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002","title":"1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr_upgrade_smartdata/#2_1","text":"\u5c06 patch\u5305 \u548c bootstrap\u811a\u672c\u4e0a\u4f20\u5230OSS\u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 oss \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/jindo-patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u200b\u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002","title":"2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr_upgrade_smartdata/#3-emr","text":"\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"3. \u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c"},{"location":"upgrade/emr_upgrade_smartdata/#4_1","text":"\u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542fHive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002","title":"4. \u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d"},{"location":"upgrade/emr_upgrade_smartdata/#_4","text":"\u65b0\u5efaEMR\u96c6\u7fa4\u65f6\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\u3002\u5177\u4f53\u64cd\u4f5c\u6b65\u9aa4\u5982\u4e0b\uff1a","title":"\u573a\u666f\u4e09\uff1a\u65b0\u5efa\u96c6\u7fa4"},{"location":"upgrade/emr_upgrade_smartdata/#1_2","text":"\u4e0b\u8f7d\u7684 jindosdk-patches.tar.gz \uff0cjindosdk-6.7.3-linux.tar.gz \u548c bootstrap_jindosdk.sh\u3002 mkdir jindo-patch cd jindo-patch wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/6.7.3/jindosdk-6.7.3-linux.tar.gz wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/bootstrap_jindosdk.sh ls -l \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 bootstrap_jindosdk.sh -rw-r----- 1 hadoop hadoop xxxxxxxxx May 01 00:00 jindosdk-6.7.3-linux.tar.gz -rw-r----- 1 hadoop hadoop xxxx May 01 00:00 jindosdk-patches.tar.gz \u6267\u884c\u547d\u4ee4\u5236\u4f5c\u5347\u7ea7\u5305 bash bootstrap_jindosdk.sh -gen-full $JINDOSDK_VERSION \u5982 bash bootstrap_jindosdk.sh -gen-full 6.7.3 \u53c2\u6570\u8bf4\u660e\uff1a-gen\u751f\u6210lite\u5347\u7ea7\u5305\uff0c-gen-full\u8868\u793a\u751f\u6210\u5b8c\u6574\u5347\u7ea7\u5305\u3002 \u6210\u529f\u540e\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\uff1a Generated patch at /home/hadoop/jindo-patch/jindosdk-bootstrap-patches.tar.gz \u5236\u4f5c\u5b8c\u6210\uff0c\u5f97\u5230patch\u5305\uff1a jindosdk-bootstrap-patches.tar.gz \u3002","title":"1. \u5236\u4f5c\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr_upgrade_smartdata/#2_2","text":"\u5c06 patch\u5305 \u548c bootstrap\u811a\u672c\u4e0a\u4f20\u5230OSS\u4e0a\u3002 EMR \u96c6\u7fa4\u5185\u53ef\u4ee5\u901a\u8fc7 hadoop \u547d\u4ee4\u4e0a\u4f20\uff0c\u6216\u8005\u901a\u8fc7 oss \u63a7\u5236\u53f0\u3001ossutil \u6216 OSS Browser \u7b49\u5de5\u5177\u3002 hadoop dfs -mkdir -p oss://<bucket-name>/path/to/patch/ cd /home/hadoop/jindo-patch/ hadoop dfs -put jindosdk-bootstrap-patches.tar.gz oss://<bucket-name>/path/to/patch/ hadoop dfs -put bootstrap_jindosdk.sh oss://<bucket-name>/path/to/patch/ hadoop dfs -ls oss://<bucket-name>/path/to/patch/ \u200b\u53ef\u770b\u5230\u8fd4\u56de\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a Found 2 items -rw-rw-rw- 1 2634 2022-05-13 14:07 oss://<bucket-name>/.../bootstrap_jindosdk.sh -rw-rw-rw- 1 597342992 2022-05-13 13:41 oss://<bucket-name>/.../jindosdk-bootstrap-patches.tar.gz \u4f8b\u5982\uff0c\u4e0a\u4f20\u5230OSS\u7684\u8def\u5f84\u4e3a oss://<bucket-name>/path/to/jindosdk-bootstrap-patches.tar.gz \u548c oss://<bucket-name>/path/to/bootstrap_jindosdk.sh \u3002","title":"2. \u4e0a\u4f20\u5f15\u5bfc\u5347\u7ea7\u5305"},{"location":"upgrade/emr_upgrade_smartdata/#emr","text":"\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c\uff0c\u8be6\u7ec6\u4fe1\u606f\u8bf7\u53c2\u89c1 \u7ba1\u7406\u5f15\u5bfc\u64cd\u4f5c . \u5728 \u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c \u5bf9\u8bdd\u6846\u4e2d\uff0c\u586b\u5199\u914d\u7f6e\u9879\u3002 \u53c2\u6570 \u63cf\u8ff0 \u793a\u4f8b \u540d\u79f0 \u5f15\u5bfc\u64cd\u4f5c\u7684\u540d\u79f0\u3002\u4f8b\u5982\uff0c\u5347\u7ea7JINDOSDK\u3002 update_jindosdk \u811a\u672c\u4f4d\u7f6e \u9009\u62e9\u811a\u672c\u6240\u5728OSS\u7684\u4f4d\u7f6e\u3002\u811a\u672c\u8def\u5f84\u683c\u5f0f\u5fc5\u987b\u4e3aoss://* / .sh\u683c\u5f0f\u3002 oss:// /path/to/patch/bootstrap_jindosdk.sh \u53c2\u6570 \u5f15\u5bfc\u64cd\u4f5c\u811a\u672c\u7684\u53c2\u6570\uff0c\u6307\u5b9a\u811a\u672c\u4e2d\u6240\u5f15\u7528\u7684\u53d8\u91cf\u7684\u503c\u3002 -bootstrap oss:// /path/to/patch/jindosdk-bootstrap-patches.tar.gz \u6267\u884c\u8303\u56f4 \u9009\u62e9 \u96c6\u7fa4 \u3002 \u6267\u884c\u65f6\u95f4 \u9009\u62e9 \u7ec4\u4ef6\u542f\u52a8\u540e \u3002 \u6267\u884c\u5931\u8d25\u7b56\u7565 \u9009\u62e9 \u7ee7\u7eed\u6267\u884c \u3002","title":"\u5728EMR\u63a7\u5236\u53f0\u6dfb\u52a0\u5f15\u5bfc\u64cd\u4f5c"},{"location":"upgrade/emr_upgrade_smartdata/#_5","text":"\u5982\u679c\u662f\u65b0\u5efa\u96c6\u7fa4\uff0c\u5219\u9700\u8981\u91cd\u542fHive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002 \u5982\u679c\u662f\u6269\u5bb9\u65b0\u8282\u70b9\uff0c\u5219\u9700\u8981\u91cd\u542f\u5bf9\u5e94\u8282\u70b9\u4e0a\u7684 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark\u548cZeppelin\u7b49\u7ec4\u4ef6\u3002","title":"\u786e\u4fdd\u52a0\u8f7d\u5230\u6700\u65b0\u7684\u4fee\u590d"},{"location":"upgrade/emr_upgrade_smartdata/#jindosdk","text":"","title":"\u573a\u666f\u56db\uff1a\u56de\u6eda JindoSDK \u5230\u96c6\u7fa4\u521d\u59cb\u7248\u672c"},{"location":"upgrade/emr_upgrade_smartdata/#1_3","text":"\u767b\u5f55EMR\u96c6\u7fa4\u7684Master\u8282\u70b9\uff0c\u5e76\u5c06\u4e0b\u8f7d\u7684patch\u5305\u653e\u5728 hadoop \u7528\u6237\u7684HOME\u76ee\u5f55\u4e0b\uff0c\u5c06patch\u5305\u89e3\u538b\u7f29\u540e\uff0c\u4f7f\u7528 hadoop \u7528\u6237\u6267\u884c\u64cd\u4f5c\u3002 su - hadoop cd /home/hadoop/ wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/resources/jindosdk-patches.tar.gz tar zxf jindosdk-patches.tar.gz cd jindosdk-patches ls -l jindosdk-patches \u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a -rwxrwxr-x 1 hadoop hadoop 1263 May 01 00:00 apply_all.sh -rwxrwxr-x 1 hadoop hadoop 6840 May 01 00:00 apply.sh -rw-rw-r-- 1 hadoop hadoop 40 May 01 00:00 hosts -rwxrwxr-x 1 hadoop hadoop 1308 May 01 00:00 revert_all.sh -rwxrwxr-x 1 hadoop hadoop 6326 May 01 00:00 revert.sh","title":"1. \u51c6\u5907\u56de\u6eda\u811a\u672c"},{"location":"upgrade/emr_upgrade_smartdata/#2_3","text":"\u7f16\u8f91patch\u5305\u4e0b\u7684hosts\u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u7684host name\uff0c\u5982emr-header-1\u6216emr-worker-1\uff0c\u6587\u4ef6\u5185\u5bb9\u4ee5\u884c\u5206\u5272\u3002 cd jindosdk-patches vim hosts hosts\u6587\u4ef6\u5185\u5bb9\u793a\u4f8b\u5982\u4e0b\uff1a emr-header-1 emr-worker-1 emr-worker-2","title":"2. \u914d\u7f6e\u56de\u6eda\u8282\u70b9\u4fe1\u606f"},{"location":"upgrade/emr_upgrade_smartdata/#3_1","text":"\u901a\u8fc7revert_all.sh \u811a\u672c\u6267\u884c\u4fee\u590d\u64cd\u4f5c\u3002 ./revert_all.sh \u811a\u672c\u6267\u884c\u5b8c\u6210\u540e\uff0c\u8fd4\u56de\u5982\u4e0b\u63d0\u793a\u4fe1\u606f\u3002 >>> updating ... emr-header-1 >>> updating ... emr-worker-1 >>> updating ... emr-worker-2 ### DONE","title":"3. \u6267\u884c\u56de\u6eda"},{"location":"upgrade/emr_upgrade_smartdata/#4_2","text":"ls -l /opt/apps/extra-jars/ \u4ee5\u56de\u6eda\u4e3a 3.7.2 \u7248\u672c\u4e3a\u4f8b\uff0c\u8fd4\u56de\u793a\u4f8b\u5982\u4e0b\uff1a drwxr-xr-x 2 root root 4096 Apr 24 16:22 flink -rw-r--r-- 1 hadoop hadoop 189081 Apr 24 15:33 hadoop-lzo-0.4.21-SNAPSHOT.jar drwxr-xr-x 2 root root 4096 Apr 24 16:22 hive -rw-r--r-- 1 root root 1438552 Apr 24 16:22 jindotable-common.jar drwxr-xr-x 2 root root 4096 Apr 24 16:22 native -rw-r--r-- 1 root root 29389473 Apr 24 16:22 smartdata-jindofs-3.7.2.jar drwxr-xr-x 2 root root 4096 Apr 24 16:22 spark drwxr-xr-x 2 root root 4096 Apr 24 16:22 spark3","title":"4. \u786e\u8ba4\u56de\u6eda\u6210\u529f"},{"location":"upgrade/emr_upgrade_smartdata/#5_1","text":"\u8bf4\u660e \u5bf9\u4e8e\u5df2\u7ecf\u5728\u8fd0\u884c\u7684YARN\u4f5c\u4e1a\uff08Application\uff0c\u4f8b\u5982\uff0cSpark Streaming\u6216Flink\u4f5c\u4e1a\uff09\uff0c\u9700\u8981\u505c\u6b62\u4f5c\u4e1a\u540e\uff0c\u6279\u91cf\u6eda\u52a8\u91cd\u542fYARN NodeManager\u3002 Hive\u3001Presto\u3001Impala\u3001Druid\u3001Flink\u3001Solr\u3001Ranger\u3001Storm\u3001Oozie\u3001Spark \u548c Zeppelin \u7b49\u7ec4\u4ef6\u9700\u8981\u91cd\u542f\u4e4b\u540e\u624d\u80fd\u5b8c\u5168\u56de\u6eda\u3002 \u4ee5Hive\u7ec4\u4ef6\u4e3a\u4f8b\uff0c\u5728EMR\u96c6\u7fa4\u7684Hive\u670d\u52a1\u9875\u9762\uff0c\u9009\u62e9\u53f3\u4e0a\u89d2\u7684 \u64cd\u4f5c > \u91cd\u542f All Components \u3002","title":"5. \u56de\u6eda\u540e\u91cd\u542f\u670d\u52a1"}]}
\ No newline at end of file
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
index 881a0af88e2dd5f4db801be88edb11ef4856a859..d2077218e3d051f5136b0ec281195314d56634d6 100644
GIT binary patch
delta 12
Tcmb=gXOr*d;LvE8$W{pe7Kj6P

delta 12
Tcmb=gXOr*d;5b<^k*yK{8I1%G